Java+Jsoup爬虫小红书微博B站

【核心代码】

package com.example.demo;

import com.alibaba.fastjson.JSONObject;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@SpringBootTest
class DemoApplicationTests {

    /**
     * Sam
     * 643003372@qq.com
     * 2020-06-05 11:42
     * 详细说明：https://baidu.com
     */
    @Test
    void main() throws Exception {
        System.out.println("***********小红书***********");
        this.xiaohongshu();
        System.out.println("***********微博***********");
        this.weibo();
        System.out.println("***********B站***********");
        this.bilibili();
    }

    @Test
    void xiaohongshu() throws Exception {
        Document doc = Jsoup.connect("https://www.xiaohongshu.com/discovery/item/5e92cdf70000000001009b42").get();
        Elements like = doc.select(".operation-block .like span");
        Elements comment = doc.select(".operation-block .comment span");
        Elements star = doc.select(".operation-block .star span");
        System.out.println("小红书点赞量："   like.get(0).html().toString());
        System.out.println("小红书评论量："   comment.get(0).html().toString());
        System.out.println("小红书收藏量："   star.get(0).html().toString());
    }

    @Test
    void weibo() throws Exception {
        String tidResponse = this.get("https://passport.weibo.com/visitor/genvisitor?cb=gen_callback", null);
//        System.out.println("获取tid数据为:"   tidResponse);
        tidResponse = tidResponse.replaceAll("window.gen_callback && gen_callback\\(", "");
        tidResponse = tidResponse.replaceAll("\\);", "");
        JSONObject tidDataJson = JSONObject.parseObject(tidResponse);
        JSONObject tidData = (JSONObject) tidDataJson.get("data");

        String subAndSubsResponse = this.get("https://passport.weibo.com/visitor/visitor?a=incarnate&t="  URLEncoder.encode(tidData.get("tid").toString(), "UTF-8")  "&w="  (Boolean.valueOf(tidData.get("new_tid").toString())?3:2)  "&c=100&cb=cross_domain&from=weibo", null);
//        System.out.println("获取sub和subp数据为:"   subAndSubsResponse);
        subAndSubsResponse = subAndSubsResponse.replaceAll("window.cross_domain && cross_domain\\(", "");
        subAndSubsResponse = subAndSubsResponse.replaceAll("\\);", "");
        JSONObject subAndSubsDataJson = JSONObject.parseObject(subAndSubsResponse);
        JSONObject subAndSubsData = (JSONObject) subAndSubsDataJson.get("data");

        //获取微博HTML页面
        String Cookie = "SUB="   URLEncoder.encode(subAndSubsData.get("sub").toString(), "UTF-8")   "; SUBP="   URLEncoder.encode(subAndSubsData.get("subp").toString(), "UTF-8")   ";";
        Connection connect = Jsoup.connect("https://weibo.com/1234692083/Ixnp6nuPk");
        connect.header("Cookie", Cookie);

        String html = "";
        Matcher m = Pattern.compile("FM.view\\(.*\\)").matcher(connect.get().html());
        while (m.find()){
            String val = m.group(0);
            val = val.replaceAll("FM.view\\(", "");
            val = val.replaceAll("\\)", "");
            JSONObject json = JSONObject.parseObject(val);
            if("pl.content.weiboDetail.index".equals(json.get("ns"))){
                html = json.get("html").toString();
            }
        }

//        System.out.println(html);
        Document doc = Jsoup.parseBodyFragment(html);
        List<String> list = new ArrayList<>();
        for (Element e : doc.select(".WB_row_line span.S_line1 em:nth-child(2)")) {
            list.add(e.html());
        }

        System.out.println("微博转发量："   list.get(1));
        System.out.println("微博评论量："   list.get(2));
        System.out.println("微博点赞量："   list.get(3));
    }

    @Test
    void bilibili() throws Exception {
        Document doc = Jsoup.connect("https://www.bilibili.com/video/BV1HE411b7nj").get();
        Matcher m = Pattern.compile("window.__INITIAL_STATE__=\\{.*\\}};").matcher(doc.html());
        String aid = "";
        while (m.find()){
            String val = m.group(0);
            val = val.replaceAll("window.__INITIAL_STATE__=\\{", "{");
            val = val.replaceAll("\\}};", "}}");
            JSONObject json = JSONObject.parseObject(val);
            aid = json.get("aid").toString();
        }
        String response = this.get("https://api.bilibili.com/x/web-interface/archive/stat?aid=" aid, null);
//        System.out.println("根据aid  获取数据为："   response);
        JSONObject json = JSONObject.parseObject(response);
        JSONObject data = (JSONObject) json.get("data");
        System.out.println("B站点赞量："   data.get("like"));
        System.out.println("B站投币量："   data.get("coin"));
        System.out.println("B站收藏量："   data.get("favorite"));
        System.out.println("B站转发量："   data.get("share"));
    }

    public String get(String url, String cookie) throws Exception {
        CloseableHttpClient client = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);
        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(5000).setConnectionRequestTimeout(5000).setSocketTimeout(5000).build();
        httpGet.setConfig(requestConfig);
        httpGet.setHeader("Cookie", cookie);
        CloseableHttpResponse response = client.execute(httpGet);
        if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
            HttpEntity resEntity = response.getEntity();
            return EntityUtils.toString(resEntity);
        }
        return "";
    }

}