c#网络爬虫-采集51job数据（html分析基于HtmlAgilityPack）

【实例截图】

【核心代码】

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.Threading;
namespace DataCollect
{
    public partial class MainForm : Form
    {
        public MainForm()
        {
            InitializeComponent();
        }

        List<Job> list = new List<Job>();//定义用于存放数据的集合
        CancellationTokenSource cts = new CancellationTokenSource();
        private void btnCollect_Click(object sender, EventArgs e)
        {
            cts.Cancel();
            //数据采集方法
            Task task1 = new Task(Collect);
            task1.Start();
           // Collect();
        }
        /// <summary>
        /// 数据采集
        /// </summary>
        private void Collect()
        {
            int index = 1;
            int pageCount = 1;
            do
            {
                var htmlDocument = DataLoad(index);
                if (index==1)
                {
                   HtmlNode hnNode= htmlDocument.DocumentNode.SelectSingleNode("//input[@id='hidTotalPage']");
                   pageCount=int.Parse(hnNode.Attributes["value"].Value);
                    this.listView1.Invoke(new Action(() => {
                        this.lblTotalPages.Text = pageCount.ToString();
                    }));
                    SetLabel(this.lblTotalPages, pageCount.ToString());
                }
                Thread.Sleep(2000);
                    //信息解析
                //var htmlDocument = DataLoad(1);//弱类型赋值

                HtmlNode hn = htmlDocument.DocumentNode.SelectSingleNode("//div[@class='el title']");
                //递归（循环检测还有没有兄弟元素）
                GetNode(hn.NextSibling);
                index  ;
                SetLabel(this.lblCurrentNo, index.ToString());
                SetLabel(this.lblItems, (50 * pageCount).ToString());
                
                //数据组合

            } while (index<pageCount);
        }
        private void SetLabel(Label lbl,string value)
        {
            lbl.Invoke(new Action(() =>
            {
                lbl.Text = value;
            }));
        }
        private void GetNode(HtmlNode parentNode)
        {
            Thread.Sleep(100);
            if(parentNode!=null)
            {
                if (parentNode.ChildNodes.Count > 0)
                {
                    Job job = new Job();
                    job.Position = parentNode.ChildNodes[1].InnerText.Replace("\r\n", "").Trim();//只取标签中间的内容
                    job.Company = parentNode.ChildNodes[3].InnerText;
                    job.Address = parentNode.ChildNodes[5].InnerText;
                    job.Salary = parentNode.ChildNodes[7].InnerText;
                    job.CreateTime = parentNode.ChildNodes[9].InnerText;
                    list.Add(job);//添加到集合

                    this.listView1.Invoke(new Action(()=> {
                        //创建控件对象
                        ListViewItem lvi = new ListViewItem();
                        lvi.Text = job.Position;
                        lvi.SubItems.AddRange(new string[] {job.Company, job.Address, job.Salary, job.CreateTime });
                        //数据添加到界面的控件
                        this.listView1.Items.Add(lvi);
                    }));
                    
                }
            }
            //判断到分页阶段
           if(parentNode.NextSibling.Attributes["class"] != null &&
                parentNode.NextSibling.Attributes["class"].Value == "dw_page")
            {
                return;//结束
            }


            GetNode(parentNode.NextSibling);//调用自己
        }
        private HtmlAgilityPack.HtmlDocument DataLoad(int _index)
        {
            HtmlWeb htmlWeb = new HtmlWeb();//创建Html解析对象
            htmlWeb.OverrideEncoding = Encoding.GetEncoding("gbk");
            string url = $"https://search.51job.com/list/120200,000000,0000,00,9,99,.net,2,{_index}.html";
            return htmlWeb.Load(url);
        }

        private void btnStop_Click(object sender, EventArgs e)
        {
            cts.Cancel();
        }
    }
}