HTMLAgilityPack + ScrapySharp 采集爱站关键词工具源码

【核心代码】

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ScrapySharp.Extensions;
using ScrapySharp.Network; 
using HtmlAgilityPack;
using System.Web;

namespace BaiduTools
{
    class Program
    {
        static void Main(string[] args)
        {
            string word = "张善友";
            string url = string.Format("http://ci.aizhan.com/{0}/", HttpUtility.UrlEncode(word));
            var uri = new Uri(url);
            var browser1 = new ScrapingBrowser();  
            var html1 = browser1.DownloadString(uri);
            var htmlDocument = new HtmlDocument();
            htmlDocument.LoadHtml(html1);
            var html = htmlDocument.DocumentNode;

            var content = html.CssSelect("div.box_05");

            foreach (var htmlNode in content)
            {
                //Console.WriteLine(htmlNode.InnerHtml);
                var query = from table in htmlNode.SelectNodes("//table").Cast<HtmlNode>()
                            from row in table.SelectNodes("tr").Cast<HtmlNode>()
                            from cell in row.SelectNodes("th|td").Cast<HtmlNode>()
                            select new { Table = table.Id, CellText = cell.InnerText, HTML = row.InnerHtml};

                List<BaiduKeyWord> baiduKeyWords = new List<BaiduKeyWord>();
                var list = query.ToList();
                int colCount = 7;
                int counts = list.Count / colCount;
                for (int i = 1; i < counts; i  )
                {
                    int start = i * colCount;
                    var keywords = list.GetRange(start, colCount);
                    BaiduKeyWord keyWord = new BaiduKeyWord();
                    for (int j = 0; j < colCount; j  )
                    {
                        if (j == 0)
                        {
                            keyWord.Id = Convert.ToInt32(keywords[j].CellText);
                        }
                        if (j == 1)
                        {
                            keyWord.KeyWord = keywords[j].CellText;
                        }
                        if (j == 2)
                        {
                            keyWord.SearchCount = Convert.ToInt32(keywords[j].CellText);
                        }
                        if (j == 3)
                        {
                            keyWord.IncludeCount = Convert.ToInt32(keywords[j].CellText);
                        }
                        if (j == 4)
                        {
                            keyWord.FirstItem = keywords[j].CellText;
                        }
                        if (j == 5)
                        {
                            keyWord.SecondItem = keywords[j].CellText;
                        }
                    }
                    baiduKeyWords.Add(keyWord);
                }

                foreach (var cell in baiduKeyWords)
                {
                    Console.WriteLine("{0}: {1}:{2}:{3}", cell.Id, cell.KeyWord, cell.SearchCount,cell.IncludeCount);
                }
            }
            Console.Read();

        }
    }
}