批量获取网站标题

【实例简介】

批量获取网站的标题

【实例截图】

【核心代码】

using HttpCodeLib;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Windows.Forms;

namespace 批量获取网站标题
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }
        protected List<ListViewItem> ItemsSource
        {
            get;
            private set;
        }

        protected List<ListViewItem> CurrentCacheItemsSource
        {
            get;
            private set;
        }
        private void button2_Click(object sender, EventArgs e)
        {
            stop = 0;
            logg = 0;
            if (listView1.Items.Count > 0)
            {
                button2.Enabled = false;
                button5.Enabled = false;
                button1.Enabled = false;
                button3.Enabled = false;
                button4.Enabled = false;
                button6.Visible = true;
                button7.Visible = true;
                progressBar1.Value = 0;
                progressBar1.Visible = true;
                progressBar1.Maximum = listView1.Items.Count;

                //ThreadPool.SetMaxThreads(20, 20);//允许线程池中运行最多 20 个线程
               

                for (int i = 0; i < listView1.Items.Count&&stop==0; i  )
                {
                    Thread th = new Thread(get_biaoti);
                    th.IsBackground = true;
                    string url = listView1.Items[i].SubItems[1].Text;
                    List<string> list = new List<string>();
                    list.Add(i.ToString());
                    list.Add(url);
                    th.Start(list);//可以执行进程
                    thCount  ;
                    while (thCount >= maxThCount||logg==1)
                    {
                        Thread.Sleep(10);
                        Application.DoEvents();
                    }

                  // ThreadPool.QueueUserWorkItem(new WaitCallback(get_biaoti), list);
                  
                }

               
            }

        }
        int thCount = 0;
        int maxThCount = 20;

        int logg = 0;
        int stop = 0;


        /// <summary>
        /// 获取标题
        /// </summary>
        /// <param name="obj">页面内容</param>
    
        private void get_biaoti(object obj)
        {

            int i = (int.Parse((obj as List<string>)[0]));
            string url = (obj as List<string>)[1];
            string res = string.Empty;
            if (jiekou == 1)
            {

                //string url=obj.ToString();
                //string res = get_html(url);
                url = string.Format(@"http://www.link114.cn/get.php?title&{0}&398666685656", url);
                 res = get_html(url);
                //string result = GetRegexStr(res);
                string[] result = res.Split(new char[] { ':' });
                if (result.Length == 0)
                    res = "超时";
                if (result.Length == 2)
                {
                    res = result[1];
                    if (res == "-1")
                        res = "网址错误";
                    if (res == "")
                        res = "无法访问";
                }
            }
            if (jiekou == 2)
            {
                res = get_html(url);
                string result = GetRegexStr(res);
                if (result.Length == 0)
                    res = "超时";
                else
                    res = result;
 
            }




            thCount--;
            //Thread.Sleep(300);//试试等待来响应
            this.BeginInvoke(new Action(() => ThreadCallBack(i, res)));


        }
        private void ThreadCallBack(int i, string str)
        {

            //int MaxWorkerThreads, miot, AvailableWorkerThreads, aiot;
            //ThreadPool.GetMaxThreads(out MaxWorkerThreads, out miot);
            //AvailableWorkerThreads = aiot = 0;
            //获得可用的线程数量
            //ThreadPool.GetAvailableThreads(out AvailableWorkerThreads, out aiot);
            //label2.Text = (MaxWorkerThreads - AvailableWorkerThreads).ToString();
            label2.Text = thCount.ToString();
            progressBar1.PerformStep();
            this.listView1.Items[i].SubItems[2].Text = str.ToString();//修改状态
            // listView1.Refresh();
            listView1.Invalidate();
            //this.ItemsSource = items;
            //this.CurrentCacheItemsSource = this.ItemsSource;
            //LoadListViewItems(this.CurrentCacheItemsSource);
            if (progressBar1.Value == progressBar1.Maximum)
            {
                progressBar1.Visible = false;
                button2.Enabled = true;
                button5.Enabled = true;
                button1.Enabled = true;
                button3.Enabled = true;
                button4.Enabled = true;
                button6.Visible = false;
                button7.Visible = false;
            }
            //if ((MaxWorkerThreads - AvailableWorkerThreads) == 0)
            //    MessageBox.Show("执行完毕");


            listView1.Items[i].EnsureVisible();
            


        }






       /// <summary>
       /// 获取网页内容
       /// </summary>
       /// <param name="url">网址</param>
       /// <returns></returns>
        public string get_html(string url)
        {
            try
            {
                HttpHelpers helper = new HttpHelpers();//请求执行对象
                HttpItems items;//请求参数对象
                HttpResults hr = new HttpResults();//请求结果对象

                string StrCookie = "";//设置初始Cookie值
                string res = string.Empty;//请求结果,请求类型不是图片时有效
                // string url = "www.baidu.com";//请求地址
                items = new HttpItems();//每次重新初始化请求对象
                // items.Timeout = 1000;
                items.URL = url;//设置请求地址
                items.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:17.0) Gecko/20100101 Firefox/17.0";//设置UserAgent
                items.Cookie = StrCookie;//设置字符串方式提交cookie
                items.Allowautoredirect = true;//设置自动跳转(True为允许跳转) 如需获取跳转后URL 请使用 hr.RedirectUrl
                items.ContentType = "application/x-www-form-urlencoded";//内容类型
                hr = helper.GetHtml(items, ref StrCookie);//提交请求

                res = hr.Html;//具体结果
                return res;//返回具体结果
            }
            catch (Exception e)
            {
                return e.ToString();
            }
        }
        ///*调用方法: 直接粘贴内容至Code中,调用GetRegexStr("这里填写要处理的字符串")*/
        System.Text.RegularExpressions.Regex reg;//正则表达式变量

        ///// <summary>
        ///// 正则表达式获取文本结果
        ///// </summary>
        ///// <param name="reString">请替换为需要处理的字符串</param>
        ///// <returns>处理结果</returns>
        private string GetRegexStr(string reString)
        {

            string regexCode = "(?<=<title>).*?(?=</title>)";
            reg = new System.Text.RegularExpressions.Regex(regexCode);
            string math = reg.Match(reString).ToString();
            return math;
        }
        int i = 1;

        private void button3_Click(object sender, EventArgs e)
        {

            if (textBox1.Text != "")
            {
                ListViewItem item = null;
                item = new ListViewItem();
                item.SubItems[0].Text = (i  ).ToString();
                item.SubItems.Add(textBox1.Text.ToString());
                item.SubItems.Add("等待检测");

                items.Add(item);
                this.ItemsSource = items;
                this.CurrentCacheItemsSource = this.ItemsSource;
                LoadListViewItems(this.CurrentCacheItemsSource);
                listView1.Items[i - 2].EnsureVisible();
                //listView1.setSelection(listView1.getBottom());  


            }
        }

        private void Form1_FormClosing(object sender, FormClosingEventArgs e)
        {
            //Application.Exit();
            System.Environment.Exit(0);

        }
        //int thCount = 0;
        //int maxThCount = 10;
        List<ListViewItem> items = new List<ListViewItem>();
        private void daoru(object fName)
        {

            ListViewItem item = null;
            string[] str_ = (string[])fName;
            foreach (string file in str_)
            {
                string[] str = File.ReadAllLines(file, Encoding.ASCII);
                foreach (string url in str)
                {
                    item = new ListViewItem();
                    item.Text = (i  ).ToString();
                    item.SubItems.Add(url.ToString());
                    item.SubItems.Add("等待检测");
                    items.Add(item);





                }
                this.BeginInvoke(new Action(() => ThreadCallBack_2()));
                //listView1.BeginUpdate();
                //listView1.EndUpdate();
            }
            //}), null);


        }
        private void ThreadCallBack_2()
        {
            this.ItemsSource = items;
            this.CurrentCacheItemsSource = this.ItemsSource;
            LoadListViewItems(this.CurrentCacheItemsSource);



            //listView1.VirtualMode = false;

        }
        private void button1_Click(object sender, EventArgs e)
        {

            if (openFileDialog1.ShowDialog() == DialogResult.OK)
            {


                string[] fName = openFileDialog1.FileNames;



                ThreadPool.QueueUserWorkItem(new WaitCallback(daoru), fName);//线程池

            }


        }




        /// <summary>
        /// 导出数据到txt文件.<br></br>
        /// 2009-04-24 YJ 定义函数.<br></br>
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button4_Click(object sender, EventArgs e)
        {
            if (listView1.Items.Count > 0)
            {
                //此处的文本文件在工程下Bin的程序集目录下
                string stFilePath = Application.StartupPath.Trim()   "//导出文件"   DateTime.Now.ToString("yyyy年MM月dd日hh时mm分ss秒")   ".txt";
                StreamWriter swStream;
                if (File.Exists(stFilePath))
                {
                    swStream = new StreamWriter(stFilePath);
                }
                else
                {
                    swStream = File.CreateText(stFilePath);
                }

                for (int i = 0; i < listView1.Items.Count; i  )
                {
                    for (int j = 0; j < listView1.Items[i].SubItems.Count; j  )
                    {
                        string _strTemp = listView1.Items[i].SubItems[j].Text;
                        swStream.Write(_strTemp);
                        //插入"<----->"作为分隔符,可以任取
                        if (j == 0)
                            swStream.Write(".");
                        if (j == 1)
                            swStream.Write("<----->");
                    }
                    swStream.WriteLine();
                }

                //关闭流,释放资源
                swStream.Flush();
                swStream.Close();
                //导入Txt文件后,自动打开文件
                //Process.Start("notepad.exe", stFilePath);
            }
            else
                MessageBox.Show("列表为空，无法导出", "错误");
        }

        private void Form1_Load(object sender, EventArgs e)
        {
            //this.listView1.ListViewItemSorter = new Common.ListViewColumnSorter();
            //this.listView1.ColumnClick  = new ColumnClickEventHandler(Common.ListViewHelper.ListView_ColumnClick);
            radioButton1.Checked=true;
            Control.CheckForIllegalCrossThreadCalls = false;



        }
        private void LoadListViewItems(List<ListViewItem> items)
        {
            //listView.Items.Clear();
            //if (items == null)
            //{
            //    //stripStatusInfo.Text = "当前总共记录数为：0";
            //    return;
            //}

            // listView1.GridLines = true;
            listView1.FullRowSelect = true;
            //listView1.View = View.Details;
            //listView1.Scrollable = true;
            //listView1.MultiSelect = false;
            // listView1.HeaderStyle = ColumnHeaderStyle.Clickable;
            //listView1.Visible = true;

            listView1.VirtualListSize = items.Count;
            listView1.VirtualMode = true;
            listView1.RetrieveVirtualItem  = new RetrieveVirtualItemEventHandler(listView_RetrieveVirtualItem);

            // stripStatusInfo.Text = "当前总共记录数为："   items.Count;
        }
        void listView_RetrieveVirtualItem(object sender, RetrieveVirtualItemEventArgs e)
        {
            if (this.CurrentCacheItemsSource == null || this.CurrentCacheItemsSource.Count == 0)
            {
                return;
            }

            e.Item = this.CurrentCacheItemsSource[e.ItemIndex];
            if (e.ItemIndex == this.CurrentCacheItemsSource.Count)
            {
                this.CurrentCacheItemsSource = null;
            }
        }
        private void button5_Click(object sender, EventArgs e)
        {

            items.Clear();
            this.ItemsSource = items;
            this.CurrentCacheItemsSource = this.ItemsSource;
            LoadListViewItems(this.CurrentCacheItemsSource);
            i = 1;
        }

        private void button6_Click(object sender, EventArgs e)
        {
            //logg = true;
            //ThreadPool.SetMaxThreads(0, 0);
        }

        private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
        {

        }
        int jiekou = 1;
        private void radioButton1_CheckedChanged(object sender, EventArgs e)
        {
            if (radioButton1.Checked)
                jiekou = 1;
            else
                jiekou = 2;
        }

        private void button6_Click_1(object sender, EventArgs e)
        {
            //Thread.Sleep(0);

            if (button6.Text == "暂  停")
            {
                logg = 1;//暂停标志位
                button6.Text = "继  续";
            }
            else
            {
                logg = 0;//暂停标志位
                button6.Text = "暂  停";
 
            }

        }

        private void button7_Click(object sender, EventArgs e)
        {
            stop = 1;
            label2.Text = "0";
            progressBar1.Visible = false;
            button2.Enabled = true;
            button5.Enabled = true;
            button1.Enabled = true;
            button3.Enabled = true;
            button4.Enabled = true;
            button6.Text = "暂  停";
            button6.Visible = false;
            button7.Visible = false;
           

        }




    }
}