C# 网络蜘蛛采集邮箱 示例代码

【实例截图】

【核心代码】

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Diagnostics;
using FireCrawler.Lib;

namespace FireCrawler
{
    public partial class MainForm : Form
    {
        #region Fields

        private WebSiteDetector _detector;

        #endregion

        #region Properties

        // number of bytes downloaded
        private int nByteCount;
        private int ByteCount
        {
            get { return nByteCount; }
            set
            {
                nByteCount = value;
                this.statusStrip.Text = Commas(nByteCount / 1024   1)   " KB";
            }
        }

        // number of Uri's found
        private int nURLCount;
        private int URLCount
        {
            get { return nURLCount; }
            set
            {
                nURLCount = value;
                this.statusBarPanelURLs.Text = Commas(nURLCount)   " URL found";
            }
        }


        // number of files downloaded
        private int nFileCount;
        private int FileCount
        {
            get { return nFileCount; }
            set
            {
                nFileCount = value;
                this.statusBarPanelFiles.Text = Commas(nFileCount)   " file(s) downloaded";
            }
        }

        #endregion


        public MainForm()
        {
            InitializeComponent();

            dataGridThreads.AutoGenerateColumns = false;

            dataGridContacts.AutoGenerateColumns = false;

            _detector = new WebSiteDetector();

            _detector.StatusChanged  = new DetectorStatusChangedEventHandler(DetectorStatusChanged);
            _detector.ContactsFound  = new DetectorContactsFoundEventHandler(DetectorContactsFound);

           

        }





        #region Helpers

        string Commas(int nNum)
        {
            string str = nNum.ToString();
            int nIndex = str.Length;
            while (nIndex > 3)
            {
                str = str.Insert(nIndex - 3, ",");
                nIndex -= 3;
            }
            return str;
        }

        void ShowSettingsDialog()
        {
            SettingsForm dialog = new SettingsForm();
            if (dialog.ShowDialog() == DialogResult.OK)
            {
            }
        }

        #endregion

        #region UI Events


        private void exitToolStripMenuItem_Click(object sender, EventArgs e)
        {
            this.Close();
        }

        private void settingsToolStripMenuItem_Click(object sender, EventArgs e)
        {
            ShowSettingsDialog();
        }
        
        #endregion

        private void buttonGo_Click(object sender, EventArgs e)
        {
            var text = txtSeeds.Text.Trim();
            var seeds = text.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
            if (seeds == null || seeds.Length == 0)
            {   
                MessageBox.Show(this, "请输入搜索网址","提示");
                return;
            }

            if (_detector.Status == DetectorStatus.Start)
            {
                MessageBox.Show(this, "当前正在搜索中.....","提示");
                return;
            }

            var d = MessageBox.Show(this, "是否开启搜索任务?","启动提示", MessageBoxButtons.OKCancel);
            if (d == System.Windows.Forms.DialogResult.Cancel)
                return;

            for (int i = 0; i < seeds.Length; i  )
            {
                var item = seeds[i];
                if (item.ToLower().StartsWith("http:") == false)
                    item = "http://"   item;
                seeds[i] = item;
            }

            int layers = (int)Settings.Layers;

            _detector.InitSeeds(seeds,layers);
            _detector.Start();

            tabcontrol.SelectedIndex = 2;

        }
        
        private void buttonStop_Click(object sender, EventArgs e)
        {
            if (_detector.Status != DetectorStatus.Start)
                return;
            
            var d = MessageBox.Show(this, "是否关闭当前任务?", "关闭提示", MessageBoxButtons.OKCancel);
            if (d == System.Windows.Forms.DialogResult.Cancel)
                return;

            _detector.Abort();
        }

        delegate void UpdateDataGridCallback(WebSiteDetector d);

        private void UpdateDataGrid(WebSiteDetector d)
        {
            try
            {
                if (this.dataGridThreads.InvokeRequired)
                {
                    UpdateDataGridCallback callback = new UpdateDataGridCallback(UpdateDataGrid);
                    this.Invoke(callback, new object[] { d });
                }
                else
                {
                    dataGridThreads.DataSource = typeof(CrawlerThread[]);
                    dataGridThreads.DataSource = d.Crawlers;
                }
            }
            catch (ObjectDisposedException)
            {
            }
        }

        private void UpdateDataGridContactOutput(WebSiteDetector d)
        {
            try
            {
                if (this.dataGridContacts.InvokeRequired)
                {
                    UpdateDataGridCallback callback = new UpdateDataGridCallback(UpdateDataGridContactOutput);
                    this.Invoke(callback, new object[] { d });
                }
                else
                {
                    dataGridContacts.DataSource = typeof(ContactOutputItem[]);

                    dataGridContacts.DataSource = d.OutputItems.ToArray();
                }
            }
            catch (ObjectDisposedException)
            {
            }
        }

        delegate void UpdateStatusStripCallback();

        private void UpdateStatusStrip()
        {
            if (this.statusStrip.InvokeRequired)
            {
                UpdateStatusStripCallback callback = new UpdateStatusStripCallback(UpdateStatusStrip);
                this.Invoke(callback, new object[] { });
            }
            else
            {

                statusBarPanelURLs.Text = string.Format("待检测URL: {0}", _detector.UrlsQueueFrontier.Count.ToString());
                statusBarPanelFiles.Text = string.Format("已检测URL: {0}", _detector.CrawleHistroy.Count.ToString());

                statusBarPanelSpeed.Text = string.Format("速度: {0:0.00}KB/sec", _detector.GetDetectSpeed());

                statusBarPanelByteCount.Text = string.Format("流量: {0:0.00}MB", 1.0 * _detector.TotalSize / 1024 / 1024);
            }
        }

        private void DetectorStatusChanged(object sender, DetectorStatusChangedEventArgs e)
        {
            WebSiteDetector d = (WebSiteDetector)sender;
            UpdateDataGrid(d);

        }


        void DetectorContactsFound(object sender, DetectorContactsFoundEventArgs e)
        {
            WebSiteDetector d = (WebSiteDetector)sender;

            UpdateDataGridContactOutput(d);
        }

        private void timer_Tick(object sender, EventArgs e)
        {
            UpdateStatusStrip();
        }

        private void MainForm_FormClosing(object sender, FormClosingEventArgs e)
        {
            _detector.Abort();
            _detector.Dump("dump.txt");
        }

        private void aboutToolStripMenuItem_Click(object sender, EventArgs e)
        {
            AboutForm form = new AboutForm();
            form.ShowDialog(this);
        }

        private void btnBaidu_Click(object sender, EventArgs e)
        {   
            BaiduForm baiduForm = new BaiduForm(this);

            var d = baiduForm.ShowDialog(this);

            if(d == System.Windows.Forms.DialogResult.OK)
                buttonGo.PerformClick();
        }

        public void AcceptBaiduResult(string[] seeds)
        {
            string searchText = string.Empty;

            foreach (var seed in seeds)
            {
                searchText  = seed;
                searchText  = "\r\n";
            }

            txtSeeds.Text = searchText;
        }

        private void btnStart_Click(object sender, EventArgs e)
        {
            buttonGo.PerformClick();
        }

        private void btnExcel_Click(object sender, EventArgs e)
        {
            FireCrawler.Common.ExcelUtil.ExportDataGridViewToExcel(dataGridContacts);
        }


    }
}