C# 网络爬虫源码

【实例截图】

【核心代码】

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using NWebCrawlerLib;
using System.Diagnostics;

namespace NWebCrawler
{
    public partial class MainForm : Form
    {
        #region Fields

        private PerformanceCounter m_cpuCounter;
        private PerformanceCounter m_ramCounter;
        private Downloader m_downloader;

        #endregion

        #region Properties

        // number of bytes downloaded
        private int nByteCount;
        private int ByteCount
        {
            get { return nByteCount; }
            set
            {
                nByteCount = value;
                this.statusStrip.Text = Commas(nByteCount / 1024   1)   " KB";
            }
        }

        // number of Uri's found
        private int nURLCount;
        private int URLCount
        {
            get { return nURLCount; }
            set
            {
                nURLCount = value;
                this.statusBarPanelURLs.Text = Commas(nURLCount)   " URL found";
            }
        }

        // available memory
        private float nFreeMemory;
        private float FreeMemory
        {
            get { return nFreeMemory; }
            set
            {
                nFreeMemory = value;
                this.statusBarPanelMem.Text = nFreeMemory   " MB Available";
            }
        }

        // CPU usage
        private int nCPUUsage;
        private int CPUUsage
        {
            get { return nCPUUsage; }
            set
            {
                nCPUUsage = value;
                this.statusBarPanelCPU.Text = "CPU usage "   nCPUUsage   "%";
            }
        }

        // number of files downloaded
        private int nFileCount;
        private int FileCount
        {
            get { return nFileCount; }
            set
            {
                nFileCount = value;
                this.statusBarPanelFiles.Text = Commas(nFileCount)   " file(s) downloaded";
            }
        }

        #endregion


        public MainForm()
        {
            InitializeComponent();

            m_cpuCounter = new PerformanceCounter();

            m_cpuCounter.CategoryName = "Processor";
            m_cpuCounter.CounterName = "% Processor Time";
            m_cpuCounter.InstanceName = "_Total";

            m_ramCounter = new PerformanceCounter("Memory", "Available MBytes");

            m_downloader = new Downloader();
            m_downloader.StatusChanged  = new DownloaderStatusChangedEventHandler(DownloaderStatusChanged);
        }

        #region Helpers

        string Commas(int nNum)
        {
            string str = nNum.ToString();
            int nIndex = str.Length;
            while (nIndex > 3)
            {
                str = str.Insert(nIndex - 3, ",");
                nIndex -= 3;
            }
            return str;
        }

        void ShowSettingsDialog()
        {
            SettingsForm dialog = new SettingsForm();
            if (dialog.ShowDialog() == DialogResult.OK)
            {
            }
        }

        #endregion

        #region UI Events


        private void exitToolStripMenuItem_Click(object sender, EventArgs e)
        {
            this.Close();
        }

        private void settingsToolStripMenuItem_Click(object sender, EventArgs e)
        {
            ShowSettingsDialog();
        }

        private void buttonResume_Click(object sender, EventArgs e)
        {
            m_downloader.Resume();
        }

        private void buttonPause_Click(object sender, EventArgs e)
        {
            m_downloader.Suspend();
        }

        private void buttonStop_Click(object sender, EventArgs e)
        {
            m_downloader.Abort();
        }

        private void buttonSettings_Click(object sender, EventArgs e)
        {
            ShowSettingsDialog();
        }

        #endregion

        private void buttonGo_Click(object sender, EventArgs e)
        {
            m_downloader.InitSeeds(new string[] { txtSeed.Text });
            m_downloader.Start();
        }

        delegate void UpdateDataGridCallback(Downloader d);

        private void UpdateDataGrid(Downloader d)
        {
            try
            {
                if (this.dataGridThreads.InvokeRequired)
                {
                    UpdateDataGridCallback callback = new UpdateDataGridCallback(UpdateDataGrid);
                    this.Invoke(callback, new object[] { d });
                }
                else
                {
                    dataGridThreads.DataSource = typeof(CrawlerThread[]);
                    dataGridThreads.DataSource = d.Crawlers;
                    dataGridThreads.Columns["Name"].Width = 100;
                    dataGridThreads.Columns["Status"].Width = 100;
                }
            }
            catch (ObjectDisposedException)
            {
            }
        }

        delegate void UpdateStatusStripCallback();

        private void UpdateStatusStrip()
        {
            if (this.statusStrip.InvokeRequired)
            {
                UpdateStatusStripCallback callback = new UpdateStatusStripCallback(UpdateStatusStrip);
                this.Invoke(callback, new object[] { });
            }
            else
            {
                statusBarPanelCPU.Text = string.Format("CPU: {0:0.00}%", m_cpuCounter.NextValue());
                statusBarPanelMem.Text = string.Format("Mem: {0:0.00}MB", m_ramCounter.NextValue());
                statusBarPanelURLs.Text = string.Format("URLs: {0}", m_downloader.UrlsQueueFrontier.Count.ToString());
                statusBarPanelFiles.Text = string.Format("Files: {0}", m_downloader.CrawleHistroy.Count.ToString());
                statusBarPanelSpeed.Text = string.Format("Speed: {0:0.00}KB/sec", m_downloader.GetDownloadSpeed());
                statusBarPanelByteCount.Text = string.Format("Total size: {0:0.00}MB", 1.0 * m_downloader.TotalSize / 1024 / 1024);
            }
        }

        private void DownloaderStatusChanged(object sender, DownloaderStatusChangedEventArgs e)
        {
            Downloader d = (Downloader)sender;
            UpdateDataGrid(d);

        }

        private void timer_Tick(object sender, EventArgs e)
        {
            UpdateStatusStrip();
        }

        private void MainForm_FormClosing(object sender, FormClosingEventArgs e)
        {
            m_downloader.Abort();
            m_downloader.Dump("dump.txt");
        }
    }
}