网络数据采集并保存到数据库实例

【核心代码】

using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Data;
using System.Data.SqlClient;

namespace WebSpider
{
    class ChainNode : WebSpider.AbsChain
    {
        protected override void Process(string html)
        {
            try
            {
                   string title = string.Empty;
                Regex re = new Regex(@"href=(?<web_url>[\s\S]*?)>|href=""(?<web_url>[\s\S]*?)""|href='(?<web_url>[\s\S]*?)'");
                MatchCollection mc = re.Matches(html);
                foreach (Match m in mc)
                {
                    string url = m.Groups["web_url"].ToString();

                    //去除头部的'与"
                    if ((url.IndexOf("'") == 0) || (url.IndexOf("\"") == 0))
                    {
                        url = url.Remove(0, 1);
                        if (url.IndexOf("'") != -1)
                        {
                            url = url.Remove(url.IndexOf("'"), 1);
                        }
                        if (url.IndexOf("\"") != -1)
                        {
                            url = url.Remove(url.IndexOf("\""), 1);
                        }
                    }
                    if (url.IndexOf(" ") != -1)
                    {
                        url = url.Remove(url.IndexOf(" "));
                    }
                    if (url.IndexOf("http://") != -1)
                    {
                     
                        re = new Regex(@"<title>(?<title>[\s\S]*?)</title>");
                        Match temp = re.Match(html.ToLower());
                        title = temp.Groups["title"].ToString();
                        if (!string.IsNullOrEmpty(title))
                        {
                            AddUrl(url, title);
                            Console.WriteLine(url);
                        }
                       
                        UrlStack.Instance.Push(url);
                    }
                }
                Console.Write(this.Url "    " title);
                Console.WriteLine();
            }
            catch(Exception ex)
            {
                System.Windows.Forms.MessageBox.Show(ex.Message);
            }
        }

        private void AddUrl(string url,string title)
        {
            //using (SqlConnection conn = new SqlConnection())
            //{
            //    conn.ConnectionString = System.Configuration.ConfigurationManager.AppSettings["DB"];
            //    conn.Open();

            //    using (SqlCommand cmd = conn.CreateCommand())
            //    {
            //        cmd.CommandText = "AddWeb";
            //        cmd.CommandType = CommandType.StoredProcedure;
            //        cmd.Parameters.AddWithValue("@url", url);
            //        cmd.Parameters.AddWithValue("@title", title);

            //        cmd.ExecuteNonQuery();
            //    }
            //}
            SqlConnection cn = new SqlConnection("Data Source=192.168.0.187;Initial Catalog=SpiderDB;User ID=netcc;Password=fax99");
            cn.Open();
            SqlCommand cmd = new SqlCommand();
            cmd.Connection = cn;
            cmd.CommandType = CommandType.StoredProcedure;
            cmd.CommandText = "AddWeb";
            cmd.Parameters.AddWithValue("@url", url);
            cmd.Parameters.AddWithValue("@title", title);
            cmd.ExecuteNonQuery();
            cn.Close();
        }
    }

    class MyServer : WebSpider.AbsThreadManager
    {
        protected override AbsChain GetChainHeader()
        {
            return new ChainNode();
        }
    }

    class Program
    {
        static void Main(string[] args)
        {
            MyServer server = new MyServer();
            //server.Start("http://trims.fibre2fashion.com/beads/");
            server.Start("http://www.baidu.com");
            Console.Read();
            server.Stop();
        }      
    }
}