网络通用采集工具 完整源码下载

【实例简介】

Soukey采摘数据采集软件是一套面向个人站长、中小企业提供的一套免费开源的数据采集软件，可用于文章、论坛、博、企业信息等各种网站信息的采集，同时可 自由发布到文件、数据库，通知也支持Web在线发布功能。同时，如果您了解技术，还可在Soukey采摘基础上扩展更加丰富的功能，包括：舆情监控、RSS制作、行业化 数据采集等功能。 Soukey采摘数据采集软件使用不受任何限制，源代码使用请遵循BSD许可协议。

1、多任务、多线程，支持一个任务运行多个实例； 

2 、支持图片、Flash及文件下载； 

3、网址配置支持参数定制，及外部字典参数； 

4、支持Cookie、POST采集； 

5、支持导航、自动翻页； 

6 、可采集Ajax数据； 

7、采集数据临时存储、断点续采； 

8 、支持数据导出，文件、数据库； 

9 、在线数据发布； 

10 、数据采集支持采集数据加工； 

11 、支持任务计划； 

12 、支持任务触发器； 

13、提供网址编/解码工具； 

14 、提供Mini浏览器； 

15、支持简体中文及英文；

【核心代码】

using System;
using System.Collections.Generic;
using System.Text;
using SoukeyNetget.Plan;
using System.Data;
using SoukeyNetget.Task;
using System.Data.OleDb;
using System.Data.SqlClient;
using MySql.Data.MySqlClient;

namespace SoukeyNetget.Listener
{
    class cRunTask
    {
        
        Queue<cTaskPlan> m_runningTasks;

        public cRunTask()
        {
            m_runningTasks = new Queue<cTaskPlan>();
        }

        ~cRunTask()
        {
        }

        public void AddTask(cTaskPlan task)
        {
            m_runningTasks.Enqueue(task);
            
            RunTask();
        }

        //Ö´ÐÐÈÎÎñ
        private void RunTask()
        {
            cPlanRunLog rLog = new cPlanRunLog();
            rLog.OpenLogFile();

            while (m_runningTasks.Count > 0)
            {
                cTaskPlan tp = m_runningTasks.Dequeue();

                switch (tp.RunTaskType)
                {
                    case (int)cGlobalParas.RunTaskType.SoukeyTask :
                        RunSoukeyTask(tp.RunTaskName);
                        break ;
                    case (int)cGlobalParas.RunTaskType.OtherTask :
                        RunOtherTask(tp.RunTaskName, tp.RunTaskPara);
                        break ;
                    case (int)cGlobalParas.RunTaskType.DataTask :
                        RunDataTask(tp.RunTaskName, tp.RunTaskPara);
                        break;
                }

                //дÈÕÖ¾
                rLog.InsertLog(cGlobalParas.LogType.RunPlanTask,tp.PlanID ,tp.PlanName , (cGlobalParas.RunTaskType)tp.RunTaskType, tp.RunTaskName, tp.RunTaskPara);
               
            }

            rLog.CloseLogFile();
            rLog = null;

        }

        //Èç¹ûÊÇSoukey²ÉÕªÈÎÎñÔòÐèÒª½«ÐÅÏ¢·´À¡µ½Ö÷½çÃ棬ÓÉÖ÷½çÃæÔËÐÐ
        //µ±Ç°Ö§³ÖÁÙʱ·½°¸£¬ºóÆÚ»áÌṩרÃÅÔËÐÐSoukeyÈÎÎñµÄÒýÇ棬¿ÉÒÔ½øÐÐ
        //ºǫ́ÔËÐÐSoukey²ÉÕªÈÎÎñ¡£

        private void RunDataTask(string DataType, string Para)
        {
            switch (int.Parse(DataType))
            {
                case (int)cGlobalParas.DatabaseType.Access :
                    ExecuteAccessQuery(Para);
                    break;
                case (int)cGlobalParas.DatabaseType.MSSqlServer:
                    ExecuteMSSqlQuery(Para);
                    break;
                case (int)cGlobalParas.DatabaseType.MySql:
                    ExecuteMySqlQuery(Para);
                    break;
                default :
                    break;
            }
        }

        private void ExecuteAccessQuery(string Para)
        {

            string strconn = Para.Substring(0, Para.IndexOf("Para"));
            string QueryName = Para.Substring(Para.IndexOf("Para=")   5, Para.Length - Para.IndexOf("Para=") - 5);

            OleDbConnection conn = new OleDbConnection();
            conn.ConnectionString = strconn;

            try
            {
                conn.Open();
            }
            catch (System.Exception ex)
            {
                throw ex;
            }

            OleDbCommand com = new OleDbCommand();
            com.Connection = conn;
            com.CommandText = QueryName;
            com.CommandType = CommandType.StoredProcedure;
            try
            {
                int result = com.ExecuteNonQuery();
            }
            catch (System.Exception ex)
            {
                throw ex;
            }

            conn.Close();

           
        }

        private void ExecuteMSSqlQuery(string Para)
        {
            string strconn = Para.Substring(0, Para.IndexOf("Para"));
            string QueryName = Para.Substring(Para.IndexOf("Para=") 5, Para.Length - Para.IndexOf("Para=")-5);

            SqlConnection conn = new SqlConnection();
            conn.ConnectionString = strconn;

            try
            {
                conn.Open();
            }
            catch (System.Exception ex)
            {
                throw ex;
            }

            SqlCommand com = new SqlCommand();
            com.Connection = conn;
            com.CommandText = QueryName;
            com.CommandType = CommandType.StoredProcedure;
            try
            {
                int result = com.ExecuteNonQuery();
            }
            catch (System.Exception ex)
            {
                throw ex;
            }

            conn.Close();
        }

        private void ExecuteMySqlQuery(string Para)
        {

            string strconn = Para.Substring(0, Para.IndexOf("Para"));
            string QueryName = Para.Substring(Para.IndexOf("Para=")   5, Para.Length - Para.IndexOf("Para=") - 5);

            MySqlConnection conn = new MySqlConnection();
            conn.ConnectionString = strconn;

            try
            {
                conn.Open();
            }
            catch (System.Exception ex)
            {
                throw ex;
                
            }

            MySqlCommand com = new MySqlCommand();
            com.Connection = conn;
            com.CommandText = QueryName;
            com.CommandType = CommandType.StoredProcedure;
            try
            {
                int result = com.ExecuteNonQuery();
            }
            catch (System.Exception ex)
            {
                throw ex;
            }

            conn.Close();
          
        }

        private void RunSoukeyTask(string TaskName)
        {
            //»ñÈ¡TaskID
            //cTaskClass tClass = new cTaskClass();
            //string tClassPath=tClass.GetTaskClassPathByName();
            //tClass = null;

            //string tName = tClassPath   "\\"   TaskName.Substring(TaskName.IndexOf("\\"), TaskName.Length - TaskName.IndexOf("\\"));

            //cTask t = new cTask();
            //t.LoadTask(tName);
            //Int64 tID = t.TaskID;
            //t = null;

            //´¥·¢ÔËÐÐÈÎÎñ½øÐÐÈÎÎñµÄÖ´ÐвÙ×÷
            e_RunSoukeyTaskEvent(this, new cRunTaskEventArgs(cGlobalParas.MessageType.RunSoukeyTask, TaskName ,""));

        }

        //Èç¹ûÊÇÆäËûÈÎÎñ£¬ÔòÖ±½ÓÔËÐÐ
        private void RunOtherTask(string FileName, string Para)
        {
             System.Diagnostics.Process.Start (FileName, Para);
        }

        private readonly Object m_eventLock = new Object();

        #region ʼþ

        /// ²É¼¯ÈÎÎñÍê³Éʼþ
        private event EventHandler<cRunTaskEventArgs> e_RunSoukeyTaskEvent;
        internal event EventHandler<cRunTaskEventArgs> RunSoukeyTaskEvent
        {
            add { lock (m_eventLock) { e_RunSoukeyTaskEvent  = value; } }
            remove { lock (m_eventLock) { e_RunSoukeyTaskEvent -= value; } }
        }

        #endregion
    }
}