实例介绍
【实例简介】
【实例截图】
【核心代码】
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Text.RegularExpressions;
using System.Collections;
using System.Data;
using System.Net;
using System.Runtime.InteropServices;
using Wesley.Crawler.SimpleCrawler.Models;
using System.Diagnostics;
namespace Wesley.Crawler.SimpleCrawler
{
class Program
{
static void Main1(string[] args)
{
//测试代理IP是否生效：http://1212.ip138.com/ic.asp
//测试当前爬虫的User-Agent：http://www.whatismyuseragent.net
//1.抓取城市
//CityCrawler();
//2.抓取酒店
Stopwatch sw = new Stopwatch();
sw.Start();
HotelCrawler();
sw.Stop();
TimeSpan ts2 = sw.Elapsed;
string a = string.Format("Stopwatch总共花费{0}ms.", ts2.TotalMilliseconds);
Console.WriteLine("携程北京耗时：" ts2.TotalMilliseconds * 1000/60 "秒");
//3.并发抓取示例
//ConcurrentCrawler();
Console.ReadKey();
}
/// <summary>
/// 抓取城市列表
/// </summary>
public static void CityCrawler() {
var cityUrl = "http://hotels.ctrip.com/citylist";//定义爬虫入口URL
var cityList = new List<City>();//定义泛型列表存放城市名称及对应的酒店URL
var cityCrawler = new SimpleCrawler();//调用刚才写的爬虫程序
cityCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
cityCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
cityCrawler.OnCompleted = (s, e) =>
{
//使用正则表达式清洗网页源代码中的数据
var links = Regex.Matches(e.PageSource, @"<a[^>] href=""*(?<href>/hotel/[^>\s] )""\s*[^>]*>(?<text>(?!.*img).*?)</a>", RegexOptions.IgnoreCase);
foreach (Match match in links)
{
var city = new City
{
CityName = match.Groups["text"].Value,
Uri = new Uri("http://hotels.ctrip.com" match.Groups["href"].Value
)
};
if (!cityList.Contains(city)) cityList.Add(city);//将数据加入到泛型列表
Console.WriteLine(city.CityName "|" city.Uri);//将城市名称及URL显示到控制台
}
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！合计 " links.Count " 个城市。");
Console.WriteLine("耗时：" e.Milliseconds "毫秒");
Console.WriteLine("线程：" e.ThreadId);
Console.WriteLine("地址：" e.Uri.ToString());
};
cityCrawler.Start(new Uri(cityUrl)).Wait();//没被封锁就别使用代理：60.221.50.118:8090
}
/// <summary>
/// 抓取酒店列表
/// </summary>
public static void HotelCrawler() {
//var hotelUrl = "http://hotels.ctrip.com/hotel/beijing1#ctm_ref=hod_hp_sb_lst";
var hotelUrl = "http://hotels.ctrip.com/hotel/beijing1#ctm_ref=hod_hp_sb_lst";
var hotelList = new List<Hotel>();
var hotelCrawler = new SimpleCrawler();
hotelCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
hotelCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
hotelCrawler.OnCompleted = (s, e) =>
{
#region 获取最大页数
int max_num = 1;
try
{
var max_num_str = HtmlTag.GetBetween(e.PageSource, "data-pagecount=", "name=\"\"");
max_num = int.Parse(max_num_str);
}
catch (Exception)
{
max_num = 1;
}
#endregion
var hotelLists = new List<string>();
for (int i = 1; i < max_num; i )
{
hotelLists.Add(string.Format("http://hotels.ctrip.com/hotel/beijing1/p{0}", i));
//hotelLists.Add(string.Format("http://hotels.ctrip.com/hotel/beijing1/p{0}", i));
}
var hotelList1 = hotelLists;
var hotelCrawler1 = new SimpleCrawler();
hotelCrawler1.OnStart = (a, b) =>
{
Console.WriteLine("爬虫开始抓取地址：" b.Uri.ToString());
};
hotelCrawler1.OnError = (a, b) =>
{
Console.WriteLine("爬虫抓取出现错误：" b.Uri.ToString() "，异常消息：" b.Exception.Message);
};
hotelCrawler1.OnCompleted = (a, b) =>
{
var links = Regex.Matches(b.PageSource, @"""><a[^>] href=""*(?<href>/hotel/[^>\s] )""\s*data-dopost[^>]*><span[^>] >.*?</span>(?<text>.*?)</a>", RegexOptions.IgnoreCase);
foreach (Match match in links)
{
var hotel = new Hotel
{
HotelName = match.Groups["text"].Value,
Uri = new Uri("http://hotels.ctrip.com" match.Groups["href"].Value
)
};
if (!hotelList.Contains(hotel)) hotelList.Add(hotel);//将数据加入到泛型列表
Console.WriteLine(hotel.HotelName "|" hotel.Uri);//将酒店名称及详细页URL显示到控制台
}
Console.WriteLine();
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！");
Console.WriteLine("耗时：" b.Milliseconds "毫秒");
Console.WriteLine("线程：" b.ThreadId);
Console.WriteLine("地址：" b.Uri.ToString());
};
Parallel.For(0, hotelList1.Count, (i) =>
{
var hotel = hotelList1[i];
hotelCrawler1.Start(new Uri(hotel)).Wait();
});
};
hotelCrawler.Start(new Uri(hotelUrl)).Wait();//没被封锁就别使用代理：60.221.50.118:8090
ConcurrentCrawler(hotelList);
}
/// <summary>
/// 并发抓取示例
/// </summary>
public static void ConcurrentCrawler(List<Hotel> hotelLists)
{
var hotelList = hotelLists;
var hotelCrawler = new SimpleCrawler();
hotelCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
hotelCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
hotelCrawler.OnCompleted = (s, e) =>
{
Hotel h=new Hotel();
h.HotelName= HtmlTag.GetBetween(e.PageSource, "<h1>", "</h1>");
h.Price =0;
h.Uri = e.Uri;
DbHelperOra.ExecuteSql(string.Format("insert into t_ctrip(HOTEL_NAME,PRICE,URL) values('{0}','{1}','{2}')",
h.HotelName, h.Price, h.Uri));
Console.WriteLine();
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！");
Console.WriteLine("耗时：" e.Milliseconds "毫秒");
Console.WriteLine("线程：" e.ThreadId);
Console.WriteLine("地址：" e.Uri.ToString());
};
Parallel.For(0, hotelList.Count, (i) =>
{
var hotel = hotelList[i];
hotelCrawler.Start(hotel.Uri).Wait();
});
}
/// <summary>
/// 并发抓取示例
/// </summary>
public static void ConcurrentCrawler() {
var hotelList = new List<Hotel>() {
new Hotel { HotelName="遵义浙商酒店", Uri=new Uri("http://hotels.ctrip.com/hotel/4983680.html?isFull=F") },
new Hotel { HotelName="遵义森林大酒店", Uri=new Uri("http://hotels.ctrip.com/hotel/1665124.html?isFull=F") },
};
var hotelCrawler = new SimpleCrawler();
hotelCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
hotelCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
hotelCrawler.OnCompleted = (s, e) =>
{
Console.WriteLine();
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！");
Console.WriteLine("耗时：" e.Milliseconds "毫秒");
Console.WriteLine("线程：" e.ThreadId);
Console.WriteLine("地址：" e.Uri.ToString());
};
Parallel.For(0, 2, (i) =>
{
var hotel = hotelList[i];
hotelCrawler.Start(hotel.Uri).Wait();
});
}
}
}
标签： 爬虫
好例子网口号：伸出你的我的手 — 分享！
相关软件
网友评论
小贴士
感谢您为本站写下的评论，您的评论对其它用户来说具有重要的参考价值，所以请认真填写。
- 类似“顶”、“沙发”之类没有营养的文字，对勤劳贡献的楼主来说是令人沮丧的反馈信息。
- 相信您也不想看到一排文字/表情墙，所以请不要反馈意义不大的重复字符，也请尽量不要纯表情的回复。
- 提问之前请再仔细看一遍楼主的说明，或许是您遗漏了。
- 请勿到处挖坑绊人、招贴广告。既占空间让人厌烦，又没人会搭理，于人于己都无利。
关于好例子网
本站旨在为广大IT学习爱好者提供一个非营利性互相学习交流分享平台。本站所有资源都可以被免费获取学习研究。本站资源来自网友分享，对搜索内容的合法性不具有预见性、识别性、控制性，仅供学习研究，请务必在下载后24小时内给予删除，不得用于其他任何用途，否则后果自负。基于互联网的特殊性，平台无法对用户传输的作品、信息、内容的权属或合法性、安全性、合规性、真实性、科学性、完整权、有效性等进行实质审查；无论平台是否已进行审查，用户均应自行承担因其传输的作品、信息、内容而可能或已经产生的侵权或权属纠纷等法律责任。本站所有资源不代表本站的观点或立场，基于网友分享，根据中国法律《信息网络传播权保护条例》第二十二与二十三条之规定，若资源存在侵权或相关问题请联系本站客服人员，点此联系我们。关于更多版权及免责申明参见 版权及免责申明
支持(0) 盖楼(回复)