Scrapy框架简单应用:爬取免费的的西刺代理IP

【实例简介】利用Scrapy框架爬取免费的的西刺代理IP

内含模拟浏览器，利用代理池反反爬虫

【实例截图】

from urllib.request import ProxyHandler, build_opener 

import re 

from urllib import request 

''' 本段注释代码，将获取的信息保存到MongoDB数据库 

client = MongoClient('localhost', 27017) 

db_auth = client.admin db_auth.authenticate("root", "123") 

db = client['代理'] collection = db['IP代理'] '''  

'''以文本方式保存到本地''' f = open('可用代理.txt','w',encoding='utf-8') 

class ProxyPipeline(object): 

    def process_item(self, item, spider):

         for i in range(0,len(item["IP"])): 

            #print(item['name'][i])  

            proxy = item['IP'][i] ':' item["port"][i]

       proxy_handler = ProxyHandler({'http': 'http://'  proxy})
       opener = build_opener(proxy_handler) 

    try:
           head = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36', }
                url = 'http://www.xdaili.cn/monitor'  

        req = request.Request(url, headers=head)
              response = opener.open(req)
                 data = response.read().decode('utf-8') # print(data)  

         if data: 

            '''保存到数据库  ret = collection.find_one({'IP': item['IP'][i], '端口': item["port"][i]})  

                    if ret:  # print('已经存在了')  pass  

                    else:  collection.save({'IP': item['IP'][i], '端口': item['port'][i], '匿名度': item['nmd'][i],  '类型': item['type'][i], '位置': item['addr'][i], '最后验证时间': item['lastime'][i]})  # 向数据库插入一条记录  '''  

                        f.write( proxy)
                                  f.write("\n") 

                        print(proxy) 

    except Exception as e: 

    #print(proxy "不可用")  pass

getip.py

 
import scrapy from proxy.items import ProxyItem import re class GetipSpider(scrapy.Spider):
    name = 'getip'  allowed_domains = ['xicidaili.com']
    start_urls = [ 'http://www.xicidaili.com/wn/',  'http://www.xicidaili.com/wt/',  ] def parse(self, response):
        item = ProxyItem()
        item["IP"] = response.xpath("//tr[@class = 'odd']/td[2]/text()").extract()
        item["port"] = response.xpath("//tr[@class = 'odd']/td[3]/text()").extract()
        item["nmd"] = response.xpath("//tr[@class = 'odd']/td[5]/text()").extract()
        item["type"] = response.xpath("//tr[@class = 'odd']/td[6]/text()").extract()
        item["addr"] = response.xpath("//tr[@class = 'odd']/td[4]/a/text()").extract()
        item["lastime"] = response.xpath("//tr[@class = 'odd']/td[10]/text()").extract() #print(item)  yield item #rang = response.xpath("//div[@id = 'listnav']/ul/li[6]/a/text()").extract()        #下一页，前5页最后验证时间临近的IP  #print(int(rang[0]) 1)  type = response.xpath("//div[@class='pagination']/a[last()]/@href").extract()[0]
        type = re.findall(r'/(.*?)/',type,re.S) #print(type)  for i in range(2,4):
            next_page = response.urljoin('http://www.xicidaili.com/%s/%s/') % (type[0],i) print(next_page) #next_page = response.urljoin('https://www.kuaidaili.com%s%s/') % type % i  yield scrapy.Request(next_page, callback=self.parse)