实例介绍
【实例截图】
【核心代码】
# -*- coding:utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import os
import threading
import datetime
from bs4 import BeautifulSoup
import requests
import re
import pymysql
from download import download
import csv
class Job_qcwy(threading.Thread):
def __init__(self, jobarea):
threading.Thread.__init__(self)
# self.url = 'http://search.51job.com/jobsearch/search_result.php?fromJs=1&jobarea=' jobarea '&keyword=' keyword '&keywordtype=2&curr_page=' str(pn) '&lang=c&stype=2&postchannel=0000&fromType=1&confirmdate=9'
self.city = jobarea
self.keywords = ['大数据']
self.headers = {
'Host': 'search.51job.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36',
'Accept': 'text/html,application/xhtml xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
}
self.filepath = '/Users/edz/Documents/project/PositionAnalysis/positions.csv'
# self.data = {
# 'fromJs': 1,
# 'jobarea': jobarea,
# 'funtype': '',
# 'industrytype': '',
# 'keyword': keyword,
# 'keywordtype': 2,
# 'lang': 'c',
# 'stype': 2,
# 'postchannel': 0000,
# 'fromType': 1,
# 'confirmdate': 9,
# 'curr_page': pn
# }
self.time = datetime.datetime.now()
if self.time.month < 10:
self.month = "0" str(self.time.month)
else:
self.month = str(self.time.month)
if self.time.day < 10:
self.day = "0" str(self.time.day)
else:
self.day = str(self.time.day)
self.today = self.month "-" self.day
self.session = requests.Session()
def run(self):
self.conn()
f = open(self.filepath, 'a')
w = csv.writer(f)
try:
with self.db.cursor() as cur:
for i in range(len(self.keywords)):
for j in range(1,2):
self.url = 'http://search.51job.com/jobsearch/search_result.php?fromJs=1&jobarea=' self.city '&keyword=' self.keywords[i] \
'&keywordtype=2&curr_page=' str(j) '&lang=c&stype=2&postchannel=0000&fromType=1&confirmdate=9'
html = download(self.url, self.headers)
# html = self.session.get(self.url, headers=self.headers)
html.encoding = 'gbk'
bsobj = BeautifulSoup(html.text, 'html.parser')
table = bsobj.find('div', {'class', 'dw_table'})
all_els = table.findAll('div', {'class', 'el'})
if all_els:
print len(all_els)
for i in range(1, len(all_els)):
today = all_els[i].find('span', {'class', 't5'}).get_text()
if today == self.today: # 判断发布的职位是否是当天的
link = all_els[i].find('a')['href']
print link
positionId = re.findall(r'\d ', link)[1]
s = "SELECT `Position` FROM `51job_position_info` where `PositionId`='" positionId "'"
if cur.execute(s) == 0:
position, salary, area, Description_and_requirements, companyName, companyNature, companyPersonnel, companyIntroduction = self.get_info(link)
publish_time = self.today
print position, salary, area, companyName
# w.writerow([positionId, position, salary, Description_and_requirements, area, publish_time, companyName, companyNature, companyPersonnel, companyIntroduction])
try:
sql = "INSERT INTO `51job_position_info` (`PositionId`, `Position`, `Salary`, `Description`, `Address`, `Publish_time`, `Company`, `CompanyNature`, `CompanyPersonnel`, `CompanyIntroduction` ) " \
"VALUE ('" positionId "','" position "','" salary "','" Description_and_requirements "','" area "','" publish_time "','" companyName "','" companyNature "','" companyPersonnel "','" companyIntroduction "')"
cur.execute(sql)
self.db.commit()
print sql
except Exception as e:
print e
else:
print positionId "已存在"
else:
print all_els
finally:
f.close()
self.db.close()
# 数据库设置
def conn(self):
self.db = pymysql.connect(
host='localhost',
user='root',
password='123456',
db='PositionAnalysis',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor
)
# 获取职位的详细信息
def get_info(self, link):
try:
self.headers['Host'] = 'jobs.51job.com'
html = download(link, self.headers)
# html = self.session.get(link, headers=self.headers)
html.encoding = 'gbk'
bs = BeautifulSoup(html.text, 'html.parser')
thjob = bs.find('div', {'class', 'tHeader tHjob'})
cn = thjob.find('div', {'class', 'cn'})
# 职位
JobName = cn.find('h1').get_text()
lname = cn.find('span').get_text()
if lname == None:
lname = ''
# 薪资
salary = cn.find('strong').get_text()
if salary == None:
salary = ''
infos = bs.find('div', {'class', 'tCompany_main'}).find('div', {'class', 'bmsg job_msg inbox'}).get_text().replace('\t', '').replace('\r', '')
# sps = bs.findAll('span', {'class', 'sp4'})
# # 经验要求
# experience = sps[0].get_text().replace('\n', '')
# # 学历要求
# education = sps[1].get_text().replace('\n', '')
if infos != '':
if re.findall(r'(.*?)\n', infos)[0] == '':
#职位描述和任职要求
try:
Description_and_requirements = re.findall(r'(.*?)\n', infos)[1] re.findall(r'(.*?)\n', infos)[2]
except:
Description_and_requirements = ''
else:
try:
Description_and_requirements = re.findall(r'(.*?)\n', infos)[0] re.findall(r'(.*?)\n', infos)[1]
except:
Description_and_requirements = ''
else:
Description_and_requirements = ''
a = bs.find('div', {'class', 'bmsg inbox'})
if a != None:
ar = a.find('p', {'class', 'fp'})
# 工作地点
area = lname '-' re.findall(r'span>(.*?)<', str(ar))[0].replace('\t', '').decode('utf-8')
else:
area = lname
#公司
companyName = cn.find('p', {'class', 'cname'}).find('a').get_text()
if companyName == None:
companyName = ''
company = cn.find('p', {'class', 'msg ltype'}).get_text().replace('\r', '').replace('\t', '').replace(' ', '')
try:
company = company.decode('utf-8').encode('utf-8')
except:
pass
if company:
if re.findall(r'(.*?)\xc2\xa0', company) != []:
q = re.findall(r'(.*?)\xc2\xa0', company)[0]
if re.findall(r'\d', q) == []:
# 公司性质
companyNature = q
if len(re.findall(r'(.*?)\xc2\xa0', company)) > 2:
# 公司规模
try:
companyPersonnel = re.findall(r'(.*?)\xc2\xa0', company)[4]
except:
companyPersonnel = ''
else:
companyPersonnel = ''
else:
companyNature = ''
companyPersonnel = q
else:
q = re.findall(r'(.*?)\xa0\xa0', company)[0]
if re.findall(r'\d', q) == []:
companyNature = q
if len(re.findall(r'(.*?)\xa0\xa0', company)) > 2:
try:
companyPersonnel = re.findall(r'(.*?)\xa0\xa0', company)[2]
except:
companyPersonnel = ''
else:
companyPersonnel = ''
else:
companyNature = ''
companyPersonnel = q
else:
companyNature = ''
companyPersonnel = ''
#公司描述
try:
companyIntroduction = bs.find('div', {'class', 'tmsg inbox'}).get_text().replace('\t', '').replace('\n', '')
except:
companyIntroduction = ''
return JobName, salary, area, Description_and_requirements, companyName, companyNature, companyPersonnel, companyIntroduction
except Exception as e:
print link
print e.message
if __name__ == '__main__':
# filepath = '/Users/edz/Documents/project/PositionAnalysis/positions.csv'
# if not os.path.isfile(filepath):
# f = open(filepath, 'a')
# writer = csv.writer(f)
# writer.writerow(['positionId', 'position', 'salary', 'Description_and_requirements', 'area', 'publish_time','companyName', 'companyNature', 'companyPersonnel', 'companyIntroduction'])
# f.close()
jobarea = ['010000']
threads = []
for i in range(len(jobarea)):
t = Job_qcwy(jobarea[i])
threads.append(t)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
网友评论
小贴士
感谢您为本站写下的评论，您的评论对其它用户来说具有重要的参考价值，所以请认真填写。
- 类似“顶”、“沙发”之类没有营养的文字，对勤劳贡献的楼主来说是令人沮丧的反馈信息。
- 相信您也不想看到一排文字/表情墙，所以请不要反馈意义不大的重复字符，也请尽量不要纯表情的回复。
- 提问之前请再仔细看一遍楼主的说明，或许是您遗漏了。
- 请勿到处挖坑绊人、招贴广告。既占空间让人厌烦，又没人会搭理，于人于己都无利。
关于好例子网
本站旨在为广大IT学习爱好者提供一个非营利性互相学习交流分享平台。本站所有资源都可以被免费获取学习研究。本站资源来自网友分享，对搜索内容的合法性不具有预见性、识别性、控制性，仅供学习研究，请务必在下载后24小时内给予删除，不得用于其他任何用途，否则后果自负。基于互联网的特殊性，平台无法对用户传输的作品、信息、内容的权属或合法性、安全性、合规性、真实性、科学性、完整权、有效性等进行实质审查；无论平台是否已进行审查，用户均应自行承担因其传输的作品、信息、内容而可能或已经产生的侵权或权属纠纷等法律责任。本站所有资源不代表本站的观点或立场，基于网友分享，根据中国法律《信息网络传播权保护条例》第二十二与二十三条之规定，若资源存在侵权或相关问题请联系本站客服人员，点此联系我们。关于更多版权及免责申明参见 版权及免责申明
支持(0) 盖楼(回复)