PYTHON爬虫示例

【实例截图】

【核心代码】

#encoding:utf-8
#!/usr/bin/env python

from urllib import urlretrieve
from time import ctime, sleep
from random import randint
from operator import add, mul
from functools import partial

def firstNonBlank(lines):
	for eachLine in lines:
		if not eachLine.strip():
			continue
		else:
			#sleep(1)
			#print eachLine
			return eachLine
			
def firstLast(webpage):
	print 'webpage= ', webpage
	f = open(webpage)
	lines = f.readlines()
	#print 'lines=',lines
	f.close()
	print firstNonBlank(lines),lines.reverse()
	print firstNonBlank(lines),
	
def download(url='http://www.baidu.com/', process=firstLast):
	try:
		retval = urlretrieve(url)[0]
		print 'retval= ',retval
	except IOError:
		retval = None
	if retval: # do some processing
		process(retval)

def newfoo(arg1, arg2, *nkw, **kw):
	'''display regular args and all variable args'''
	print 'arg1 is:', arg1
	print 'arg2 is:', arg2 
	for eachNKW in nkw:
		print 'additional non-keyword arg:', eachNKW
	for eachKW in kw.keys():
		print "additional keyword arg '%s': %s" % \
		(eachKW, kw[eachKW])
	
def testit(func, *nkwargs, **kwargs):
	try:
		retval = func(*nkwargs, **kwargs)
		result = (True, retval)
	except Exception, diag:
		result = (False, str(diag))
	return result
	
def test():
	funcs = (int, long, float)
	vals = (1234, 12.34, '1234', '12.34')
	for eachFunc in funcs:
		print '-' * 20
		for eachVal in vals:
			retval = testit(eachFunc, eachVal)
			if retval[0]:
				print '%s(%s) =' % \
				(eachFunc.__name__, `eachVal`), retval[1]
			else:
				print '%s(%s) = FAILED:' % \
				(eachFunc.__name__, `eachVal`), retval[1]
'''jishu'''
def odd(n):
	return n % 2
	
def counter(start_at=0): 
	count = [start_at] 
	def incr():
		count[0]  = 1
		return count[0]
	return incr	
				
if __name__ == '__main__':
	allNums = []
	for eachNum in range(9):
		radNum= randint(1, 99)
		allNums.append(radNum)
		print '[%02d]=[%02d]' % (eachNum,radNum)
	print [n for n in allNums if n%2]
	print filter(odd, allNums)
	map1 = map((lambda x: x 2), [0, 1, 2, 3, 4, 5])
	map2 = map(lambda x, y: x   y, [1,3,5], [2,4,6])
	print map1
	print map2
	print reduce((lambda x,y: x y), range(5))
	baseTwo = partial(int, base=2)
	baseTwo.__doc__ = 'Convert base 2 string to an int.'
	baseTwo('10010')
	print 'use Closures'
	count = counter(5)
	print count()
	print count()
	print count()
	print 'end use Closures'
	
	
	
	test()
	newfoo('wolf', 3, 'projects',1,2,3, freud=90, gamble=96)
	aTuple = (6, 7, 8)
	aDict = {'z': 9}
	newfoo(1, 2, 3, x=4, y=5, *aTuple, **aDict)
	download()