python实现多线程采集的2个代码例子

2016-06-24来源：

这篇文章主要介绍了python多线程采集代码例子,使用了threading、queue、mysqldb等模块,需要的朋友可以参考下。

代码一：

代码如下:

#!/usr/bin/python

# -*- coding: utf-8 -*-

#encoding=utf-8

import threading

import queue

import sys

import urllib2

import re

import mysqldb

# 数据库变量设置

db_host = '127.0.0.1'

db_user = xxxx

db_passwd = xxxxxxxx

db_name = xxxx

# 变量设置

thread_limit = 3

jobs = queue.queue(5)

singlelock = threading.lock()

info = queue.queue()

def workerbee(inputlist):

for x in xrange(thread_limit):

print 'thead {0} started.'.format(x)

t = spider()

t.start()

for i in inputlist:

try:

jobs.put(i, block=true, timeout=5)

except:

singlelock.acquire()

print the queue is full !

singlelock.release()

# wait for the threads to finish

singlelock.acquire() # acquire the lock so we can print

print waiting for threads to finish.

singlelock.release() # release the lock

jobs.join() # this command waits for all threads to finish.

# while not jobs.empty():

# print jobs.get()

def gettitle(url,time=10):

response = urllib2.urlopen(url,timeout=time)

html = response.read()

response.close()

reg = r'<title>(.*?)</title>'

title = re.compile(reg).findall(html)

# title = title[0].decode('gb2312','replace').encode('utf-8')

title = title[0]

return title

class spider(threading.thread):

def run(self):

while 1:

try:

job = jobs.get(true,1)

singlelock.acquire()

title = gettitle(job[1])

info.put([job[0],title], block=true, timeout=5)

# print 'this {0} is {1}'.format(job[1],title)

singlelock.release()

jobs.task_done()

except:

break;

if __name__ == '__main__':

con = none

urls = []

try:

con = mysqldb.connect(db_host,db_user,db_passwd,db_name)

cur = con.cursor()

cur.execute('select id,url from `table_name` where `status`=0 limit 10')

rows = cur.fetchall()

for row in rows:

# print row

urls.append([row[0],row[1]])

workerbee(urls)

while not info.empty():

print info.get()

finally:

if con:

con.close()代码二：

代码如下:

#!/usr/bin/python

# -*- coding: utf-8 -*-

#encoding=utf-8

#filename:robot.py

import threading,queue,sys,urllib2,re

# 变量设置

thread_limit = 3 #设置线程数

jobs = queue.queue(5) #设置队列长度

singlelock = threading.lock() #设置一个线程锁,避免重复调用

urls = ['http://xxx.com/w/n/2013-04-28/1634703505.shtml','http://xxx.com/w/n/2013-04-28/1246703487.shtml','http://xxx.com/w/n/2013-04-28/1028703471.shtml','http://xxx.com/w/n/2013-04-27/1015703426.shtml','http://xxx.com/w/n/2013-04-26/1554703373.shtml','http://xxx.com/w/n/2013-04-26/1512703346.shtml','http://xxx.com/w/n/2013-04-26/1453703334.shtml','http://xxx.com/w/n/2013-04-26/1451703333.shtml','http://xxx.com/w/n/2013-04-26/1445703329.shtml','http://xxx.com/w/n/2013-04-26/1434703322.shtml','http://xxx.com/w/n/2013-04-26/1433703321.shtml','http://xxx.com/w/n/2013-04-26/1433703320.shtml','http://xxx.com/w/n/2013-04-26/1429703318.shtml','http://xxx.com/w/n/2013-04-26/1429703317.shtml','http://xxx.com/w/n/2013-04-26/1409703297.shtml','http://xxx.com/w/n/2013-04-26/1406703296.shtml','http://xxx.com/w/n/2013-04-26/1402703292.shtml','http://xxx.com/w/n/2013-04-26/1353703286.shtml','http://xxx.com/w/n/2013-04-26/1348703284.shtml','http://xxx.com/w/n/2013-04-26/1327703275.shtml','http://xxx.com/w/n/2013-04-26/1239703265.shtml','http://xxx.com/w/n/2013-04-26/1238703264.shtml','http://xxx.com/w/n/2013-04-26/1231703262.shtml','http://xxx.com/w/n/2013-04-26/1229703261.shtml','http://xxx.com/w/n/2013-04-26/1228703260.shtml','http://xxx.com/w/n/2013-04-26/1223703259.shtml','http://xxx.com/w/n/2013-04-26/1218703258.shtml','http://xxx.com/w/n/2013-04-26/1202703254.shtml','http://xxx.com/w/n/2013-04-26/1159703251.shtml','http://xxx.com/w/n/2013-04-26/1139703233.shtml']

def workerbee(inputlist):

for x in xrange(thread_limit):

print 'thead {0} started.'.format(x)

t = spider()

t.start()

for i in inputlist:

try:

jobs.put(i, block=true, timeout=5)

except:

singlelock.acquire()

print the queue is full !

singlelock.release()