問題描述

用多進程爬取數據寫入文件，運行沒有報錯，但是打開文件卻亂碼。

用多線程改寫時卻沒有這個問題，一切正常。下面是數據寫入文件的代碼：

def Get_urls(start_page,end_page): print ’ run task {} ({})’.format(start_page,os.getpid()) url_text = codecs.open(’url.txt’,’a’,’utf-8’)for i in range(start_page,end_page+1): pageurl=baseurl1+str(i)+baseurl2+searchword response = requests.get(pageurl, headers=header) soup = BeautifulSoup(response.content, ’html.parser’) a_list=soup.find_all(’a’) for a in a_list:if a.text!=’’and ’wssd_content.jsp?bookid’in a[’href’]: text=a.text.strip() url=baseurl+str(a[’href’]) url_text.write(text+’t’+url+’n’)url_text.close()

多進程用的進程池

def Multiple_processes_test(): t1 = time.time() print ’parent process {} ’.format(os.getpid()) page_ranges_list = [(1,3),(4,6),(7,9)] pool = multiprocessing.Pool(processes=3) for page_range in page_ranges_list:pool.apply_async(func=Get_urls,args=(page_range[0],page_range[1])) pool.close() pool.join() t2 = time.time() print ’時間：’,t2-t1

問題解答

圖片上已經說了，文件以錯誤的編碼形式載入了，說明你多進程寫入的時候，編碼不是utf-8

文件第一行添加:

#coding: utf-8回答3：

打開同一個文件，相當危險，出錯機率相當大，多線程不出錯，極有可能是GIL,多進程沒有鎖，因此容易出錯了。

url_text = codecs.open(’url.txt’,’a’,’utf-8’)

建議改為生產者消費都模式!

比如這樣

# -*- coding: utf-8 -* -import timeimport osimport codecsimport multiprocessingimport requestsfrom bs4 import BeautifulSoupbaseurl = ’’baseurl1 = ’’baseurl2 = ’’pageurl = ’’searchword = ’’header = {}def fake(url, **kwargs): class Response(object):pass o = Response() o.content = ’<a href='http://m.b3g6.com/{}/wssd_content.jsp?bookid'>foo</a>’.format(url) return orequests.get = fakedef Get_urls(start_page, end_page, queue): print(’run task {} ({})’.format(start_page, os.getpid())) try:for i in range(start_page, end_page + 1): pageurl = baseurl1 + str(i) + baseurl2 + searchword response = requests.get(pageurl, headers=header) soup = BeautifulSoup(response.content, ’html.parser’) a_list = soup.find_all(’a’) for a in a_list:if a.text != ’’and ’wssd_content.jsp?bookid’in a[’href’]: text = a.text.strip() url = baseurl + str(a[’href’]) queue.put(text + ’t’ + url + ’n’) except Exception as e:import tracebacktraceback.print_exc()def write_file(queue): print('start write file') url_text = codecs.open(’url.txt’, ’a’, ’utf-8’) while True:line = queue.get()if line is None: breakprint('write {}'.format(line))url_text.write(line) url_text.close()def Multiple_processes_test(): t1 = time.time() manager = multiprocessing.Manager() queue = manager.Queue() print ’parent process {} ’.format(os.getpid()) page_ranges_list = [(1, 3), (4, 6), (7, 9)] consumer = multiprocessing.Process(target=write_file, args=(queue,)) consumer.start() pool = multiprocessing.Pool(processes=3) results = [] for page_range in page_ranges_list:result = pool.apply_async(func=Get_urls, args=(page_range[0], page_range[1], queue ))results.append(result) pool.close() pool.join() queue.put(None) consumer.join() t2 = time.time() print ’時間：’, t2 - t1if __name__ == ’__main__’: Multiple_processes_test()結果

foo /4/wssd_content.jsp?bookidfoo /5/wssd_content.jsp?bookidfoo /6/wssd_content.jsp?bookidfoo /1/wssd_content.jsp?bookidfoo /2/wssd_content.jsp?bookidfoo /3/wssd_content.jsp?bookidfoo /7/wssd_content.jsp?bookidfoo /8/wssd_content.jsp?bookidfoo /9/wssd_content.jsp?bookid