Commit daac0ed9 authored by Jean-Benoist Leger's avatar Jean-Benoist Leger

parallel get

parent 17ef2ab2
......@@ -30,6 +30,7 @@ import re
import time
import configobj
import html2text
import multiprocessing
def get_articles_from_page(link):
articles={}
......@@ -88,8 +89,13 @@ def get_code(codename,codeids):
articles = {}
for link in links:
articles.update(get_articles_from_page(link))
pool = multiprocessing.Pool(processes=25)
pool_outputs = pool.map(get_articles_from_page,links)
pool.close()
pool.join()
for po in pool_outputs:
articles.update(po)
return articles
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment