一、使用Flask模拟阻塞,利用线程池爬取数据
#模拟Flask #pip install flask from flask import Flask,render_template #返回一个模板文件需要导入render_tamplate from time import sleep app = Flask(__name__) @app.route(‘/bobo‘) def index_1(): return ‘hello world‘ @app.route(‘/jay‘) def index_2(): sleep(2) return render_template(‘test.html‘) @app.route(‘/tom‘) def index_3(): sleep(2) return render_template(‘test.html‘) if __name__ == ‘__main__‘: app.run(threaded=True) #括号里面也可以写debug=True 也可以不写
线程池代码:
# 同步代码 # import requests # import time # # urls = [ # ‘http://127.0.0.1:5000/jay‘, # ‘http://127.0.0.1:5000/tom‘, # ‘http://127.0.0.1:5000/jay‘, # ‘http://127.0.0.1:5000/tom‘, # ] # start = time.time() # for url in urls: # page_text = requests.get(url).text # print(len(page_text)) # # print(‘总耗时‘,time.time()-start) #异步代码 import requests import time from multiprocessing.dummy import Pool pool = Pool(4) urls = [ ‘http://127.0.0.1:5000/jay‘, ‘http://127.0.0.1:5000/tom‘, ‘http://127.0.0.1:5000/jay‘, ‘http://127.0.0.1:5000/tom‘, ] start = time.time() def get_request(url): page_text = requests.get(url).text print(len(page_text)) def parse(page_text): print(‘模仿解析‘) page_text_list = pool.map(get_request,urls) #这个map做数据请求 pool.map(parse,page_text_list) #这个map做数据解析 print(‘总耗时‘,time.time()-start)
原文地址:https://www.cnblogs.com/guniang/p/11718215.html
时间: 2024-10-07 20:18:13