Python多线程爬虫教程\_threading实战详解

import threading
import queue
import requests
<p>url_queue = queue.Queue()
results = []</p><p>def worker():
while True:
url = url_queue.get()
if url is None:  # 退出信号
break
try:
resp = requests.get(url, timeout=5)
results.append((url, resp.status_code))
except Exception as e:
results.append((url, f"error: {e}"))
url_queue.task_done()  # 标记完成</p><h1>启动 4 个线程</h1><p>threads = []
for _ in range(4):
t = threading.Thread(target=worker)
t.start()
threads.append(t)</p><h1>添加任务</h1><p>for u in ["<a target='_blank'  href='https://www.17golang.com/gourl/?redirect=MDAwMDAwMDAwML57hpSHp6VpkrqbYLx2eayza4KafaOkbLS3zqSBrJvPsa5_0Ia6sWuR4Juaq6t9nq5roGCUgXuytMyerpV6iZXHe3vUmsyZr5vTk6a8eYanvpGjpn2MhqKu3LOijnmMlbN4cpSSt89pkqp5qLBkep6yo6Nkf42hpLLdyqKBrIXRsot-lpHdz3Y' rel='nofollow'>https://httpbin.org/delay/1</a>", "<a target='_blank'  href='https://www.17golang.com/gourl/?redirect=MDAwMDAwMDAwML57hpSHp6VpkrqbYLx2eayza4KafaOkbLS3zqSBrJvPsa5_0Ia6sWuR4Juaq6t9nq5roGCUgXuytMyerpV6iZXHe3vUmsyZr5vTk6a8eYanvpGjpn2ihqKu3LOijnmMlbN4cpSSt89pkqp5qLBkep6yo6Nkf42hpLLdyqKBrIXRsot-lpHdz3Y' rel='nofollow'>https://httpbin.org/delay/2</a>"]:
url_queue.put(u)</p><p>url_queue.join()  # 等所有任务完成</p><h1>发送退出信号</h1><p>for _ in threads:
url_queue.put(None)
for t in threads:
t.join()
</p>

import csv
lock = threading.Lock()
<p>def save_to_csv(url, status):
with lock:  # 确保同一时间只有一个线程在写
with open("log.csv", "a", newline="") as f:
writer = csv.writer(f)
writer.writerow([url, status])
</p>