在Python中实现基于代理IP的多线程网络爬虫的方法

import requests
import threading
import queue

# 设置代理IP
proxies = {
    "http": "http://127.0.0.1:8888",
    "https": "https://127.0.0.1:8888"
}

# 设置要爬取的URL列表
urls = [
    "https://www.pidancode.com",
    "https://www.pidancode.com/page1",
    "https://www.pidancode.com/page2",
    "https://www.pidancode.com/page3",
    "https://www.pidancode.com/page4",
    "https://www.pidancode.com/page5"
]

# 设置线程数和队列
thread_count = 4
work_queue = queue.Queue()

# 将所有URL放入队列
for url in urls:
    work_queue.put(url)

# 定义爬取函数
def crawl():
    while True:
        url = work_queue.get()
        try:
            # 使用代理IP发起请求
            response = requests.get(url, proxies=proxies)
            print(response.text)
        except Exception as e:
            print("Error: ", e)
        finally:
            work_queue.task_done()

# 创建线程池并启动线程
for i in range(thread_count):
    t = threading.Thread(target=crawl)
    t.daemon = True
    t.start()

# 等待队列中的所有任务完成
work_queue.join()

print("All tasks are done.")