Python 实现模拟登录和爬取网页时的性能优化技巧

import requests
from bs4 import BeautifulSoup

login_url = "http://www.pidancode.com/login"
data_url = "http://www.pidancode.com/data"

#第一步：发送登录请求，获取登录页面
session = requests.Session()
response = session.get(login_url)
soup = BeautifulSoup(response.text, "html.parser")
csrf_token = soup.find("input", {"name": "csrf_token"})["value"]

#第二步：构造登录表单，加入参数，发送登录请求
login_data = {
    "username": "your_username",
    "password": "your_password",
    "csrf_token": csrf_token
}
session.post(login_url, data=login_data)
cookies = session.cookies.get_dict()  #保存cookie

#第三步：使用保存的cookie请求需要登录才能访问的页面
response = requests.get(data_url, cookies=cookies)
print(response.text)

import requests
from concurrent.futures import ThreadPoolExecutor

urls = ["http://www.pidancode.com/page={}".format(i) for i in range(1, 11)]

def get_page(url):
    response = requests.get(url)
    return response.text

with ThreadPoolExecutor(max_workers=5) as executor:
    for result in executor.map(get_page, urls):
        print(result)

import requests
from functools import wraps

cache = {}

def memoize(func):
    @wraps(func)
    def wrap(*args):
        if args in cache:
            return cache[args]
        else:
            result = func(*args)
            cache[args] = result
            return result
    return wrap

@memoize
def get_page(url):
    response = requests.get(url)
    return response.text

urls = ["http://www.pidancode.com/page={}".format(i) for i in range(1, 11)]
for url in urls:
    print(get_page(url))

import requests

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    "Referer": "http://www.pidancode.com/",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
    "Connection": "keep-alive"
}

response = requests.get("http://www.pidancode.com", headers=headers)
print(response.text)