如何使用 Python 实现模拟登录和爬取网站的递归抓取

import requests

# 创建 Session
session = requests.Session()

# 登录信息
login_data = {
    "username": "your_username",
    "password": "your_password"
}

# 登录请求 URL
login_url = "http://www.pidancode.com/login/"

# 发送登录请求
response = session.post(login_url, data=login_data)

# 检查是否登录成功
if response.status_code == 200:
    print("登录成功！")
else:
    print("登录失败！")
    exit()

# 打印登录后的 Cookie
print(session.cookies)

from bs4 import BeautifulSoup

# 抓取的 URL
url = "http://www.pidancode.com/"

# 发送抓取请求
response = session.get(url)

# 使用 BeautifulSoup 将 html 解析成树形结构
soup = BeautifulSoup(response.text, "html.parser")

# 获取当前页面中的所有链接
all_links = soup.find_all("a")

# 打印链接列表
for link in all_links:
    print(link.get("href"))

def recursive_crawl(url, session):
    # 发送抓取请求
    response = session.get(url)

    # 使用 BeautifulSoup 将 html 解析成树形结构
    soup = BeautifulSoup(response.text, "html.parser")

    # 获取当前页面中的所有链接
    all_links = soup.find_all("a")

    # 打印链接列表
    for link in all_links:
        link_url = link.get("href")
        if link_url.startswith("http://www.pidancode.com/") and link_url not in visited_links:
            visited_links.add(link_url)
            print(link_url)
            recursive_crawl(link_url, session)

# 指定抓取链接的起始 URL
start_url = "http://www.pidancode.com"

# 记录已经访问过的链接，避免重复抓取
visited_links = set()

recursive_crawl(start_url, session)