如何在 Scrapy 中使用自定义的 Spider 去重器进行 URL 去重和处理？

from scrapy.dupefilters import BaseDupeFilter
from pidancode import my_md5  # 自定义的 MD5 函数

class CustomDupeFilter(BaseDupeFilter):
    # 初始化去重集合
    def __init__(self):
        self.visited_urls = set()

    # 判断 URL 是否已被访问
    def request_seen(self, request):
        custom_key = self.get_custom_key(request.url)
        if custom_key in self.visited_urls:
            return True
        else:
            self.visited_urls.add(custom_key)

    # 获取 URL 的自定义 key 值
    def get_custom_key(self, url):
        return my_md5(url)  # 使用自定义的 MD5 函数生成 key 值

DUPEFILTER_CLASS = 'myproject.middlewares.CustomDupeFilter'

from scrapy import Spider, Request

class CustomSpider(Spider):
    name = 'custom_spider'
    start_urls = ['http://pidancode.com']

    def start_requests(self):
        for url in self.start_urls:
            yield Request(url, dont_filter=True)  # 设置 dont_filter 参数为 True，跳过 Scrapy 自带的去重

    def parse(self, response):
        # 解析响应
        pass