# setting.py 为项目配置文件

# 配置API服务

HOST = "0.0.0.0"               # IP
PORT = 5000                    # 监听端口


# 配置数据库

DB_CONN = 'redis://:pwd@127.0.0.1:8888/0'


# 配置 ProxyFetcher

PROXY_FETCHER = [
    "freeProxy05",      # 快代理
    "freeProxy10",      # 89ip API
    "freeProxy11",      # 稻壳代理
    "freeProxy16",      # ProxyScrape
    "freeProxy18",      # Proxifly
]

PROXY_FETCHER_EXTRA = [
    "freeProxy07",      # 云代理 ip3366
    "freeProxy12",      # 66免费代理 API
    "freeProxy13",      # 齐云代理
    "freeProxy14",      # 积流代理
    "freeProxy15",      # Geonode
]

POOL_SIZE_MIN = 30             # 代理池低于该值时触发补采
POOL_SIZE_CHECK_TRIGGER = 60   # 低于该值时补充源开始介入

TARGET_SITE_VALIDATION = True
TARGET_SITE_URLS = [
    "http://q.10jqka.com.cn/thshy/detail/field/199112/order/desc/page/1/ajax/1/code/881101",
]
TARGET_SITE_REQUIRED_MARKERS = ["<table"]
TARGET_SITE_REJECT_MARKERS = ["Nginx forbidden", "request info:", "请求过于频繁"]
TARGET_SITE_COOKIE_URL = "http://localhost:3000/"
TARGET_SITE_COOKIE_NAME = "v"

FETCH_INTERVAL = 8             # 核心源抓取间隔(分钟)
CHECK_INTERVAL = 5             # 代理校验间隔(分钟)
EXTRA_FETCH_INTERVAL = 30      # 补充源抓取间隔(分钟)

import requests

def get_proxy():
    return requests.get("http://127.0.0.1:5010/get/").json()

def delete_proxy(proxy):
    requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy))

# your spider code

def getHtml():
    # ....
    retry_count = 5
    proxy = get_proxy().get("proxy")
    while retry_count > 0:
        try:
            html = requests.get('http://www.example.com', proxies={"http": "http://{}".format(proxy)})
            # 使用代理访问
            return html
        except Exception:
            retry_count -= 1
    # 删除代理池中代理
    delete_proxy(proxy)
    return None

class ProxyFetcher(object):
    # ....

    # 自定义代理源获取方法
    @staticmethod
    def freeProxyCustom1():  # 命名不和已有重复即可

        # 通过某网站或者某接口或某数据库获取代理
        # 假设你已经拿到了一个代理列表
        proxies = ["x.x.x.x:3128", "x.x.x.x:80"]
        for proxy in proxies:
            yield proxy
        # 确保每个proxy都是 host:ip正确的格式返回

PROXY_FETCHER = [
    "freeProxy05",
    "freeProxy10",
    "freeProxy11",
    "freeProxy16",
    # ....
    "freeProxyCustom1"  #  # 确保名字和你添加方法名字一致
]