const failoverPolicy = {
retries: 3,
retryOn: [429, 502, 503, 504],
switchPoolOnFinalRetry: true,
backupPool: "residential-eu-secondary",
backoffMs: [200, 500, 1500],
rotateIpOnRetry: true,
};
import time
import requests
def fetch_with_failover(url, proxy_pools, max_retries=3):
"""Fetch URL with automatic proxy failover."""
backoff = [0.2, 0.5, 1.5]
for attempt in range(max_retries):
pool_index = min(attempt, len(proxy_pools) - 1)
proxy = proxy_pools[pool_index]
try:
response = requests.get(
url,
proxies={"http": proxy, "https": proxy},
timeout=15,
)
if response.status_code in [429, 502, 503, 504]:
time.sleep(backoff[attempt])
continue
return response
except requests.exceptions.RequestException:
time.sleep(backoff[attempt])
continue
raise Exception(f"All {max_retries} attempts failed for {url}")