Initial project import
This commit is contained in:
XM
@@ -0,0 +1,415 @@
|
||||
import copy
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from services.config import DEFAULT_PROXY_RUNTIME
|
||||
from services.proxy_service import (
|
||||
ClearanceBundle,
|
||||
FlareSolverrClearanceProvider,
|
||||
ProxySettingsStore,
|
||||
normalize_proxy_url,
|
||||
)
|
||||
|
||||
|
||||
class FakeConfig:
|
||||
def __init__(self, legacy_proxy: str = "", runtime: dict[str, object] | None = None) -> None:
|
||||
self.legacy_proxy = legacy_proxy
|
||||
self.runtime = runtime if runtime is not None else copy.deepcopy(DEFAULT_PROXY_RUNTIME)
|
||||
|
||||
def get_proxy_settings(self) -> str:
|
||||
return self.legacy_proxy
|
||||
|
||||
def get_proxy_runtime_settings(self) -> dict[str, object]:
|
||||
return copy.deepcopy(self.runtime)
|
||||
|
||||
|
||||
def make_runtime(**overrides: object) -> dict[str, object]:
|
||||
runtime = copy.deepcopy(DEFAULT_PROXY_RUNTIME)
|
||||
clearance = overrides.pop("clearance", None)
|
||||
runtime.update(overrides)
|
||||
if clearance is not None:
|
||||
runtime["clearance"].update(clearance) # type: ignore[index,union-attr]
|
||||
return runtime
|
||||
|
||||
|
||||
class ProxyServiceTests(unittest.TestCase):
|
||||
def test_normalize_proxy_url_strips_and_converts_socks_schemes(self) -> None:
|
||||
self.assertEqual(normalize_proxy_url(" http://proxy.example:8080 "), "http://proxy.example:8080")
|
||||
self.assertEqual(normalize_proxy_url("\thttps://proxy.example:8443\n"), "https://proxy.example:8443")
|
||||
self.assertEqual(normalize_proxy_url(" socks://proxy.example:1080 "), "socks5h://proxy.example:1080")
|
||||
self.assertEqual(normalize_proxy_url("socks5://proxy.example:1080"), "socks5h://proxy.example:1080")
|
||||
self.assertEqual(normalize_proxy_url(" socks5h://proxy.example:1080 "), "socks5h://proxy.example:1080")
|
||||
self.assertEqual(normalize_proxy_url(" "), "")
|
||||
|
||||
def test_build_session_kwargs_keeps_legacy_global_proxy_when_runtime_disabled(self) -> None:
|
||||
store = ProxySettingsStore(FakeConfig(legacy_proxy=" http://legacy.example:8080 "))
|
||||
|
||||
kwargs = store.build_session_kwargs(impersonate="chrome")
|
||||
|
||||
self.assertEqual(kwargs["impersonate"], "chrome")
|
||||
self.assertEqual(kwargs["proxy"], "http://legacy.example:8080")
|
||||
|
||||
def test_runtime_proxy_is_limited_to_upstream_scope_by_default(self) -> None:
|
||||
runtime = make_runtime(enabled=True, egress_mode="single_proxy", proxy_url="http://runtime.example:8080")
|
||||
store = ProxySettingsStore(FakeConfig(legacy_proxy="http://legacy.example:8080", runtime=runtime))
|
||||
|
||||
self.assertEqual(
|
||||
store.build_session_kwargs()["proxy"],
|
||||
"http://legacy.example:8080",
|
||||
)
|
||||
self.assertEqual(
|
||||
store.build_session_kwargs(upstream=True)["proxy"],
|
||||
"http://runtime.example:8080",
|
||||
)
|
||||
|
||||
def test_account_proxy_wins_over_runtime_and_global_proxy(self) -> None:
|
||||
runtime = make_runtime(enabled=True, egress_mode="single_proxy", proxy_url="http://runtime.example:8080")
|
||||
store = ProxySettingsStore(FakeConfig(legacy_proxy="http://legacy.example:8080", runtime=runtime))
|
||||
|
||||
kwargs = store.build_session_kwargs(account={"proxy": " socks://account.example:1080 "}, upstream=True)
|
||||
|
||||
self.assertEqual(kwargs["proxy"], "socks5h://account.example:1080")
|
||||
|
||||
def test_proxy_runtime_single_proxy_wins_over_explicit_and_legacy_proxy_when_enabled(self) -> None:
|
||||
runtime = make_runtime(enabled=True, egress_mode="single_proxy", proxy_url=" socks5://runtime.example:1080 ")
|
||||
store = ProxySettingsStore(FakeConfig(legacy_proxy="http://legacy.example:8080", runtime=runtime))
|
||||
|
||||
kwargs = store.build_session_kwargs(proxy="http://explicit.example:8080", upstream=True)
|
||||
|
||||
self.assertEqual(kwargs["proxy"], "socks5h://runtime.example:1080")
|
||||
|
||||
def test_explicit_proxy_wins_over_legacy_global_proxy_when_runtime_disabled(self) -> None:
|
||||
store = ProxySettingsStore(FakeConfig(legacy_proxy="http://legacy.example:8080"))
|
||||
|
||||
kwargs = store.build_session_kwargs(proxy=" socks5://explicit.example:1080 ")
|
||||
|
||||
self.assertEqual(kwargs["proxy"], "socks5h://explicit.example:1080")
|
||||
|
||||
def test_resource_requests_use_resource_proxy_url_when_configured(self) -> None:
|
||||
runtime = make_runtime(
|
||||
enabled=True,
|
||||
egress_mode="single_proxy",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
resource_proxy_url=" socks5://resource.example:1080 ",
|
||||
)
|
||||
store = ProxySettingsStore(FakeConfig(legacy_proxy="http://legacy.example:8080", runtime=runtime))
|
||||
|
||||
kwargs = store.build_session_kwargs(resource=True, upstream=True)
|
||||
|
||||
self.assertEqual(kwargs["proxy"], "socks5h://resource.example:1080")
|
||||
|
||||
def test_manual_clearance_merges_cookies_and_preserves_explicit_user_agent(self) -> None:
|
||||
runtime = make_runtime(
|
||||
enabled=True,
|
||||
egress_mode="single_proxy",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
clearance={
|
||||
"enabled": True,
|
||||
"mode": "manual",
|
||||
"cf_cookies": "foo=bar; session=ok",
|
||||
"cf_clearance": "manual-token",
|
||||
"user_agent": "Manual UA",
|
||||
},
|
||||
)
|
||||
store = ProxySettingsStore(FakeConfig(runtime=runtime))
|
||||
|
||||
headers = store.build_headers(headers={"Cookie": "existing=1"}, target_url="https://chatgpt.com/backend-api")
|
||||
|
||||
self.assertEqual(headers["User-Agent"], "Manual UA")
|
||||
self.assertEqual(headers["Cookie"], "existing=1; foo=bar; session=ok; cf_clearance=manual-token")
|
||||
|
||||
headers_with_ua = store.build_headers(
|
||||
headers={"User-Agent": "Caller UA", "Cookie": "cf_clearance=caller-token"},
|
||||
target_url="https://chatgpt.com/backend-api",
|
||||
)
|
||||
|
||||
self.assertEqual(headers_with_ua["User-Agent"], "Caller UA")
|
||||
self.assertEqual(headers_with_ua["Cookie"], "cf_clearance=caller-token; foo=bar; session=ok")
|
||||
self.assertNotIn("cf_clearance=manual-token", headers_with_ua["Cookie"])
|
||||
|
||||
def test_flaresolverr_provider_parses_solution_and_filters_cookies_by_host(self) -> None:
|
||||
calls: list[tuple[str, dict[str, object], dict[str, str], float]] = []
|
||||
|
||||
def fake_request(endpoint: str, body: bytes, headers: dict[str, str], timeout: float) -> bytes:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
calls.append((endpoint, payload, headers, timeout))
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"solution": {
|
||||
"userAgent": "Flare UA",
|
||||
"cookies": [
|
||||
{"name": "cf_clearance", "value": "host-token", "domain": ".chatgpt.com"},
|
||||
{"name": "no_domain", "value": "kept"},
|
||||
{"name": "wrong_host", "value": "dropped", "domain": "example.net"},
|
||||
],
|
||||
},
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
provider = FlareSolverrClearanceProvider("http://flare.local/", request_method=fake_request)
|
||||
|
||||
bundle = provider.get_clearance(
|
||||
"https://chatgpt.com/backend-api/conversation",
|
||||
proxy_url="socks5h://proxy.example:1080",
|
||||
timeout_sec=12,
|
||||
)
|
||||
|
||||
self.assertIsNotNone(bundle)
|
||||
assert bundle is not None
|
||||
endpoint, payload, headers, timeout = calls[0]
|
||||
self.assertEqual(endpoint, "http://flare.local/v1")
|
||||
self.assertEqual(payload["cmd"], "request.get")
|
||||
self.assertEqual(payload["url"], "https://chatgpt.com/backend-api/conversation")
|
||||
self.assertEqual(payload["maxTimeout"], 12000)
|
||||
self.assertEqual(payload["proxy"], {"url": "socks5h://proxy.example:1080"})
|
||||
self.assertEqual(headers["Content-Type"], "application/json")
|
||||
self.assertEqual(timeout, 12)
|
||||
self.assertEqual(bundle.user_agent, "Flare UA")
|
||||
self.assertEqual(bundle.cookies, {"cf_clearance": "host-token", "no_domain": "kept"})
|
||||
|
||||
def test_flaresolverr_provider_keeps_only_matching_or_no_domain_cookies(self) -> None:
|
||||
def fake_request(endpoint: str, body: bytes, headers: dict[str, str], timeout: float) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"solution": {
|
||||
"userAgent": "Filtered UA",
|
||||
"cookies": [
|
||||
{"name": "wrong_host", "value": "dropped", "domain": "example.net"},
|
||||
{"name": "also_wrong", "value": "dropped-too", "domain": ".example.org"},
|
||||
{"name": "no_domain", "value": "kept"},
|
||||
],
|
||||
},
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
provider = FlareSolverrClearanceProvider("http://flare.local", request_method=fake_request)
|
||||
|
||||
bundle = provider.get_clearance("https://chatgpt.com", timeout_sec=5)
|
||||
|
||||
self.assertIsNotNone(bundle)
|
||||
assert bundle is not None
|
||||
self.assertEqual(bundle.cookies, {"no_domain": "kept"})
|
||||
self.assertEqual(bundle.user_agent, "Filtered UA")
|
||||
|
||||
def test_flaresolverr_provider_drops_all_wrong_domain_cookies(self) -> None:
|
||||
def fake_request(endpoint: str, body: bytes, headers: dict[str, str], timeout: float) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"solution": {
|
||||
"cookies": [
|
||||
{"name": "wrong_host", "value": "dropped", "domain": "example.net"},
|
||||
],
|
||||
},
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
provider = FlareSolverrClearanceProvider("http://flare.local", request_method=fake_request)
|
||||
|
||||
self.assertIsNone(provider.get_clearance("https://chatgpt.com", timeout_sec=5))
|
||||
|
||||
def test_cached_flaresolverr_bundle_is_merged_by_build_headers_and_can_be_invalidated(self) -> None:
|
||||
runtime = make_runtime(
|
||||
enabled=True,
|
||||
egress_mode="single_proxy",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
clearance={
|
||||
"enabled": True,
|
||||
"mode": "flaresolverr",
|
||||
"flaresolverr_url": "http://flare.local",
|
||||
"timeout_sec": 5,
|
||||
},
|
||||
)
|
||||
bundle = ClearanceBundle(
|
||||
target_host="chatgpt.com",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
cookies={"cf_clearance": "flare-token"},
|
||||
user_agent="Flare UA",
|
||||
)
|
||||
|
||||
class FakeProvider:
|
||||
def get_clearance(self, target_url: str, proxy_url: str = "", timeout_sec: int = 60) -> ClearanceBundle | None:
|
||||
return bundle
|
||||
|
||||
store = ProxySettingsStore(FakeConfig(runtime=runtime), clearance_provider_factory=lambda _url: FakeProvider())
|
||||
store.refresh_clearance(target_url="https://chatgpt.com", force=True)
|
||||
|
||||
headers = store.build_headers(headers={"Cookie": "existing=1"}, target_url="https://chatgpt.com/backend-api")
|
||||
|
||||
self.assertEqual(headers["User-Agent"], "Flare UA")
|
||||
self.assertEqual(headers["Cookie"], "existing=1; cf_clearance=flare-token")
|
||||
|
||||
store.invalidate_clearance(target_url="https://chatgpt.com")
|
||||
self.assertEqual(
|
||||
store.build_headers(headers={"Cookie": "existing=1"}, target_url="https://chatgpt.com/backend-api"),
|
||||
{"Cookie": "existing=1"},
|
||||
)
|
||||
|
||||
def test_flaresolverr_refresh_failure_keeps_old_cached_bundle(self) -> None:
|
||||
runtime = make_runtime(
|
||||
enabled=True,
|
||||
egress_mode="single_proxy",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
clearance={
|
||||
"enabled": True,
|
||||
"mode": "flaresolverr",
|
||||
"flaresolverr_url": "http://flare.local",
|
||||
"timeout_sec": 5,
|
||||
},
|
||||
)
|
||||
first_bundle = ClearanceBundle(
|
||||
target_host="chatgpt.com",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
cookies={"cf_clearance": "old-token"},
|
||||
user_agent="Old UA",
|
||||
)
|
||||
|
||||
class FakeProvider:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def get_clearance(self, target_url: str, proxy_url: str = "", timeout_sec: int = 60) -> ClearanceBundle | None:
|
||||
self.calls += 1
|
||||
return first_bundle if self.calls == 1 else None
|
||||
|
||||
provider = FakeProvider()
|
||||
store = ProxySettingsStore(FakeConfig(runtime=runtime), clearance_provider_factory=lambda _url: provider)
|
||||
|
||||
refreshed = store.refresh_clearance(target_url="https://chatgpt.com", force=True)
|
||||
fallback = store.refresh_clearance(target_url="https://chatgpt.com", force=True)
|
||||
|
||||
self.assertIsNotNone(refreshed)
|
||||
assert refreshed is not None
|
||||
self.assertEqual(refreshed.cookies, first_bundle.cookies)
|
||||
self.assertIsNotNone(refreshed.expires_at)
|
||||
assert refreshed.expires_at is not None
|
||||
self.assertAlmostEqual(refreshed.expires_at - refreshed.created_at, 3600, places=2)
|
||||
self.assertIs(fallback, refreshed)
|
||||
self.assertEqual(provider.calls, 2)
|
||||
|
||||
def test_flaresolverr_bundle_expires_after_refresh_interval(self) -> None:
|
||||
runtime = make_runtime(
|
||||
enabled=True,
|
||||
egress_mode="single_proxy",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
clearance={
|
||||
"enabled": True,
|
||||
"mode": "flaresolverr",
|
||||
"flaresolverr_url": "http://flare.local",
|
||||
"timeout_sec": 5,
|
||||
"refresh_interval": 60,
|
||||
},
|
||||
)
|
||||
|
||||
class FakeProvider:
|
||||
def get_clearance(self, target_url: str, proxy_url: str = "", timeout_sec: int = 60) -> ClearanceBundle | None:
|
||||
return ClearanceBundle(
|
||||
target_host="chatgpt.com",
|
||||
proxy_url=proxy_url,
|
||||
cookies={"cf_clearance": "expiring-token"},
|
||||
user_agent="Expiring UA",
|
||||
created_at=1000,
|
||||
)
|
||||
|
||||
store = ProxySettingsStore(FakeConfig(runtime=runtime), clearance_provider_factory=lambda _url: FakeProvider())
|
||||
with patch("services.proxy_service.time.time", return_value=1000):
|
||||
bundle = store.refresh_clearance(target_url="https://chatgpt.com", force=True)
|
||||
|
||||
self.assertIsNotNone(bundle)
|
||||
assert bundle is not None
|
||||
self.assertEqual(bundle.expires_at, 1060)
|
||||
self.assertTrue(bundle.is_valid_for("chatgpt.com", "http://runtime.example:8080", now=1059))
|
||||
self.assertFalse(bundle.is_valid_for("chatgpt.com", "http://runtime.example:8080", now=1060))
|
||||
|
||||
def test_profile_repr_does_not_expose_clearance_secrets(self) -> None:
|
||||
runtime = make_runtime(
|
||||
enabled=True,
|
||||
clearance={
|
||||
"enabled": True,
|
||||
"mode": "manual",
|
||||
"cf_cookies": "foo=secret-cookie",
|
||||
"cf_clearance": "secret-clearance",
|
||||
},
|
||||
)
|
||||
profile = ProxySettingsStore(FakeConfig(runtime=runtime)).get_profile()
|
||||
|
||||
text = repr(profile)
|
||||
|
||||
self.assertNotIn("secret-cookie", text)
|
||||
self.assertNotIn("secret-clearance", text)
|
||||
|
||||
def test_proxy_test_error_redacts_proxy_credentials(self) -> None:
|
||||
class FailingSession:
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
pass
|
||||
|
||||
def get(self, *args: object, **kwargs: object) -> object:
|
||||
raise RuntimeError("proxy failed for http://user:pass@proxy.example:8080")
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
with patch("services.proxy_service.Session", FailingSession):
|
||||
result = __import__("services.proxy_service", fromlist=["test_proxy"]).test_proxy(
|
||||
"http://user:pass@proxy.example:8080"
|
||||
)
|
||||
|
||||
self.assertFalse(result["ok"])
|
||||
self.assertIn("[REDACTED]", result["error"])
|
||||
self.assertNotIn("user:pass", result["error"])
|
||||
|
||||
def test_concurrent_flaresolverr_refresh_uses_single_flight_per_proxy_and_host(self) -> None:
|
||||
runtime = make_runtime(
|
||||
enabled=True,
|
||||
egress_mode="single_proxy",
|
||||
proxy_url="http://runtime.example:8080",
|
||||
clearance={
|
||||
"enabled": True,
|
||||
"mode": "flaresolverr",
|
||||
"flaresolverr_url": "http://flare.local",
|
||||
"timeout_sec": 5,
|
||||
},
|
||||
)
|
||||
|
||||
class SlowProvider:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def get_clearance(self, target_url: str, proxy_url: str = "", timeout_sec: int = 60) -> ClearanceBundle | None:
|
||||
with self.lock:
|
||||
self.calls += 1
|
||||
time.sleep(0.15)
|
||||
return ClearanceBundle(
|
||||
target_host=urlparse(target_url).hostname or "",
|
||||
proxy_url=proxy_url,
|
||||
cookies={"cf_clearance": "thread-token"},
|
||||
user_agent="Thread UA",
|
||||
)
|
||||
|
||||
provider = SlowProvider()
|
||||
store = ProxySettingsStore(FakeConfig(runtime=runtime), clearance_provider_factory=lambda _url: provider)
|
||||
workers = 6
|
||||
start = threading.Barrier(workers)
|
||||
|
||||
def refresh() -> ClearanceBundle | None:
|
||||
start.wait(timeout=5)
|
||||
return store.refresh_clearance(target_url="https://chatgpt.com", force=True)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers) as executor:
|
||||
results = list(executor.map(lambda _index: refresh(), range(workers)))
|
||||
|
||||
self.assertEqual(provider.calls, 1)
|
||||
self.assertTrue(all(result is results[0] for result in results))
|
||||
self.assertIsNotNone(results[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user