Initial project import
This commit is contained in:
XM
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Idempotently merge WARP/FlareSolverr proxy_runtime defaults into config.json."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/145.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
DEFAULT_PROXY_RUNTIME: dict[str, Any] = {
|
||||
"enabled": False,
|
||||
"egress_mode": "direct",
|
||||
"proxy_url": "",
|
||||
"resource_proxy_url": "",
|
||||
"skip_ssl_verify": False,
|
||||
"reset_session_status_codes": [403],
|
||||
"clearance": {
|
||||
"enabled": False,
|
||||
"mode": "none",
|
||||
"cf_cookies": "",
|
||||
"cf_clearance": "",
|
||||
"user_agent": DEFAULT_USER_AGENT,
|
||||
"browser": "chrome",
|
||||
"flaresolverr_url": "",
|
||||
"timeout_sec": 60,
|
||||
"refresh_interval": 3600,
|
||||
"warm_up_on_start": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
value = os.getenv(name)
|
||||
if value is None:
|
||||
return default
|
||||
lowered = value.strip().lower()
|
||||
if lowered in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if lowered in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, minimum: int = 0) -> int:
|
||||
try:
|
||||
return max(minimum, int(os.getenv(name, str(default))))
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return default
|
||||
|
||||
|
||||
def _warp_runtime_defaults() -> dict[str, Any]:
|
||||
clearance_enabled = _env_bool("CHATGPT2API_PROXY_RUNTIME_CLEARANCE_ENABLED", True)
|
||||
clearance_mode = os.getenv("CHATGPT2API_PROXY_RUNTIME_CLEARANCE_MODE", "flaresolverr").strip().lower()
|
||||
if clearance_mode not in {"none", "manual", "flaresolverr"}:
|
||||
clearance_mode = "flaresolverr" if clearance_enabled else "none"
|
||||
if not clearance_enabled:
|
||||
clearance_mode = "none"
|
||||
|
||||
egress_mode = os.getenv("CHATGPT2API_PROXY_RUNTIME_EGRESS_MODE", "single_proxy").strip().lower()
|
||||
if egress_mode not in {"direct", "single_proxy"}:
|
||||
egress_mode = "single_proxy"
|
||||
|
||||
runtime = copy.deepcopy(DEFAULT_PROXY_RUNTIME)
|
||||
runtime.update(
|
||||
{
|
||||
"enabled": _env_bool("CHATGPT2API_PROXY_RUNTIME_ENABLED", True),
|
||||
"egress_mode": egress_mode,
|
||||
"proxy_url": os.getenv("CHATGPT2API_PROXY_RUNTIME_PROXY_URL", "http://privoxy:8118").strip(),
|
||||
"resource_proxy_url": os.getenv("CHATGPT2API_PROXY_RUNTIME_RESOURCE_PROXY_URL", "").strip(),
|
||||
"skip_ssl_verify": _env_bool("CHATGPT2API_PROXY_RUNTIME_SKIP_SSL_VERIFY", False),
|
||||
"reset_session_status_codes": [
|
||||
int(part.strip())
|
||||
for part in os.getenv("CHATGPT2API_PROXY_RUNTIME_RESET_STATUS_CODES", "403").split(",")
|
||||
if part.strip().isdigit() and 100 <= int(part.strip()) <= 599
|
||||
] or [403],
|
||||
}
|
||||
)
|
||||
runtime["clearance"].update(
|
||||
{
|
||||
"enabled": clearance_enabled,
|
||||
"mode": clearance_mode,
|
||||
"user_agent": os.getenv("CHATGPT2API_PROXY_RUNTIME_USER_AGENT", DEFAULT_USER_AGENT).strip() or DEFAULT_USER_AGENT,
|
||||
"browser": os.getenv("CHATGPT2API_PROXY_RUNTIME_BROWSER", "chrome").strip() or "chrome",
|
||||
"flaresolverr_url": os.getenv("CHATGPT2API_FLARESOLVERR_URL", "http://flaresolverr:8191").strip(),
|
||||
"timeout_sec": _env_int("CHATGPT2API_PROXY_RUNTIME_CLEARANCE_TIMEOUT_SEC", 60, 1),
|
||||
"refresh_interval": _env_int("CHATGPT2API_PROXY_RUNTIME_CLEARANCE_REFRESH_INTERVAL", 3600, 60),
|
||||
"warm_up_on_start": _env_bool("CHATGPT2API_PROXY_RUNTIME_WARM_UP_ON_START", False),
|
||||
}
|
||||
)
|
||||
return runtime
|
||||
|
||||
|
||||
def _deep_fill_missing(target: dict[str, Any], defaults: dict[str, Any]) -> bool:
|
||||
changed = False
|
||||
for key, value in defaults.items():
|
||||
if key not in target:
|
||||
target[key] = copy.deepcopy(value)
|
||||
changed = True
|
||||
continue
|
||||
if isinstance(target[key], dict) and isinstance(value, dict):
|
||||
changed = _deep_fill_missing(target[key], value) or changed
|
||||
return changed
|
||||
|
||||
|
||||
def _looks_like_repository_default(runtime: Any) -> bool:
|
||||
if not isinstance(runtime, dict):
|
||||
return True
|
||||
candidate = copy.deepcopy(runtime)
|
||||
_deep_fill_missing(candidate, DEFAULT_PROXY_RUNTIME)
|
||||
return candidate == DEFAULT_PROXY_RUNTIME
|
||||
|
||||
|
||||
def _mask_url(value: str) -> str:
|
||||
return re.sub(r"(https?://)([^\s/@:]+):([^\s/@]+)@", r"\1[REDACTED]@", value or "", flags=re.I)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
config_path = Path(os.getenv("CHATGPT2API_CONFIG_FILE", "/app/config.json"))
|
||||
if not config_path.exists():
|
||||
print(f"Config file not found, creating {config_path}")
|
||||
data: dict[str, Any] = {}
|
||||
else:
|
||||
try:
|
||||
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"Invalid JSON in {config_path}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not isinstance(data, dict):
|
||||
print(f"Config root must be an object: {config_path}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
desired = _warp_runtime_defaults()
|
||||
existing = data.get("proxy_runtime")
|
||||
changed = False
|
||||
|
||||
if _looks_like_repository_default(existing) or _env_bool("CHATGPT2API_PROXY_RUNTIME_FORCE", False):
|
||||
data["proxy_runtime"] = desired
|
||||
changed = True
|
||||
print("Created proxy_runtime defaults")
|
||||
else:
|
||||
runtime = existing if isinstance(existing, dict) else {}
|
||||
changed = _deep_fill_missing(runtime, DEFAULT_PROXY_RUNTIME)
|
||||
data["proxy_runtime"] = runtime
|
||||
print("Proxy runtime already configured")
|
||||
|
||||
if changed:
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = json.dumps(data, ensure_ascii=False, indent=2) + "\n"
|
||||
tmp_path = config_path.with_suffix(config_path.suffix + ".tmp")
|
||||
try:
|
||||
tmp_path.write_text(payload, encoding="utf-8")
|
||||
tmp_path.replace(config_path)
|
||||
except OSError as exc:
|
||||
# Docker bind-mounted single files can reject atomic rename with EBUSY.
|
||||
# Fall back to in-place write so the init job works with both file and
|
||||
# directory mounts.
|
||||
if getattr(exc, "errno", None) != 16:
|
||||
raise
|
||||
config_path.write_text(payload, encoding="utf-8")
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
runtime = data.get("proxy_runtime") if isinstance(data.get("proxy_runtime"), dict) else {}
|
||||
clearance = runtime.get("clearance") if isinstance(runtime.get("clearance"), dict) else {}
|
||||
print(
|
||||
"Proxy runtime summary: "
|
||||
f"enabled={bool(runtime.get('enabled'))}, "
|
||||
f"egress_mode={runtime.get('egress_mode')}, "
|
||||
f"proxy_url={_mask_url(str(runtime.get('proxy_url') or ''))}, "
|
||||
f"clearance_mode={clearance.get('mode')}, "
|
||||
f"flaresolverr_url={_mask_url(str(clearance.get('flaresolverr_url') or ''))}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
存储后端数据迁移脚本
|
||||
|
||||
用法：
|
||||
python scripts/migrate_storage.py --from json --to postgres
|
||||
python scripts/migrate_storage.py --from postgres --to git
|
||||
python scripts/migrate_storage.py --export accounts.json
|
||||
python scripts/migrate_storage.py --import accounts.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目根目录到 Python 路径
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parents[1] / "data"
|
||||
|
||||
from services.storage.factory import create_storage_backend
|
||||
|
||||
|
||||
def export_to_json(output_file: str):
|
||||
"""导出当前存储后端的数据到 JSON 文件"""
|
||||
print(f"[migrate] Exporting data to {output_file}")
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
storage = create_storage_backend(DATA_DIR)
|
||||
accounts = storage.load_accounts()
|
||||
|
||||
output_path = Path(output_file)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(
|
||||
json.dumps(accounts, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"[migrate] Exported {len(accounts)} accounts to {output_file}")
|
||||
|
||||
|
||||
def import_from_json(input_file: str):
|
||||
"""从 JSON 文件导入数据到当前存储后端"""
|
||||
print(f"[migrate] Importing data from {input_file}")
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
input_path = Path(input_file)
|
||||
if not input_path.exists():
|
||||
print(f"[migrate] Error: File not found: {input_file}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
accounts = json.loads(input_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(accounts, list):
|
||||
print(f"[migrate] Error: Invalid JSON format, expected array")
|
||||
sys.exit(1)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"[migrate] Error: Invalid JSON: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
storage = create_storage_backend(DATA_DIR)
|
||||
storage.save_accounts(accounts)
|
||||
|
||||
print(f"[migrate] Imported {len(accounts)} accounts")
|
||||
|
||||
|
||||
def migrate_data(from_backend: str, to_backend: str):
|
||||
"""从一个存储后端迁移到另一个"""
|
||||
print(f"[migrate] Migrating from {from_backend} to {to_backend}")
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
# 保存原始环境变量
|
||||
original_backend = os.environ.get("STORAGE_BACKEND")
|
||||
|
||||
try:
|
||||
# 从源后端读取数据
|
||||
os.environ["STORAGE_BACKEND"] = from_backend
|
||||
from_storage = create_storage_backend(DATA_DIR)
|
||||
accounts = from_storage.load_accounts()
|
||||
print(f"[migrate] Loaded {len(accounts)} accounts from {from_backend}")
|
||||
|
||||
# 写入目标后端
|
||||
os.environ["STORAGE_BACKEND"] = to_backend
|
||||
to_storage = create_storage_backend(DATA_DIR)
|
||||
to_storage.save_accounts(accounts)
|
||||
print(f"[migrate] Saved {len(accounts)} accounts to {to_backend}")
|
||||
|
||||
print(f"[migrate] Migration completed successfully!")
|
||||
|
||||
finally:
|
||||
# 恢复原始环境变量
|
||||
if original_backend:
|
||||
os.environ["STORAGE_BACKEND"] = original_backend
|
||||
elif "STORAGE_BACKEND" in os.environ:
|
||||
del os.environ["STORAGE_BACKEND"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="ChatGPT2API 存储后端数据迁移工具",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
# 从 JSON 迁移到 PostgreSQL
|
||||
python scripts/migrate_storage.py --from json --to postgres
|
||||
|
||||
# 从 PostgreSQL 迁移到 Git
|
||||
python scripts/migrate_storage.py --from postgres --to git
|
||||
|
||||
# 导出当前数据到 JSON 文件
|
||||
python scripts/migrate_storage.py --export backup.json
|
||||
|
||||
# 从 JSON 文件导入数据
|
||||
python scripts/migrate_storage.py --import backup.json
|
||||
|
||||
环境变量:
|
||||
STORAGE_BACKEND - 存储后端类型 (json, sqlite, postgres, git)
|
||||
DATABASE_URL - 数据库连接字符串
|
||||
GIT_REPO_URL - Git 仓库地址
|
||||
GIT_TOKEN - Git 访问令牌
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--from",
|
||||
dest="from_backend",
|
||||
choices=["json", "sqlite", "postgres", "git"],
|
||||
help="源存储后端",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--to",
|
||||
dest="to_backend",
|
||||
choices=["json", "sqlite", "postgres", "git"],
|
||||
help="目标存储后端",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--export",
|
||||
dest="export_file",
|
||||
metavar="FILE",
|
||||
help="导出数据到 JSON 文件",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--import",
|
||||
dest="import_file",
|
||||
metavar="FILE",
|
||||
help="从 JSON 文件导入数据",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 检查参数
|
||||
if args.from_backend and args.to_backend:
|
||||
migrate_data(args.from_backend, args.to_backend)
|
||||
elif args.export_file:
|
||||
export_to_json(args.export_file)
|
||||
elif args.import_file:
|
||||
import_from_json(args.import_file)
|
||||
else:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
listen-address 0.0.0.0:8118
|
||||
toggle 1
|
||||
enable-remote-toggle 0
|
||||
enable-remote-http-toggle 0
|
||||
enable-edit-actions 0
|
||||
enforce-blocks 0
|
||||
buffer-limit 4096
|
||||
keep-alive-timeout 300
|
||||
socket-timeout 300
|
||||
|
||||
# Forward every HTTP/HTTPS request through the WARP SOCKS5 proxy.
|
||||
# The trailing dot is required by Privoxy syntax.
|
||||
forward-socks5t / warp-proxy:1080 .
|
||||
@@ -0,0 +1,385 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import imaplib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from email import message_from_bytes, policy
|
||||
from email.header import decode_header, make_header
|
||||
from email.message import Message
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
TOKEN_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/token"
|
||||
GRAPH_MESSAGES_URL = "https://graph.microsoft.com/v1.0/me/messages"
|
||||
GRAPH_SCOPE = "offline_access https://graph.microsoft.com/Mail.Read"
|
||||
IMAP_SCOPE = "offline_access https://outlook.office.com/IMAP.AccessAsUser.All"
|
||||
DEFAULT_IMAP_HOST = "outlook.office365.com"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OutlookCredential:
|
||||
email: str
|
||||
password: str
|
||||
client_id: str
|
||||
refresh_token: str
|
||||
line_number: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HttpResponse:
|
||||
status_code: int
|
||||
text: str
|
||||
|
||||
def json(self) -> Any:
|
||||
return json.loads(self.text)
|
||||
|
||||
|
||||
def _clean(value: str) -> str:
|
||||
return value.replace("\ufeff", "").replace("\u00a0", " ").strip()
|
||||
|
||||
|
||||
def _redact_email(email: str) -> str:
|
||||
local, sep, domain = email.partition("@")
|
||||
if not sep:
|
||||
return "***"
|
||||
if len(local) <= 2:
|
||||
masked = local[:1] + "***"
|
||||
else:
|
||||
masked = local[:2] + "***" + local[-1:]
|
||||
return f"{masked}@{domain}"
|
||||
|
||||
|
||||
def parse_credentials(path: Path) -> list[OutlookCredential]:
|
||||
credentials: list[OutlookCredential] = []
|
||||
for line_number, raw_line in enumerate(path.read_text(encoding="utf-8-sig").splitlines(), start=1):
|
||||
line = _clean(raw_line)
|
||||
if not line or "----" not in line:
|
||||
continue
|
||||
parts = [_clean(part) for part in line.split("----", 3)]
|
||||
if len(parts) != 4:
|
||||
continue
|
||||
email, password, client_id, refresh_token = parts
|
||||
if "@" not in email or not client_id or not refresh_token:
|
||||
continue
|
||||
credentials.append(
|
||||
OutlookCredential(
|
||||
email=email,
|
||||
password=password,
|
||||
client_id=client_id,
|
||||
refresh_token=refresh_token,
|
||||
line_number=line_number,
|
||||
)
|
||||
)
|
||||
return credentials
|
||||
|
||||
|
||||
def _http_request(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str] | None = None,
|
||||
data: dict[str, str] | None = None,
|
||||
params: dict[str, Any] | None = None,
|
||||
timeout: float = 30,
|
||||
) -> HttpResponse:
|
||||
target = url
|
||||
if params:
|
||||
query = urllib.parse.urlencode(params)
|
||||
target = f"{url}?{query}"
|
||||
|
||||
body: bytes | None = None
|
||||
request_headers = dict(headers or {})
|
||||
if data is not None:
|
||||
body = urllib.parse.urlencode(data).encode("utf-8")
|
||||
request_headers.setdefault("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
request = urllib.request.Request(target, data=body, headers=request_headers, method=method.upper())
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return HttpResponse(
|
||||
status_code=int(response.status),
|
||||
text=response.read().decode("utf-8", errors="replace"),
|
||||
)
|
||||
except urllib.error.HTTPError as error:
|
||||
return HttpResponse(
|
||||
status_code=int(error.code),
|
||||
text=error.read().decode("utf-8", errors="replace"),
|
||||
)
|
||||
|
||||
|
||||
def exchange_refresh_token(credential: OutlookCredential, scope: str, timeout: float) -> str:
|
||||
response = _http_request(
|
||||
"POST",
|
||||
TOKEN_URL,
|
||||
data={
|
||||
"client_id": credential.client_id,
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": credential.refresh_token,
|
||||
"scope": scope,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
try:
|
||||
data = response.json()
|
||||
except Exception:
|
||||
data = {}
|
||||
if response.status_code != 200:
|
||||
detail = data.get("error_description") or data.get("error") or response.text[:300]
|
||||
raise RuntimeError(f"token refresh failed: HTTP {response.status_code}, {detail}")
|
||||
access_token = str(data.get("access_token") or "").strip()
|
||||
if not access_token:
|
||||
raise RuntimeError("token refresh response did not include access_token")
|
||||
return access_token
|
||||
|
||||
|
||||
def _graph_sender(message: dict[str, Any]) -> str:
|
||||
sender = message.get("from") or {}
|
||||
if isinstance(sender, dict):
|
||||
address = sender.get("emailAddress") or {}
|
||||
if isinstance(address, dict):
|
||||
return str(address.get("address") or address.get("name") or "")
|
||||
return ""
|
||||
|
||||
|
||||
def read_graph_messages(access_token: str, limit: int, timeout: float) -> list[dict[str, str]]:
|
||||
response = _http_request(
|
||||
"GET",
|
||||
GRAPH_MESSAGES_URL,
|
||||
headers={"Authorization": f"Bearer {access_token}", "Accept": "application/json"},
|
||||
params={
|
||||
"$top": max(1, min(limit, 50)),
|
||||
"$orderby": "receivedDateTime desc",
|
||||
"$select": "subject,receivedDateTime,from,bodyPreview",
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
try:
|
||||
data = response.json()
|
||||
except Exception:
|
||||
data = {}
|
||||
if response.status_code != 200:
|
||||
detail = data.get("error", {}).get("message") if isinstance(data.get("error"), dict) else response.text[:300]
|
||||
raise RuntimeError(f"graph messages failed: HTTP {response.status_code}, {detail}")
|
||||
items = data.get("value") if isinstance(data, dict) else None
|
||||
if not isinstance(items, list):
|
||||
raise RuntimeError("graph messages response did not include value[]")
|
||||
return [
|
||||
{
|
||||
"received": str(item.get("receivedDateTime") or ""),
|
||||
"from": _graph_sender(item),
|
||||
"subject": str(item.get("subject") or ""),
|
||||
"preview": str(item.get("bodyPreview") or ""),
|
||||
}
|
||||
for item in items
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
|
||||
|
||||
def _decode_header(value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
return str(make_header(decode_header(value)))
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
|
||||
def _message_text_preview(message: Message, limit: int = 240) -> str:
|
||||
text_parts: list[str] = []
|
||||
parts = message.walk() if message.is_multipart() else [message]
|
||||
for part in parts:
|
||||
if part.get_content_maintype() == "multipart":
|
||||
continue
|
||||
content_type = part.get_content_type()
|
||||
if content_type not in {"text/plain", "text/html"}:
|
||||
continue
|
||||
try:
|
||||
payload = part.get_content()
|
||||
except Exception:
|
||||
continue
|
||||
if payload:
|
||||
text_parts.append(str(payload))
|
||||
if content_type == "text/plain" and text_parts:
|
||||
break
|
||||
text = "\n".join(text_parts)
|
||||
text = re.sub(r"<[^>]+>", " ", text)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text[:limit]
|
||||
|
||||
|
||||
def _parse_imap_message(raw: bytes, include_preview: bool) -> dict[str, str]:
|
||||
message = message_from_bytes(raw, policy=policy.default)
|
||||
received = ""
|
||||
try:
|
||||
parsed = parsedate_to_datetime(str(message.get("Date") or ""))
|
||||
received = parsed.isoformat()
|
||||
except Exception:
|
||||
received = str(message.get("Date") or "")
|
||||
return {
|
||||
"received": received,
|
||||
"from": _decode_header(str(message.get("From") or "")),
|
||||
"subject": _decode_header(str(message.get("Subject") or "")),
|
||||
"preview": _message_text_preview(message) if include_preview else "",
|
||||
}
|
||||
|
||||
|
||||
def read_imap_messages(access_token: str, email: str, host: str, limit: int, include_preview: bool) -> list[dict[str, str]]:
|
||||
auth_string = f"user={email}\x01auth=Bearer {access_token}\x01\x01"
|
||||
mailbox = imaplib.IMAP4_SSL(host)
|
||||
try:
|
||||
mailbox.authenticate("XOAUTH2", lambda _: auth_string.encode("utf-8"))
|
||||
status, _ = mailbox.select("INBOX", readonly=True)
|
||||
if status != "OK":
|
||||
raise RuntimeError("imap select INBOX failed")
|
||||
status, data = mailbox.uid("search", None, "ALL")
|
||||
if status != "OK" or not data or not data[0]:
|
||||
return []
|
||||
uids = data[0].split()[-max(1, limit) :]
|
||||
messages: list[dict[str, str]] = []
|
||||
for uid in reversed(uids):
|
||||
status, fetched = mailbox.uid("fetch", uid, "(RFC822)")
|
||||
if status != "OK":
|
||||
continue
|
||||
raw_payload = next((item[1] for item in fetched if isinstance(item, tuple) and isinstance(item[1], bytes)), b"")
|
||||
if raw_payload:
|
||||
messages.append(_parse_imap_message(raw_payload, include_preview))
|
||||
return messages
|
||||
finally:
|
||||
try:
|
||||
mailbox.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def print_messages(messages: list[dict[str, str]], include_preview: bool) -> None:
|
||||
if not messages:
|
||||
print(" no recent messages")
|
||||
return
|
||||
for index, message in enumerate(messages, start=1):
|
||||
print(f" {index}. {message['received']} | {message['from']} | {message['subject']}")
|
||||
if include_preview and message.get("preview"):
|
||||
print(f" preview: {message['preview']}")
|
||||
|
||||
|
||||
def test_credential(
|
||||
credential: OutlookCredential,
|
||||
mode: str,
|
||||
limit: int,
|
||||
timeout: float,
|
||||
imap_host: str,
|
||||
include_preview: bool,
|
||||
show_email: bool,
|
||||
) -> bool:
|
||||
label = credential.email if show_email else _redact_email(credential.email)
|
||||
print(f"[line {credential.line_number}] {label}")
|
||||
errors: list[str] = []
|
||||
|
||||
if mode in {"graph", "auto"}:
|
||||
try:
|
||||
access_token = exchange_refresh_token(credential, GRAPH_SCOPE, timeout)
|
||||
messages = read_graph_messages(access_token, limit, timeout)
|
||||
print(" graph: ok")
|
||||
print_messages(messages, include_preview)
|
||||
return True
|
||||
except Exception as error:
|
||||
errors.append(f"graph: {error}")
|
||||
if mode == "graph":
|
||||
print(f" graph: failed - {error}")
|
||||
return False
|
||||
|
||||
if mode in {"imap", "auto"}:
|
||||
try:
|
||||
access_token = exchange_refresh_token(credential, IMAP_SCOPE, timeout)
|
||||
messages = read_imap_messages(access_token, credential.email, imap_host, limit, include_preview)
|
||||
print(" imap: ok")
|
||||
print_messages(messages, include_preview)
|
||||
return True
|
||||
except Exception as error:
|
||||
errors.append(f"imap: {error}")
|
||||
if mode == "imap":
|
||||
print(f" imap: failed - {error}")
|
||||
return False
|
||||
|
||||
print(" failed")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
return False
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Test Outlook/Hotmail mailbox access from lines formatted as email----password----client_id----refresh_token.",
|
||||
)
|
||||
parser.add_argument("--file", default=r"D:\Desktop\yx.txt", help="Credential text file path.")
|
||||
parser.add_argument("--mode", choices=("auto", "graph", "imap"), default="auto", help="Mailbox read method.")
|
||||
parser.add_argument("--limit-accounts", type=int, default=1, help="How many accounts to test.")
|
||||
parser.add_argument("--message-limit", type=int, default=5, help="How many recent messages to list per account.")
|
||||
parser.add_argument("--timeout", type=float, default=30, help="HTTP request timeout in seconds.")
|
||||
parser.add_argument("--imap-host", default=DEFAULT_IMAP_HOST, help="IMAP host for XOAUTH2 mode.")
|
||||
parser.add_argument("--preview", action="store_true", help="Print body preview/snippet. Disabled by default.")
|
||||
parser.add_argument("--show-email", action="store_true", help="Print full email address. Secrets are never printed.")
|
||||
parser.add_argument("--json", action="store_true", help="Only parse the file and print non-secret account metadata as JSON.")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
path = Path(args.file)
|
||||
if not path.exists():
|
||||
print(f"file not found: {path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
credentials = parse_credentials(path)
|
||||
if not credentials:
|
||||
print(f"no valid credentials parsed from: {path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
limit_accounts = max(1, int(args.limit_accounts or 1))
|
||||
selected = credentials[:limit_accounts]
|
||||
|
||||
if args.json:
|
||||
print(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"line": item.line_number,
|
||||
"email": item.email if args.show_email else _redact_email(item.email),
|
||||
"client_id": item.client_id,
|
||||
"has_password": bool(item.password),
|
||||
"has_refresh_token": bool(item.refresh_token),
|
||||
}
|
||||
for item in selected
|
||||
],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
print(f"parsed {len(credentials)} credential(s), testing {len(selected)}")
|
||||
success = 0
|
||||
for credential in selected:
|
||||
if test_credential(
|
||||
credential=credential,
|
||||
mode=str(args.mode),
|
||||
limit=max(1, int(args.message_limit or 1)),
|
||||
timeout=max(1.0, float(args.timeout or 30)),
|
||||
imap_host=str(args.imap_host),
|
||||
include_preview=bool(args.preview),
|
||||
show_email=bool(args.show_email),
|
||||
):
|
||||
success += 1
|
||||
print(f"summary: {success}/{len(selected)} account(s) readable")
|
||||
return 0 if success == len(selected) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
存储后端测试脚本
|
||||
|
||||
用法：
|
||||
python scripts/test_storage.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目根目录到 Python 路径
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parents[1] / "data"
|
||||
|
||||
from services.storage.factory import create_storage_backend
|
||||
|
||||
|
||||
def test_storage():
|
||||
"""测试当前配置的存储后端"""
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
print("=" * 60)
|
||||
print("ChatGPT2API 存储后端测试")
|
||||
print("=" * 60)
|
||||
|
||||
# 显示当前配置
|
||||
backend_type = os.getenv("STORAGE_BACKEND", "json")
|
||||
print(f"\n当前存储后端: {backend_type}")
|
||||
|
||||
if backend_type in ("sqlite", "postgres", "postgresql", "mysql", "database"):
|
||||
database_url = os.getenv("DATABASE_URL", "")
|
||||
if database_url:
|
||||
# 隐藏密码
|
||||
if "://" in database_url and "@" in database_url:
|
||||
protocol, rest = database_url.split("://", 1)
|
||||
if "@" in rest:
|
||||
credentials, host = rest.split("@", 1)
|
||||
if ":" in credentials:
|
||||
username, _ = credentials.split(":", 1)
|
||||
database_url = f"{protocol}://{username}:****@{host}"
|
||||
print(f"数据库连接: {database_url}")
|
||||
else:
|
||||
print(f"数据库连接: 本地 SQLite (data/accounts.db)")
|
||||
|
||||
elif backend_type == "git":
|
||||
repo_url = os.getenv("GIT_REPO_URL", "")
|
||||
branch = os.getenv("GIT_BRANCH", "main")
|
||||
file_path = os.getenv("GIT_FILE_PATH", "accounts.json")
|
||||
print(f"Git 仓库: {repo_url}")
|
||||
print(f"Git 分支: {branch}")
|
||||
print(f"文件路径: {file_path}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
try:
|
||||
# 创建存储后端
|
||||
print("\n[1/5] 创建存储后端...")
|
||||
storage = create_storage_backend(DATA_DIR)
|
||||
print("✅ 存储后端创建成功")
|
||||
|
||||
# 获取后端信息
|
||||
print("\n[2/5] 获取后端信息...")
|
||||
info = storage.get_backend_info()
|
||||
print(f"✅ 后端类型: {info.get('type')}")
|
||||
print(f" 描述: {info.get('description')}")
|
||||
for key, value in info.items():
|
||||
if key not in ('type', 'description'):
|
||||
print(f" {key}: {value}")
|
||||
|
||||
# 健康检查
|
||||
print("\n[3/5] 执行健康检查...")
|
||||
health = storage.health_check()
|
||||
status = health.get("status")
|
||||
if status == "healthy":
|
||||
print(f"✅ 健康状态: {status}")
|
||||
else:
|
||||
print(f"❌ 健康状态: {status}")
|
||||
print(f" 错误: {health.get('error')}")
|
||||
return False
|
||||
|
||||
# 读取数据
|
||||
print("\n[4/5] 读取账号数据...")
|
||||
accounts = storage.load_accounts()
|
||||
print(f"✅ 成功读取 {len(accounts)} 个账号")
|
||||
|
||||
# 写入测试（可选）
|
||||
print("\n[5/5] 测试写入功能...")
|
||||
test_account = {
|
||||
"access_token": "test_token_" + str(os.getpid()),
|
||||
"type": "Free",
|
||||
"status": "测试",
|
||||
"quota": 0,
|
||||
"email": "test@example.com",
|
||||
}
|
||||
|
||||
# 添加测试账号
|
||||
test_accounts = accounts + [test_account]
|
||||
storage.save_accounts(test_accounts)
|
||||
print("✅ 写入测试账号成功")
|
||||
|
||||
# 验证写入
|
||||
reloaded = storage.load_accounts()
|
||||
if len(reloaded) == len(test_accounts):
|
||||
print("✅ 验证写入成功")
|
||||
else:
|
||||
print(f"❌ 验证失败: 期望 {len(test_accounts)} 个账号，实际 {len(reloaded)} 个")
|
||||
return False
|
||||
|
||||
# 恢复原始数据
|
||||
storage.save_accounts(accounts)
|
||||
print("✅ 恢复原始数据")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ 所有测试通过！")
|
||||
print("=" * 60)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ 测试失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = test_storage()
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -0,0 +1,92 @@
|
||||
"""验证 OAuth 账号的自动刷新是否生效。
|
||||
|
||||
用法（在容器内，工作目录 /app）：
|
||||
uv run python scripts/verify_oauth_refresh.py # 只读诊断，安全
|
||||
uv run python scripts/verify_oauth_refresh.py --force # 真正触发一次刷新
|
||||
|
||||
只读模式：列出每个账号的 access_token 剩余有效期、是否带 refresh_token、
|
||||
上次刷新时间与上次刷新错误，判断"有没有料可刷"。
|
||||
--force ：对每个带 refresh_token 的账号强制走一次 refresh_access_token(force=True)，
|
||||
直接验证 refresh_token + 本项目 client_id 能否从 OpenAI 换出新 access_token。
|
||||
这会真实轮换 access_token（新 token 有效，不损坏账号）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from services.account_service import account_service
|
||||
|
||||
|
||||
def _fmt_remaining(seconds: int | None) -> str:
|
||||
if seconds is None:
|
||||
return "无法解析 exp"
|
||||
if seconds <= 0:
|
||||
return f"已过期 {(-seconds) // 3600}h"
|
||||
return f"{seconds // 3600}h{(seconds % 3600) // 60}m 后过期"
|
||||
|
||||
|
||||
def diagnose() -> list[str]:
|
||||
"""只读：打印每个账号的刷新就绪状态，返回带 refresh_token 的 token 列表。"""
|
||||
tokens = account_service.list_tokens()
|
||||
print(f"账号总数: {len(tokens)}\n")
|
||||
refreshable: list[str] = []
|
||||
for token in tokens:
|
||||
account = account_service.get_account(token) or {}
|
||||
remaining = account_service._token_expires_in(token)
|
||||
has_rt = bool(str(account.get("refresh_token") or "").strip())
|
||||
if has_rt:
|
||||
refreshable.append(token)
|
||||
print(f"- email={account.get('email') or '(未知)'}")
|
||||
print(f" access_token[:20] = {token[:20]}...")
|
||||
print(f" 距过期 = {_fmt_remaining(remaining)}")
|
||||
print(f" refresh_token = {'有 ✅' if has_rt else '无 ❌（无法自动刷新）'}")
|
||||
print(f" last_token_refresh_at = {account.get('last_token_refresh_at')}")
|
||||
print(f" last_token_refresh_error = {account.get('last_token_refresh_error')}")
|
||||
print()
|
||||
return refreshable
|
||||
|
||||
|
||||
def force_refresh(tokens: list[str]) -> None:
|
||||
"""对每个账号 force 刷新一次，并对比前后状态判断成败。"""
|
||||
if not tokens:
|
||||
print("没有带 refresh_token 的账号，无法验证刷新。")
|
||||
return
|
||||
print("=" * 60)
|
||||
print(f"开始对 {len(tokens)} 个账号 force 刷新（真实调用 OpenAI）...\n")
|
||||
ok = 0
|
||||
for token in tokens:
|
||||
before = account_service.get_account(token) or {}
|
||||
new_token = account_service.refresh_access_token(token, force=True, event="manual_verify")
|
||||
after = account_service.get_account(new_token) or {}
|
||||
err = str(after.get("last_token_refresh_error") or "").strip()
|
||||
rotated = new_token != token
|
||||
success = bool(new_token) and not err
|
||||
if success:
|
||||
ok += 1
|
||||
print(f"- email={before.get('email') or '(未知)'}")
|
||||
print(f" 旧 access_token[:20] = {token[:20]}...")
|
||||
print(f" 新 access_token[:20] = {new_token[:20]}...")
|
||||
print(f" token 是否轮换 = {'是' if rotated else '否（exp 未到刷新窗口时可能返回原值）'}")
|
||||
print(f" last_token_refresh_at = {after.get('last_token_refresh_at')}")
|
||||
print(f" last_token_refresh_error = {after.get('last_token_refresh_error') or '无'}")
|
||||
print(f" >>> 刷新结果 = {'成功 ✅' if success else '失败 ❌'}")
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(f"汇总: {ok}/{len(tokens)} 个账号刷新成功")
|
||||
if ok == len(tokens):
|
||||
print("✅ 自动刷新机制对这些账号完全可用——refresh_token 与 client_id 匹配。")
|
||||
else:
|
||||
print("❌ 有账号刷新失败，看上面的 last_token_refresh_error，或 docker logs 里的 [oauth-login]/refresh 日志。")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
do_force = "--force" in sys.argv[1:]
|
||||
refreshable = diagnose()
|
||||
if do_force:
|
||||
force_refresh(refreshable)
|
||||
else:
|
||||
print("提示: 加 --force 参数可真正触发一次刷新以验证能否成功。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user