71 lines
2.4 KiB
Python
71 lines
2.4 KiB
Python
"""Shared retry and cached-failure detection for the live profiling stages."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
|
|
|
|
def _positive_int(name: str, default: int) -> int:
|
|
try:
|
|
return max(1, int(os.getenv(name, default)))
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
def _positive_float(name: str, default: float) -> float:
|
|
try:
|
|
return max(0.0, float(os.getenv(name, default)))
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
MAX_REQUEST_ATTEMPTS = _positive_int("PROFILE_MAX_REQUEST_ATTEMPTS", 5)
|
|
RETRY_BASE_SECONDS = _positive_float("PROFILE_RETRY_BASE_SECONDS", 15.0)
|
|
RETRY_MAX_SECONDS = _positive_float("PROFILE_RETRY_MAX_SECONDS", 120.0)
|
|
REQUEST_INTERVAL_SECONDS = _positive_float("PROFILE_REQUEST_INTERVAL_SECONDS", 1.0)
|
|
REQUEST_TIMEOUT_SECONDS = _positive_float("PROFILE_REQUEST_TIMEOUT_SECONDS", 90.0)
|
|
STREAM_HEARTBEAT_SECONDS = _positive_float("PROFILE_STREAM_HEARTBEAT_SECONDS", 15.0)
|
|
|
|
|
|
def retry_delay_seconds(attempt: int) -> float:
|
|
"""Return capped exponential backoff for a one-based failed attempt."""
|
|
delay = RETRY_BASE_SECONDS
|
|
for _ in range(max(0, attempt - 1)):
|
|
if delay >= RETRY_MAX_SECONDS:
|
|
return RETRY_MAX_SECONDS
|
|
delay *= 2
|
|
return min(RETRY_MAX_SECONDS, delay)
|
|
|
|
|
|
def response_is_retryable_failure(response: str) -> bool:
|
|
"""Identify API-failure and no-credential simulation response sentinels."""
|
|
normalized = response.lstrip().lower()
|
|
return normalized.startswith("error: api call failed for ") or (
|
|
normalized.startswith("this is a simulated response from ")
|
|
and "because no provider api key was provided" in normalized
|
|
)
|
|
|
|
|
|
def evaluation_is_retryable_failure(evaluation: dict) -> bool:
|
|
"""Identify evaluator failures and old zero scores produced from API errors."""
|
|
score = evaluation.get("score")
|
|
if score is None or (
|
|
isinstance(score, str)
|
|
and score in {"error", "evaluator_error", "simulated"}
|
|
):
|
|
return True
|
|
if isinstance(score, str) and score.upper() not in {
|
|
"E", "I", "S", "N", "T", "F", "J", "P"
|
|
}:
|
|
try:
|
|
float(score)
|
|
except ValueError:
|
|
return True
|
|
elif not isinstance(score, (int, float)) or isinstance(score, bool):
|
|
return True
|
|
details = " ".join(
|
|
str(evaluation.get(key, "")) for key in ("justification", "raw_response")
|
|
).lower()
|
|
return bool(re.search(r"rate limit|tpm limit|api error message", details))
|