Files
routstr-core/routstr/core/exceptions.py
redshift 6a8194e4d1 fix(ehbp): relay encrypted enclave error responses verbatim
EHBP encrypts response bodies, error bodies included. On a non-200 the
proxy decoded the ciphertext as UTF-8 and wrapped it in its own JSON
error envelope, so callers saw mojibake instead of the enclave's actual
reason (a real-world 404 caused by an upstream model id surfaced only as
`EHBP upstream tinfoil returned 404 ... <binary garbage>`), and the
provider status was lost behind the 424 mapping.

When the enclave response carries `Ehbp-Response-Nonce`, reconstruct the
reply verbatim -- status, nonce header and ciphertext body -- and raise
`EhbpEncryptedUpstreamError`. Failover, cooldown accounting and
reservation release are unchanged; once the provider chain is exhausted,
`proxy.py` returns the enclave reply instead of a proxy envelope, and the
EHBP client decrypts the provider's real error.

Already-plaintext non-200s (router/gateway rejections without a nonce)
keep the existing structured-error handling. X-Cashu requests refund the
token and relay the ciphertext together with the `X-Cashu` refund header.

Ciphertext is also kept out of the proxy's error logs and messages; the
log line records the byte count instead.
2026-10-17 17:18:20 +00:00

264 lines
9.5 KiB
Python

import math
from fastapi import Request
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response
from .error_scope import (
ERROR_SCOPE_UPSTREAM,
UPSTREAM_ERROR_STATUS,
UPSTREAM_UNAVAILABLE,
)
from .logging import get_logger
logger = get_logger(__name__)
class UpstreamError(Exception):
"""Exception raised when an upstream provider fails.
``code`` carries a stable, machine-readable classification (e.g.
``UPSTREAM_RATE_LIMIT``) so callers can distinguish failure kinds without
string-matching the message. ``details`` holds optional structured,
redaction-safe context. Both default to ``None`` for backwards
compatibility.
``from_upstream_response`` is True only when ``status_code`` is the status
the upstream itself answered with, as opposed to a status this proxy chose
for a transport failure, timeout or internal fault. Callers use it to
decide whether a status is safe to retry.
``scope`` is ``"upstream"`` for provider failures, reported to the caller
as ``424`` (see :mod:`routstr.core.error_scope`), or ``"node"`` for local
faults, which keep their status. ``status_code`` stays the provider's own
status whenever ``from_upstream_response`` is True; the caller-visible
mapping happens at response construction. Proxy-chosen statuses (transport
failure, timeout) may already be the caller-visible one — only read
``status_code`` as a provider status behind ``from_upstream_response``.
``upstream_body`` holds the redacted upstream error body. It is internal
and never sent to the client.
"""
def __init__(
self,
message: str,
status_code: int = 502,
code: str | None = None,
details: dict[str, object] | None = None,
from_upstream_response: bool = False,
scope: str = ERROR_SCOPE_UPSTREAM,
upstream_body: str | None = None,
):
self.message = message
self.status_code = status_code
self.code = code
self.details = details
self.from_upstream_response = from_upstream_response
self.scope = scope
self.upstream_body = upstream_body
super().__init__(message)
class EhbpEncryptedUpstreamError(UpstreamError):
"""Non-200 EHBP enclave response whose body is end-to-end encrypted.
EHBP encrypts response bodies, errors included. The proxy is a blind relay
and cannot decrypt them, so wrapping the bytes in a proxy JSON envelope
(as :func:`create_upstream_error_response` does) hands the caller mojibake
instead of the enclave's actual error. ``response`` holds the enclave reply
reconstructed verbatim — status, ``Ehbp-Response-Nonce`` and the ciphertext
body — and ``proxy.py`` returns it once the provider chain is exhausted, so
the EHBP client can decrypt the real failure.
Until then the error behaves like any other provider failure: it can fail
over to the next candidate, is counted for cooldown purposes, and its
reservation is released when no provider is left.
"""
def __init__(self, message: str, *, status_code: int, response: Response):
super().__init__(
message,
status_code=status_code,
from_upstream_response=True,
scope=ERROR_SCOPE_UPSTREAM,
)
self.response = response
class EhbpTimeoutError(UpstreamError):
"""Raised when an EHBP upstream times out waiting for a response.
Distinct from a generic :class:`UpstreamError` so callers can map the
failure to a stable ``UPSTREAM_TIMEOUT`` code instead of a misleading
``500`` internal server error. Reported as ``424``: the timeout happened on
the provider hop, not this node.
``details`` carries optional structured, redaction-safe context and is
forwarded to the client by ``create_upstream_error_response``.
"""
def __init__(self, message: str, details: dict[str, object] | None = None):
super().__init__(
message,
status_code=UPSTREAM_ERROR_STATUS,
code="UPSTREAM_TIMEOUT",
details=details,
)
class EhbpConnectionError(UpstreamError):
"""Raised when an EHBP upstream cannot be reached.
Covers transport failures while establishing the provider connection: DNS
resolution, TCP refused/reset, or a TLS error that is not a handshake
timeout. Distinct from a generic :class:`UpstreamError` so the failure is
attributed to the provider hop (``UPSTREAM_UNAVAILABLE``, reported as
``424``) instead of being flattened into a misleading node-scoped ``500``.
``details`` carries optional structured, redaction-safe context and is
forwarded to the client by ``create_upstream_error_response``.
"""
def __init__(self, message: str, details: dict[str, object] | None = None):
super().__init__(
message,
status_code=UPSTREAM_ERROR_STATUS,
code=UPSTREAM_UNAVAILABLE,
details=details,
)
def _error_message_from_detail(detail: object) -> str | None:
"""Extract a message from an HTTPException ``detail``, capped at 200 chars."""
if isinstance(detail, dict):
error = detail.get("error")
if isinstance(error, dict):
msg = error.get("message")
return str(msg)[:200] if isinstance(msg, str) else None
if isinstance(error, str):
return error[:200]
return None
if isinstance(detail, str):
return detail[:200]
return None
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Handle HTTP exceptions and include request ID in response."""
request_id = getattr(request.state, "request_id", "unknown")
# Get status code and detail - works for both FastAPI and Starlette HTTPException
status_code = getattr(exc, "status_code", 500)
detail = getattr(exc, "detail", str(exc))
path = request.url.path
error_type: str | None = None
error_code: str | None = None
if isinstance(detail, dict):
error = detail.get("error")
if isinstance(error, dict):
error_type = error.get("type")
error_code = error.get("code")
# 5xx logs as error/warning, 4xx at INFO.
if status_code >= 500:
log_fn = (
logger.warning
if error_type in {"mint_unreachable", "mint_rate_limited"}
else logger.error
)
else:
log_fn = logger.info
log_fn(
f"HTTP {status_code} on {path}: {detail}",
extra={
"request_id": request_id,
"status_code": status_code,
"detail": detail,
"path": path,
"error_type": error_type,
"error_code": error_code,
"level": "http" if status_code < 500 else "server",
},
)
# Stash for LoggingMiddleware's completion log.
request.state.error_detail = {
"error_type": error_type,
"error_code": error_code,
"error_message": _error_message_from_detail(detail),
}
if isinstance(detail, dict) and "error" in detail:
content = {"detail": detail, **detail}
else:
content = {"detail": detail}
content["request_id"] = request_id
headers = getattr(exc, "headers", None)
return JSONResponse(status_code=status_code, content=content, headers=headers)
def json_compliant(value: object) -> object:
"""Render non-finite floats as text so a reply carrying them can serialize.
``json`` parses the bare ``NaN``/``Infinity``/``-Infinity`` literals into
real floats, so a request body — and a stored row written from one — may
hold one anywhere. ``JSONResponse`` encodes with ``allow_nan=False`` and
raises on them, which would turn a reply that merely *quotes* the offending
value into a 500.
"""
if isinstance(value, float) and not math.isfinite(value):
return repr(value)
if isinstance(value, dict):
return {key: json_compliant(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [json_compliant(item) for item in value]
return value
async def validation_exception_handler(
request: Request, exc: Exception
) -> JSONResponse:
"""Answer a request-validation failure with a 422 that always serializes.
Pydantic echoes the rejected value back in each error's ``input`` field. A
non-finite float there breaks the encoder, so the 422 escapes as a 500 and
reports a client's bad rate as a server fault.
"""
request_id = getattr(request.state, "request_id", "unknown")
errors = exc.errors() if isinstance(exc, RequestValidationError) else []
return JSONResponse(
status_code=422,
content={
"detail": json_compliant(jsonable_encoder(errors)),
"request_id": request_id,
},
)
async def general_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Handle general exceptions and include request ID in response."""
request_id = getattr(request.state, "request_id", "unknown")
logger.error(
"Unhandled exception",
extra={
"request_id": request_id,
"error": str(exc),
"error_type": type(exc).__name__,
"path": request.url.path,
},
exc_info=True,
)
return JSONResponse(
status_code=500,
content={
"detail": "Internal server error, please contact support with the request ID.",
"request_id": request_id,
},
)