fmt

Multi-Provider Fallback on UpstreamError

.github/workflows/test.yml

@@ -59,25 +59,30 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
           node-version: "18"
-          cache: "npm"
-          cache-dependency-path: ui/package-lock.json
+          cache: "pnpm"
+          cache-dependency-path: ui/pnpm-lock.yaml
 
       - name: Install UI dependencies
         working-directory: ./ui
-        run: npm ci
+        run: pnpm install --frozen-lockfile
 
       - name: Run UI format check
         working-directory: ./ui
-        run: npm run format-check
+        run: pnpm run format-check
 
       - name: Run UI linting
         working-directory: ./ui
-        run: npm run lint
+        run: pnpm run lint
 
       - name: Run UI build
         working-directory: ./ui
-        run: npm run build
+        run: pnpm run build

examples/create_child_keys.py

@@ -0,0 +1,45 @@
+import json
+import sys
+
+import httpx
+
+
+def create_child_keys(base_url: str, api_key: str, count: int = 3) -> list[str]:
+    headers = {"Authorization": f"Bearer {api_key}"}
+
+    print(f"Requesting {count} child keys from {base_url}...")
+
+    child_keys = []
+
+    for i in range(count):
+        try:
+            response = httpx.post(f"{base_url}/v1/balance/child-key", headers=headers)
+            if response.status_code == 200:
+                data = response.json()
+                child_keys.append(data["api_key"])
+                print(
+                    f"  [{i + 1}] Created: {data['api_key']} (Cost: {data['cost_msats']} msats)"
+                )
+            else:
+                print(f"  [{i + 1}] Failed: {response.status_code} - {response.text}")
+        except Exception as e:
+            print(f"  [{i + 1}] Error: {str(e)}")
+
+    return child_keys
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python create_child_keys.py <api_key_or_cashu_token> [base_url]")
+        sys.exit(1)
+
+    auth_key = sys.argv[1]
+    base_url = sys.argv[2] if len(sys.argv) > 2 else "http://localhost:8000"
+
+    keys = create_child_keys(base_url, auth_key)
+
+    if keys:
+        print("\nSuccessfully created child keys:")
+        print(json.dumps(keys, indent=2))
+    else:
+        print("\nNo child keys were created.")

migrations/versions/a86e5348850b_.py

@@ -0,0 +1,42 @@
+"""
+
+Revision ID: a86e5348850b
+Revises: b9667ffc5701
+Create Date: 2026-01-10 18:57:48.475781
+"""
+
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a86e5348850b"
+down_revision = "b9667ffc5701"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Use batch_alter_table for SQLite compatibility
+    with op.batch_alter_table("api_keys", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "parent_key_hash", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.create_index(
+            batch_op.f("ix_api_keys_parent_key_hash"), ["parent_key_hash"], unique=False
+        )
+        batch_op.create_foreign_key(
+            "fk_api_keys_parent_key_hash",
+            "api_keys",
+            ["parent_key_hash"],
+            ["hashed_key"],
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("api_keys", schema=None) as batch_op:
+        batch_op.drop_constraint("fk_api_keys_parent_key_hash", type_="foreignkey")
+        batch_op.drop_index(batch_op.f("ix_api_keys_parent_key_hash"))
+        batch_op.drop_column("parent_key_hash")

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "routstr"
-version = "0.2.1"
+version = "0.3.0"
 description = "Payment proxy for your LLM endpoint using cashu and nostr."
 readme = "README.md"
 requires-python = ">=3.11"

routstr/algorithm.py

@@ -84,93 +84,26 @@ def get_provider_penalty(provider: "BaseUpstreamProvider") -> float:
     return penalty
 
 
-def should_prefer_model(
-    candidate_model: "Model",
-    candidate_provider: "BaseUpstreamProvider",
-    current_model: "Model",
-    current_provider: "BaseUpstreamProvider",
-    alias: str,
-) -> bool:
-    """Determine if candidate model should replace current model for an alias.
-
-    This is the core decision function for model prioritization. It considers:
-    1. Alias matching quality (exact match vs. canonical slug match)
-    2. Model cost (lower is better)
-    3. Provider penalties (e.g., slight preference against OpenRouter)
-
-    Args:
-        candidate_model: The new model being considered
-        candidate_provider: Provider offering the candidate model
-        current_model: The currently selected model for this alias
-        current_provider: Provider offering the current model
-        alias: The model alias being mapped
-
-    Returns:
-        True if candidate should replace current, False otherwise
-    """
-
-    def get_base_model_id(model_id: str) -> str:
-        """Get base model ID by removing provider prefix."""
-        return model_id.split("/", 1)[1] if "/" in model_id else model_id
-
-    def alias_priority(model: "Model") -> int:
-        """Rank how strong the mapping of alias->model is.
-
-        Highest priority when alias exactly equals the model ID without provider prefix.
-        Next when alias equals canonical slug without prefix. Otherwise lowest.
-        """
-        model_base = get_base_model_id(model.id)
-        if model_base == alias:
-            return 3
-        if model.canonical_slug:
-            canonical_base = get_base_model_id(model.canonical_slug)
-            if canonical_base == alias:
-                return 2
-        return 1
-
-    candidate_alias_priority = alias_priority(candidate_model)
-    current_alias_priority = alias_priority(current_model)
-
-    # If candidate has better alias match, prefer it regardless of cost
-    if candidate_alias_priority > current_alias_priority:
-        return True
-
-    # If current has better alias match, keep it regardless of cost
-    if current_alias_priority > candidate_alias_priority:
-        return False
-
-    # Same alias priority - compare costs
-    candidate_cost = calculate_model_cost_score(candidate_model)
-    current_cost = calculate_model_cost_score(current_model)
-
-    # Apply provider penalties
-    candidate_adjusted = candidate_cost * get_provider_penalty(candidate_provider)
-    current_adjusted = current_cost * get_provider_penalty(current_provider)
-
-    # Prefer lower adjusted cost
-    should_replace = candidate_adjusted < current_adjusted
-
-    return should_replace
-
-
 def create_model_mappings(
     upstreams: list["BaseUpstreamProvider"],
     overrides_by_id: dict[str, tuple],
     disabled_model_ids: set[str],
-) -> tuple[dict[str, "Model"], dict[str, "BaseUpstreamProvider"], dict[str, "Model"]]:
+) -> tuple[
+    dict[str, "Model"], dict[str, list["BaseUpstreamProvider"]], dict[str, "Model"]
+]:
     """Create optimal model mappings based on cost and provider preferences.
 
     This is the main entry point for the algorithm. It processes all upstream providers
     and creates three mappings based on cost optimization:
 
     1. model_instances: alias -> Model (all model aliases mapped to their Model objects)
-    2. provider_map: alias -> UpstreamProvider (which provider to use for each alias)
+    2. provider_map: alias -> List[UpstreamProvider] (sorted list of providers for each alias)
     3. unique_models: base_id -> Model (unique models without provider prefixes)
 
     The algorithm:
     - Processes non-OpenRouter providers first (they're typically cheaper)
     - Then processes OpenRouter models (they can still win if cheaper)
-    - For each model alias, uses should_prefer_model() to select the best provider
+    - For each model alias, collects all candidates and sorts them by priority and cost.
 
     Args:
         upstreams: List of all upstream provider instances
@@ -183,8 +116,7 @@ def create_model_mappings(
     from .payment.models import _row_to_model
     from .upstream.helpers import resolve_model_alias
 
-    model_instances: dict[str, "Model"] = {}
-    provider_map: dict[str, "BaseUpstreamProvider"] = {}
+    candidates: dict[str, list[tuple["Model", "BaseUpstreamProvider"]]] = {}
     unique_models: dict[str, "Model"] = {}
 
     # Separate OpenRouter from other providers
@@ -202,24 +134,14 @@ def create_model_mappings(
         """Get base model ID by removing provider prefix."""
         return model_id.split("/", 1)[1] if "/" in model_id else model_id
 
-    def _maybe_set_alias(
+    def _add_candidate(
         alias: str, model: "Model", provider: "BaseUpstreamProvider"
     ) -> None:
-        """Set alias to model/provider if not set or if new model is preferred."""
+        """Add candidate model/provider for an alias."""
         alias_lower = alias.lower()
-        existing_model = model_instances.get(alias_lower)
-        if not existing_model:
-            # No existing mapping, set it
-            model_instances[alias_lower] = model
-            provider_map[alias_lower] = provider
-        else:
-            # Check if candidate should replace existing
-            existing_provider = provider_map[alias_lower]
-            if should_prefer_model(
-                model, provider, existing_model, existing_provider, alias
-            ):
-                model_instances[alias_lower] = model
-                provider_map[alias_lower] = provider
+        if alias_lower not in candidates:
+            candidates[alias_lower] = []
+        candidates[alias_lower].append((model, provider))
 
     def process_provider_models(
         upstream: "BaseUpstreamProvider", is_openrouter: bool = False
@@ -266,21 +188,55 @@ def create_model_mappings(
 
             # Try to set each alias
             for alias in aliases:
-                _maybe_set_alias(alias, model_to_use, upstream)
+                _add_candidate(alias, model_to_use, upstream)
 
-    # Process non-OpenRouter providers first (they're typically cheaper)
+    # Process non-OpenRouter providers first
     for upstream in other_upstreams:
         process_provider_models(upstream, is_openrouter=False)
 
-    # Process OpenRouter last - models only win if they're cheaper or better matched
+    # Process OpenRouter last
     if openrouter:
         process_provider_models(openrouter, is_openrouter=True)
 
-    # Log provider distribution
+    # Sort candidates and build final maps
+    model_instances: dict[str, "Model"] = {}
+    provider_map: dict[str, list["BaseUpstreamProvider"]] = {}
+
+    def alias_priority(model: "Model", alias: str) -> int:
+        """Rank how strong the mapping of alias->model is."""
+        model_base = get_base_model_id(model.id)
+        if model_base == alias:
+            return 3
+        if model.canonical_slug:
+            canonical_base = get_base_model_id(model.canonical_slug)
+            if canonical_base == alias:
+                return 2
+        return 1
+
+    for alias, items in candidates.items():
+        # Sort key: (priority DESC, cost ASC)
+        # Using negative cost for DESC sort overall to keep high priority first
+        def sort_key(item: tuple["Model", "BaseUpstreamProvider"]) -> tuple[int, float]:
+            model, provider = item
+            priority = alias_priority(model, alias)
+            cost = calculate_model_cost_score(model)
+            penalty = get_provider_penalty(provider)
+            adjusted_cost = cost * penalty
+            return (priority, -adjusted_cost)
+
+        items.sort(key=sort_key, reverse=True)
+
+        best_model, best_provider = items[0]
+        model_instances[alias] = best_model
+        provider_map[alias] = [p for _, p in items]
+
+    # Log provider distribution (using top provider for stats)
     provider_counts: dict[str, int] = {}
-    for provider in provider_map.values():
-        provider_name = getattr(provider, "upstream_name", "unknown")
-        provider_counts[provider_name] = provider_counts.get(provider_name, 0) + 1
+    for providers in provider_map.values():
+        if providers:
+            provider = providers[0]
+            provider_name = getattr(provider, "upstream_name", "unknown")
+            provider_counts[provider_name] = provider_counts.get(provider_name, 0) + 1
 
     logger.debug(
         f"Updated model mappings with ({len(unique_models)} unique models and {len(model_instances)} aliases)",

routstr/auth.py

@@ -286,30 +286,55 @@ async def validate_bearer_key(
     )
 
 
+async def get_billing_key(key: ApiKey, session: AsyncSession) -> ApiKey:
+    """Returns the key that should be charged for the request."""
+    if key.parent_key_hash:
+        parent = await session.get(ApiKey, key.parent_key_hash)
+        if parent:
+            # We want to keep the total_requests and total_spent on the child key
+            # but use the balance and reserved_balance of the parent.
+            # However, pay_for_request updates reserved_balance and total_requests.
+            # To stay simple, we charge the parent's balance and update parent's total_requests.
+            return parent
+        else:
+            logger.error(
+                "Parent key not found for child key",
+                extra={
+                    "child_key_hash": key.hashed_key[:8] + "...",
+                    "parent_key_hash": key.parent_key_hash[:8] + "...",
+                },
+            )
+    return key
+
+
 async def pay_for_request(
     key: ApiKey, cost_per_request: int, session: AsyncSession
 ) -> int:
     """Process payment for a request."""
 
+    billing_key = await get_billing_key(key, session)
+
     logger.info(
         "Processing payment for request",
         extra={
             "key_hash": key.hashed_key[:8] + "...",
-            "current_balance": key.balance,
+            "billing_key_hash": billing_key.hashed_key[:8] + "...",
+            "current_balance": billing_key.balance,
             "required_cost": cost_per_request,
-            "sufficient_balance": key.balance >= cost_per_request,
+            "sufficient_balance": billing_key.balance >= cost_per_request,
         },
     )
 
-    if key.total_balance < cost_per_request:
+    if billing_key.total_balance < cost_per_request:
         logger.warning(
             "Insufficient balance for request",
             extra={
                 "key_hash": key.hashed_key[:8] + "...",
-                "balance": key.balance,
-                "reserved_balance": key.reserved_balance,
+                "billing_key_hash": billing_key.hashed_key[:8] + "...",
+                "balance": billing_key.balance,
+                "reserved_balance": billing_key.reserved_balance,
                 "required": cost_per_request,
-                "shortfall": cost_per_request - key.total_balance,
+                "shortfall": cost_per_request - billing_key.total_balance,
             },
         )
 
@@ -317,7 +342,7 @@ async def pay_for_request(
             status_code=402,
             detail={
                 "error": {
-                    "message": f"Insufficient balance: {cost_per_request} mSats required. {key.total_balance} available. (reserved: {key.reserved_balance})",
+                    "message": f"Insufficient balance: {cost_per_request} mSats required. {billing_key.total_balance} available. (reserved: {billing_key.reserved_balance})",
                     "type": "insufficient_quota",
                     "code": "insufficient_balance",
                 }
@@ -328,15 +353,16 @@ async def pay_for_request(
         "Charging base cost for request",
         extra={
             "key_hash": key.hashed_key[:8] + "...",
+            "billing_key_hash": billing_key.hashed_key[:8] + "...",
             "cost": cost_per_request,
-            "balance_before": key.balance,
+            "balance_before": billing_key.balance,
         },
     )
 
     # Charge the base cost for the request atomically to avoid race conditions
     stmt = (
         update(ApiKey)
-        .where(col(ApiKey.hashed_key) == key.hashed_key)
+        .where(col(ApiKey.hashed_key) == billing_key.hashed_key)
         .where(col(ApiKey.balance) - col(ApiKey.reserved_balance) >= cost_per_request)
         .values(
             reserved_balance=col(ApiKey.reserved_balance) + cost_per_request,
@@ -344,6 +370,16 @@ async def pay_for_request(
         )
     )
     result = await session.exec(stmt)  # type: ignore[call-overload]
+
+    # Also increment total_requests on the child key if it's different
+    if billing_key.hashed_key != key.hashed_key:
+        child_stmt = (
+            update(ApiKey)
+            .where(col(ApiKey.hashed_key) == key.hashed_key)
+            .values(total_requests=col(ApiKey.total_requests) + 1)
+        )
+        await session.exec(child_stmt)  # type: ignore[call-overload]
+
     await session.commit()
 
     if result.rowcount == 0:
@@ -351,8 +387,9 @@ async def pay_for_request(
             "Concurrent request depleted balance",
             extra={
                 "key_hash": key.hashed_key[:8] + "...",
+                "billing_key_hash": billing_key.hashed_key[:8] + "...",
                 "required_cost": cost_per_request,
-                "current_balance": key.balance,
+                "current_balance": billing_key.balance,
             },
         )
 
@@ -361,23 +398,26 @@ async def pay_for_request(
             status_code=402,
             detail={
                 "error": {
-                    "message": f"Insufficient balance: {cost_per_request} mSats required. {key.balance} available.",
+                    "message": f"Insufficient balance: {cost_per_request} mSats required. {billing_key.balance} available.",
                     "type": "insufficient_quota",
                     "code": "insufficient_balance",
                 }
             },
         )
 
-    await session.refresh(key)
+    await session.refresh(billing_key)
+    if billing_key.hashed_key != key.hashed_key:
+        await session.refresh(key)
 
     logger.info(
         "Payment processed successfully",
         extra={
             "key_hash": key.hashed_key[:8] + "...",
+            "billing_key_hash": billing_key.hashed_key[:8] + "...",
             "charged_amount": cost_per_request,
-            "new_balance": key.balance,
-            "total_spent": key.total_spent,
-            "total_requests": key.total_requests,
+            "new_balance": billing_key.balance,
+            "total_spent": billing_key.total_spent,
+            "total_requests": billing_key.total_requests,
         },
     )
 
@@ -387,9 +427,11 @@ async def pay_for_request(
 async def revert_pay_for_request(
     key: ApiKey, session: AsyncSession, cost_per_request: int
 ) -> None:
+    billing_key = await get_billing_key(key, session)
+
     stmt = (
         update(ApiKey)
-        .where(col(ApiKey.hashed_key) == key.hashed_key)
+        .where(col(ApiKey.hashed_key) == billing_key.hashed_key)
         .values(
             reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
             total_requests=col(ApiKey.total_requests) - 1,
@@ -397,27 +439,40 @@ async def revert_pay_for_request(
     )
 
     result = await session.exec(stmt)  # type: ignore[call-overload]
+
+    # Also decrement total_requests on the child key if it's different
+    if billing_key.hashed_key != key.hashed_key:
+        child_stmt = (
+            update(ApiKey)
+            .where(col(ApiKey.hashed_key) == key.hashed_key)
+            .values(total_requests=col(ApiKey.total_requests) - 1)
+        )
+        await session.exec(child_stmt)  # type: ignore[call-overload]
+
     await session.commit()
     if result.rowcount == 0:
         logger.error(
             "Failed to revert payment - insufficient reserved balance",
             extra={
                 "key_hash": key.hashed_key[:8] + "...",
+                "billing_key_hash": billing_key.hashed_key[:8] + "...",
                 "cost_to_revert": cost_per_request,
-                "current_reserved_balance": key.reserved_balance,
+                "current_reserved_balance": billing_key.reserved_balance,
             },
         )
         raise HTTPException(
             status_code=402,
             detail={
                 "error": {
-                    "message": f"failed to revert request payment: {cost_per_request} mSats required. {key.balance} available.",
+                    "message": f"failed to revert request payment: {cost_per_request} mSats required. {billing_key.balance} available.",
                     "type": "payment_error",
                     "code": "payment_error",
                 }
             },
         )
-    await session.refresh(key)
+    await session.refresh(billing_key)
+    if billing_key.hashed_key != key.hashed_key:
+        await session.refresh(key)
 
 
 async def adjust_payment_for_tokens(
@@ -428,25 +483,58 @@ async def adjust_payment_for_tokens(
     This is called after the initial payment and the upstream request is complete.
     Returns cost data to be included in the response.
     """
+    billing_key = await get_billing_key(key, session)
     model = response_data.get("model", "unknown")
 
     logger.debug(
         "Starting payment adjustment for tokens",
         extra={
             "key_hash": key.hashed_key[:8] + "...",
+            "billing_key_hash": billing_key.hashed_key[:8] + "...",
             "model": model,
             "deducted_max_cost": deducted_max_cost,
-            "current_balance": key.balance,
+            "current_balance": billing_key.balance,
             "has_usage": "usage" in response_data,
         },
     )
 
+    async def release_reservation_only() -> None:
+        """Fallback to release reservation without charging when main update fails."""
+        try:
+            release_stmt = (
+                update(ApiKey)
+                .where(col(ApiKey.hashed_key) == billing_key.hashed_key)
+                .values(
+                    reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost
+                )
+            )
+            await session.exec(release_stmt)  # type: ignore[call-overload]
+            await session.commit()
+            logger.warning(
+                "Released reservation without charging (fallback)",
+                extra={
+                    "key_hash": key.hashed_key[:8] + "...",
+                    "billing_key_hash": billing_key.hashed_key[:8] + "...",
+                    "deducted_max_cost": deducted_max_cost,
+                },
+            )
+        except Exception as e:
+            logger.error(
+                "Failed to release reservation in fallback",
+                extra={
+                    "error": str(e),
+                    "key_hash": key.hashed_key[:8] + "...",
+                    "billing_key_hash": billing_key.hashed_key[:8] + "...",
+                },
+            )
+
     match await calculate_cost(response_data, deducted_max_cost, session):
         case MaxCostData() as cost:
             logger.debug(
                 "Using max cost data (no token adjustment)",
                 extra={
                     "key_hash": key.hashed_key[:8] + "...",
+                    "billing_key_hash": billing_key.hashed_key[:8] + "...",
                     "model": model,
                     "max_cost": cost.total_msats,
                 },
@@ -454,7 +542,7 @@ async def adjust_payment_for_tokens(
             # Finalize by releasing reservation and charging max cost
             finalize_stmt = (
                 update(ApiKey)
-                .where(col(ApiKey.hashed_key) == key.hashed_key)
+                .where(col(ApiKey.hashed_key) == billing_key.hashed_key)
                 .values(
                     reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost,
                     balance=col(ApiKey.balance) - cost.total_msats,
@@ -462,26 +550,41 @@ async def adjust_payment_for_tokens(
                 )
             )
             result = await session.exec(finalize_stmt)  # type: ignore[call-overload]
+
+            # Also update total_spent on the child key if it's different
+            if billing_key.hashed_key != key.hashed_key:
+                child_stmt = (
+                    update(ApiKey)
+                    .where(col(ApiKey.hashed_key) == key.hashed_key)
+                    .values(total_spent=col(ApiKey.total_spent) + cost.total_msats)
+                )
+                await session.exec(child_stmt)  # type: ignore[call-overload]
+
             await session.commit()
             if result.rowcount == 0:
                 logger.error(
-                    "Failed to finalize max-cost payment - insufficient reserved balance",
+                    "Failed to finalize max-cost payment - retrying reservation release",
                     extra={
                         "key_hash": key.hashed_key[:8] + "...",
+                        "billing_key_hash": billing_key.hashed_key[:8] + "...",
                         "deducted_max_cost": deducted_max_cost,
-                        "current_reserved_balance": key.reserved_balance,
+                        "current_reserved_balance": billing_key.reserved_balance,
                         "total_cost": cost.total_msats,
                         "model": model,
                     },
                 )
+                await release_reservation_only()
             else:
-                await session.refresh(key)
+                await session.refresh(billing_key)
+                if billing_key.hashed_key != key.hashed_key:
+                    await session.refresh(key)
                 logger.info(
                     "Max cost payment finalized",
                     extra={
                         "key_hash": key.hashed_key[:8] + "...",
+                        "billing_key_hash": billing_key.hashed_key[:8] + "...",
                         "charged_amount": cost.total_msats,
-                        "new_balance": key.balance,
+                        "new_balance": billing_key.balance,
                         "model": model,
                     },
                 )
@@ -497,6 +600,7 @@ async def adjust_payment_for_tokens(
                 "Calculated token-based cost",
                 extra={
                     "key_hash": key.hashed_key[:8] + "...",
+                    "billing_key_hash": billing_key.hashed_key[:8] + "...",
                     "model": model,
                     "token_cost": cost.total_msats,
                     "deducted_max_cost": deducted_max_cost,
@@ -509,11 +613,15 @@ async def adjust_payment_for_tokens(
             if cost_difference == 0:
                 logger.debug(
                     "Finalizing with exact reserved cost",
-                    extra={"key_hash": key.hashed_key[:8] + "...", "model": model},
+                    extra={
+                        "key_hash": key.hashed_key[:8] + "...",
+                        "billing_key_hash": billing_key.hashed_key[:8] + "...",
+                        "model": model,
+                    },
                 )
                 finalize_stmt = (
                     update(ApiKey)
-                    .where(col(ApiKey.hashed_key) == key.hashed_key)
+                    .where(col(ApiKey.hashed_key) == billing_key.hashed_key)
                     .values(
                         reserved_balance=col(ApiKey.reserved_balance)
                         - deducted_max_cost,
@@ -522,8 +630,20 @@ async def adjust_payment_for_tokens(
                     )
                 )
                 await session.exec(finalize_stmt)  # type: ignore[call-overload]
+
+                # Also update total_spent on the child key if it's different
+                if billing_key.hashed_key != key.hashed_key:
+                    child_stmt = (
+                        update(ApiKey)
+                        .where(col(ApiKey.hashed_key) == key.hashed_key)
+                        .values(total_spent=col(ApiKey.total_spent) + total_cost_msats)
+                    )
+                    await session.exec(child_stmt)  # type: ignore[call-overload]
+
                 await session.commit()
-                await session.refresh(key)
+                await session.refresh(billing_key)
+                if billing_key.hashed_key != key.hashed_key:
+                    await session.refresh(key)
                 return cost.dict()
 
             # this should never happen why do we handle this???
@@ -533,16 +653,17 @@ async def adjust_payment_for_tokens(
                     "Additional charge required for token usage",
                     extra={
                         "key_hash": key.hashed_key[:8] + "...",
+                        "billing_key_hash": billing_key.hashed_key[:8] + "...",
                         "additional_charge": cost_difference,
-                        "current_balance": key.balance,
-                        "sufficient_balance": key.balance >= cost_difference,
+                        "current_balance": billing_key.balance,
+                        "sufficient_balance": billing_key.balance >= cost_difference,
                         "model": model,
                     },
                 )
 
                 finalize_stmt = (
                     update(ApiKey)
-                    .where(col(ApiKey.hashed_key) == key.hashed_key)
+                    .where(col(ApiKey.hashed_key) == billing_key.hashed_key)
                     .values(
                         reserved_balance=col(ApiKey.reserved_balance)
                         - deducted_max_cost,
@@ -551,30 +672,45 @@ async def adjust_payment_for_tokens(
                     )
                 )
                 result = await session.exec(finalize_stmt)  # type: ignore[call-overload]
+
+                # Also update total_spent on the child key if it's different
+                if billing_key.hashed_key != key.hashed_key:
+                    child_stmt = (
+                        update(ApiKey)
+                        .where(col(ApiKey.hashed_key) == key.hashed_key)
+                        .values(total_spent=col(ApiKey.total_spent) + total_cost_msats)
+                    )
+                    await session.exec(child_stmt)  # type: ignore[call-overload]
+
                 await session.commit()
 
                 if result.rowcount:
                     cost.total_msats = total_cost_msats
-                    await session.refresh(key)
+                    await session.refresh(billing_key)
+                    if billing_key.hashed_key != key.hashed_key:
+                        await session.refresh(key)
 
                     logger.info(
                         "Finalized payment with additional charge",
                         extra={
                             "key_hash": key.hashed_key[:8] + "...",
+                            "billing_key_hash": billing_key.hashed_key[:8] + "...",
                             "charged_amount": total_cost_msats,
-                            "new_balance": key.balance,
+                            "new_balance": billing_key.balance,
                             "model": model,
                         },
                     )
                 else:
                     logger.warning(
-                        "Failed to finalize additional charge (concurrent operation)",
+                        "Failed to finalize additional charge - releasing reservation",
                         extra={
                             "key_hash": key.hashed_key[:8] + "...",
+                            "billing_key_hash": billing_key.hashed_key[:8] + "...",
                             "attempted_charge": total_cost_msats,
                             "model": model,
                         },
                     )
+                    await release_reservation_only()
             else:
                 # Refund some of the base cost
                 refund = abs(cost_difference)
@@ -582,15 +718,16 @@ async def adjust_payment_for_tokens(
                     "Refunding excess payment",
                     extra={
                         "key_hash": key.hashed_key[:8] + "...",
+                        "billing_key_hash": billing_key.hashed_key[:8] + "...",
                         "refund_amount": refund,
-                        "current_balance": key.balance,
+                        "current_balance": billing_key.balance,
                         "model": model,
                     },
                 )
 
                 refund_stmt = (
                     update(ApiKey)
-                    .where(col(ApiKey.hashed_key) == key.hashed_key)
+                    .where(col(ApiKey.hashed_key) == billing_key.hashed_key)
                     .values(
                         reserved_balance=col(ApiKey.reserved_balance)
                         - deducted_max_cost,
@@ -599,41 +736,54 @@ async def adjust_payment_for_tokens(
                     )
                 )
                 result = await session.exec(refund_stmt)  # type: ignore[call-overload]
+
+                # Also update total_spent on the child key if it's different
+                if billing_key.hashed_key != key.hashed_key:
+                    child_stmt = (
+                        update(ApiKey)
+                        .where(col(ApiKey.hashed_key) == key.hashed_key)
+                        .values(total_spent=col(ApiKey.total_spent) + total_cost_msats)
+                    )
+                    await session.exec(child_stmt)  # type: ignore[call-overload]
+
                 await session.commit()
 
                 if result.rowcount == 0:
                     logger.error(
-                        "Failed to finalize payment - insufficient reserved balance",
+                        "Failed to finalize payment - releasing reservation",
                         extra={
                             "key_hash": key.hashed_key[:8] + "...",
+                            "billing_key_hash": billing_key.hashed_key[:8] + "...",
                             "deducted_max_cost": deducted_max_cost,
-                            "current_reserved_balance": key.reserved_balance,
+                            "current_reserved_balance": billing_key.reserved_balance,
                             "total_cost": total_cost_msats,
                             "model": model,
                         },
                     )
-                    # Still return the cost data even if we couldn't properly finalize
-                    # The reservation was already made, so the user has paid
+                    await release_reservation_only()
+                else:
+                    cost.total_msats = total_cost_msats
+                    await session.refresh(billing_key)
+                    if billing_key.hashed_key != key.hashed_key:
+                        await session.refresh(key)
 
-                cost.total_msats = total_cost_msats
-                await session.refresh(key)
-
-                logger.info(
-                    "Refund processed successfully",
-                    extra={
-                        "key_hash": key.hashed_key[:8] + "...",
-                        "refunded_amount": refund,
-                        "new_balance": key.balance,
-                        "final_cost": cost.total_msats,
-                        "model": model,
-                    },
-                )
+                    logger.info(
+                        "Refund processed successfully",
+                        extra={
+                            "key_hash": key.hashed_key[:8] + "...",
+                            "billing_key_hash": billing_key.hashed_key[:8] + "...",
+                            "refunded_amount": refund,
+                            "new_balance": billing_key.balance,
+                            "final_cost": cost.total_msats,
+                            "model": model,
+                        },
+                    )
 
             return cost.dict()
 
         case CostDataError() as error:
             logger.error(
-                "Cost calculation error during payment adjustment",
+                "Cost calculation error during payment adjustment - releasing reservation",
                 extra={
                     "key_hash": key.hashed_key[:8] + "...",
                     "model": model,
@@ -641,6 +791,7 @@ async def adjust_payment_for_tokens(
                     "error_code": error.code,
                 },
             )
+            await release_reservation_only()
 
             raise HTTPException(
                 status_code=400,
@@ -652,7 +803,12 @@ async def adjust_payment_for_tokens(
                     }
                 },
             )
-    # Fallback return to satisfy type checker; execution should not reach here
+    # Fallback: should not reach here, but release reservation just in case
+    logger.error(
+        "Unexpected fallback in adjust_payment_for_tokens - releasing reservation",
+        extra={"key_hash": key.hashed_key[:8] + "...", "model": model},
+    )
+    await release_reservation_only()
     return {
         "base_msats": deducted_max_cost,
         "input_msats": 0,

routstr/balance.py

@@ -32,16 +32,30 @@ async def get_key_from_header(
     )
 
 
-# TODO: remove this endpoint when frontend is updated
-@router.get("/", include_in_schema=False)
-async def account_info(key: ApiKey = Depends(get_key_from_header)) -> dict:
+async def get_balance_info(key: ApiKey, session: AsyncSession) -> dict:
+    from .auth import get_billing_key
+
+    billing_key = await get_billing_key(key, session)
     return {
         "api_key": "sk-" + key.hashed_key,
-        "balance": key.balance,
-        "reserved": key.reserved_balance,
+        "balance": billing_key.balance,
+        "reserved": billing_key.reserved_balance,
+        "is_child": key.parent_key_hash is not None,
+        "parent_key": "sk-" + key.parent_key_hash if key.parent_key_hash else None,
+        "total_requests": key.total_requests,
+        "total_spent": key.total_spent,
     }
 
 
+# TODO: remove this endpoint when frontend is updated
+@router.get("/", include_in_schema=False)
+async def account_info(
+    key: ApiKey = Depends(get_key_from_header),
+    session: AsyncSession = Depends(get_session),
+) -> dict:
+    return await get_balance_info(key, session)
+
+
 # TODO: Implement POST /v1/wallet/create endpoint
 # This endpoint should accept:
 # - cashu_token (required): The eCash token to deposit
@@ -66,12 +80,11 @@ async def create_balance(
 
 
 @router.get("/info")
-async def wallet_info(key: ApiKey = Depends(get_key_from_header)) -> dict:
-    return {
-        "api_key": "sk-" + key.hashed_key,
-        "balance": key.balance,
-        "reserved": key.reserved_balance,
-    }
+async def wallet_info(
+    key: ApiKey = Depends(get_key_from_header),
+    session: AsyncSession = Depends(get_session),
+) -> dict:
+    return await get_balance_info(key, session)
 
 
 class TopupRequest(BaseModel):
@@ -85,6 +98,10 @@ async def topup_wallet_endpoint(
     key: ApiKey = Depends(get_key_from_header),
     session: AsyncSession = Depends(get_session),
 ) -> dict[str, int]:
+    from .auth import get_billing_key
+
+    billing_key = await get_billing_key(key, session)
+
     if topup_request is not None:
         cashu_token = topup_request.cashu_token
     if cashu_token is None:
@@ -94,7 +111,7 @@ async def topup_wallet_endpoint(
     if len(cashu_token) < 10 or "cashu" not in cashu_token:
         raise HTTPException(status_code=400, detail="Invalid token format")
     try:
-        amount_msats = await credit_balance(cashu_token, key, session)
+        amount_msats = await credit_balance(cashu_token, billing_key, session)
     except ValueError as e:
         error_msg = str(e)
         if "already spent" in error_msg.lower():
@@ -154,7 +171,14 @@ async def refund_wallet_endpoint(
         return cached
 
     key: ApiKey = await validate_bearer_key(bearer_value, session)
-    remaining_balance_msats: int = key.balance
+
+    if key.parent_key_hash:
+        raise HTTPException(
+            status_code=400,
+            detail="Cannot refund child key. Please refund the parent key instead.",
+        )
+
+    remaining_balance_msats: int = key.total_balance
 
     if key.refund_currency == "sat":
         remaining_balance = remaining_balance_msats // 1000
@@ -227,6 +251,53 @@ async def donate(token: str, ref: str | None = None) -> str:
         return "Invalid token."
 
 
+@router.post("/child-key")
+async def create_child_key(
+    key: ApiKey = Depends(get_key_from_header),
+    session: AsyncSession = Depends(get_session),
+) -> dict:
+    """Creates a child API key that uses the parent's balance."""
+    # Check if this is already a child key
+    if key.parent_key_hash:
+        raise HTTPException(
+            status_code=400,
+            detail="Cannot create a child key for another child key.",
+        )
+
+    cost = settings.child_key_cost
+
+    if key.total_balance < cost:
+        raise HTTPException(
+            status_code=402,
+            detail=f"Insufficient balance to create child key. {cost} mSats required.",
+        )
+
+    # Deduct cost from parent
+    key.balance -= cost
+    key.total_spent += cost
+    session.add(key)
+
+    # Generate new key
+    import secrets
+
+    new_key_raw = secrets.token_hex(32)
+    new_key_hash = new_key_raw  # We use the raw key as the hash for sk- keys
+
+    child_key = ApiKey(
+        hashed_key=new_key_hash,
+        balance=0,
+        parent_key_hash=key.hashed_key,
+    )
+    session.add(child_key)
+    await session.commit()
+
+    return {
+        "api_key": "sk-" + new_key_hash,
+        "cost_msats": cost,
+        "parent_balance": key.balance,
+    }
+
+
 @router.api_route(
     "/{path:path}",
     methods=["GET", "POST", "PUT", "DELETE"],

routstr/core/admin.py

@@ -124,7 +124,7 @@ async def partial_apikeys(request: Request) -> str:
 
     rows = "".join(
         [
-            f"<tr><td>{key.hashed_key}</td><td>{key.balance}</td><td>{key.total_spent}</td><td>{key.total_requests}</td><td>{key.refund_address}</td><td>{fmt_time(key.key_expiry_time)}</td></tr>"
+            f"<tr><td>{key.hashed_key}{' <br><small>(Child of ' + key.parent_key_hash[:8] + '...)</small>' if key.parent_key_hash else ''}</td><td>{key.balance}</td><td>{key.total_spent}</td><td>{key.total_requests}</td><td>{key.refund_address}</td><td>{fmt_time(key.key_expiry_time)}</td></tr>"
             for key in api_keys
         ]
     )
@@ -158,6 +158,7 @@ async def get_temporary_balances_api(request: Request) -> list[dict[str, object]
             "total_requests": key.total_requests,
             "refund_address": key.refund_address,
             "key_expiry_time": key.key_expiry_time,
+            "parent_key_hash": key.parent_key_hash,
         }
         for key in api_keys
     ]
@@ -3080,6 +3081,9 @@ async def get_logs_api(
     level: str | None = None,
     request_id: str | None = None,
     search: str | None = None,
+    status_codes: str | None = Query(None, description="Comma-separated status codes"),
+    methods: str | None = Query(None, description="Comma-separated HTTP methods"),
+    endpoints: str | None = Query(None, description="Comma-separated endpoints"),
     limit: int = 100,
 ) -> dict[str, object]:
     """
@@ -3090,16 +3094,32 @@ async def get_logs_api(
         level: Filter by log level
         request_id: Filter by request ID
         search: Search text in message and name fields (case-insensitive)
+        status_codes: Comma-separated list of HTTP status codes
+        methods: Comma-separated list of HTTP methods
+        endpoints: Comma-separated list of endpoints
         limit: Maximum number of entries to return
 
     Returns:
         Dict containing logs and filter metadata
     """
+    status_code_list = None
+    if status_codes:
+        try:
+            status_code_list = [int(s.strip()) for s in status_codes.split(",")]
+        except ValueError:
+            pass
+
+    method_list = [m.strip() for m in methods.split(",")] if methods else None
+    endpoint_list = [e.strip() for e in endpoints.split(",")] if endpoints else None
+
     log_entries = log_manager.search_logs(
         date=date,
         level=level,
         request_id=request_id,
         search_text=search,
+        status_codes=status_code_list,
+        methods=method_list,
+        endpoints=endpoint_list,
         limit=limit,
     )
 
@@ -3110,6 +3130,9 @@ async def get_logs_api(
         "level": level,
         "request_id": request_id,
         "search": search,
+        "status_codes": status_codes,
+        "methods": methods,
+        "endpoints": endpoints,
         "limit": limit,
     }
 

routstr/core/db.py

@@ -6,7 +6,7 @@ from typing import AsyncGenerator
 from alembic import command
 from alembic.config import Config
 from sqlalchemy.ext.asyncio.engine import create_async_engine
-from sqlmodel import Field, Relationship, SQLModel, func, select
+from sqlmodel import Field, Relationship, SQLModel, func, select, update
 from sqlmodel.ext.asyncio.session import AsyncSession
 
 from .logging import get_logger
@@ -47,12 +47,23 @@ class ApiKey(SQLModel, table=True):  # type: ignore
         default=None,
         description="Currency of the cashu-token",
     )
+    parent_key_hash: str | None = Field(
+        default=None, foreign_key="api_keys.hashed_key", index=True
+    )
 
     @property
     def total_balance(self) -> int:
         return self.balance - self.reserved_balance
 
 
+async def reset_all_reserved_balances(session: AsyncSession) -> None:
+    logger.info("Resetting all reserved balances to 0")
+    stmt = update(ApiKey).values(reserved_balance=0)
+    await session.exec(stmt)  # type: ignore[call-overload]
+    await session.commit()
+    logger.info("Reserved balances reset successfully")
+
+
 class ModelRow(SQLModel, table=True):  # type: ignore
     __tablename__ = "models"
     id: str = Field(primary_key=True)

routstr/core/exceptions.py

@@ -6,6 +6,15 @@ from .logging import get_logger
 logger = get_logger(__name__)
 
 
+class UpstreamError(Exception):
+    """Exception raised when an upstream provider fails."""
+
+    def __init__(self, message: str, status_code: int = 502):
+        self.message = message
+        self.status_code = status_code
+        super().__init__(message)
+
+
 async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
     """Handle HTTP exceptions and include request ID in response."""
     request_id = getattr(request.state, "request_id", "unknown")

routstr/core/log_manager.py

@@ -105,6 +105,9 @@ class LogManager:
         level: str | None = None,
         request_id: str | None = None,
         search_text: str | None = None,
+        status_codes: list[int] | None = None,
+        methods: list[str] | None = None,
+        endpoints: list[str] | None = None,
         limit: int = 100,
     ) -> list[dict[str, Any]]:
         """
@@ -134,7 +137,13 @@ class LogManager:
 
         for log_data in iterator:
             if not self._matches_filters(
-                log_data, level, request_id, search_text_lower
+                log_data,
+                level,
+                request_id,
+                search_text_lower,
+                status_codes,
+                methods,
+                endpoints,
             ):
                 continue
 
@@ -153,6 +162,9 @@ class LogManager:
         level: str | None,
         request_id: str | None,
         search_text_lower: str | None,
+        status_codes: list[int] | None = None,
+        methods: list[str] | None = None,
+        endpoints: list[str] | None = None,
     ) -> bool:
         if level and log_data.get("levelname", "").upper() != level.upper():
             return False
@@ -160,6 +172,36 @@ class LogManager:
         if request_id and log_data.get("request_id") != request_id:
             return False
 
+        if status_codes:
+            entry_status = log_data.get("status_code")
+            if entry_status is not None:
+                try:
+                    if int(entry_status) not in status_codes:
+                        return False
+                except (ValueError, TypeError):
+                    return False
+            else:
+                return False
+
+        if methods:
+            entry_method = log_data.get("method", "").upper()
+            if entry_method not in [m.upper() for m in methods]:
+                return False
+
+        if endpoints:
+            entry_path = log_data.get("path", "")
+            matched = False
+            for endpoint in endpoints:
+                clean_endpoint = endpoint.lstrip("/")
+                if entry_path.startswith(clean_endpoint):
+                    matched = True
+                    break
+                if clean_endpoint in entry_path:
+                    matched = True
+                    break
+            if not matched:
+                return False
+
         if search_text_lower:
             message = str(log_data.get("message", "")).lower()
             name = str(log_data.get("name", "")).lower()

routstr/core/main.py

@@ -13,10 +13,7 @@ from starlette.exceptions import HTTPException
 from ..balance import balance_router, deprecated_wallet_router
 from ..discovery import providers_cache_refresher, providers_router
 from ..nip91 import announce_provider
-from ..payment.models import (
-    models_router,
-    update_sats_pricing,
-)
+from ..payment.models import models_router, update_sats_pricing
 from ..payment.price import update_prices_periodically
 from ..proxy import initialize_upstreams, proxy_router, refresh_model_maps_periodically
 from ..wallet import periodic_payout
@@ -33,9 +30,9 @@ setup_logging()
 logger = get_logger(__name__)
 
 if os.getenv("VERSION_SUFFIX") is not None:
-    __version__ = f"0.2.1-{os.getenv('VERSION_SUFFIX')}"
+    __version__ = f"0.3.0-{os.getenv('VERSION_SUFFIX')}"
 else:
-    __version__ = "0.2.1"
+    __version__ = "0.3.0"
 
 
 @asynccontextmanager
@@ -61,6 +58,10 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
         # Initialize application settings (env -> computed -> DB precedence)
         async with create_session() as session:
             s = await SettingsService.initialize(session)
+            if s.reset_reserved_balance_on_startup:
+                from .db import reset_all_reserved_balances
+
+                await reset_all_reserved_balances(session)
 
         if not s.admin_password:
             logger.warning(

routstr/core/settings.py

@@ -52,14 +52,18 @@ class Settings(BaseSettings):
     exchange_fee: float = Field(default=1.005, env="EXCHANGE_FEE")
     upstream_provider_fee: float = Field(default=1.05, env="UPSTREAM_PROVIDER_FEE")
     tolerance_percentage: float = Field(default=1.0, env="TOLERANCE_PERCENTAGE")
+    child_key_cost: int = Field(default=1000, env="CHILD_KEY_COST")
     # Minimum per-request charge in millisatoshis when model pricing is free/zero
     min_request_msat: int = Field(default=1, env="MIN_REQUEST_MSAT")
+    reset_reserved_balance_on_startup: bool = Field(
+        default=True, env="RESET_RESERVED_BALANCE_ON_STARTUP"
+    )  # deactivate in horizontal scaling setups
 
     # Network
     cors_origins: list[str] = Field(default_factory=lambda: ["*"], env="CORS_ORIGINS")
     tor_proxy_url: str = Field(default="socks5://127.0.0.1:9050", env="TOR_PROXY_URL")
     providers_refresh_interval_seconds: int = Field(
-        default=300, env="PROVIDERS_REFRESH_INTERVAL_SECONDS"
+        default=0, env="PROVIDERS_REFRESH_INTERVAL_SECONDS"
     )
     pricing_refresh_interval_seconds: int = Field(
         default=120, env="PRICING_REFRESH_INTERVAL_SECONDS"

routstr/discovery.py

@@ -6,7 +6,7 @@ from typing import Any
 
 import httpx
 import websockets
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
 
 from .core.logging import get_logger
 from .core.settings import settings
@@ -389,6 +389,9 @@ async def get_providers(
     Return cached providers. If include_json, return provider+health; otherwise provider only.
     Optional filter by pubkey.
     """
+    if settings.providers_refresh_interval_seconds == 0:
+        raise HTTPException(status_code=404, detail="Provider discovery is disabled")
+
     cache = await get_cache()
     if not cache:
         await refresh_providers_cache(pubkey=pubkey)

routstr/payment/cost_calculation.py

@@ -15,6 +15,7 @@ class CostData(BaseModel):
     input_msats: int
     output_msats: int
     total_msats: int
+    total_usd: float = 0.0
 
 
 class MaxCostData(CostData):
@@ -48,13 +49,6 @@ async def calculate_cost(  # todo: can be sync
         },
     )
 
-    cost_data = MaxCostData(
-        base_msats=max_cost,
-        input_msats=0,
-        output_msats=0,
-        total_msats=max_cost,
-    )
-
     if "usage" not in response_data or response_data["usage"] is None:
         logger.warning(
             "No usage data in response, using base cost only",
@@ -63,7 +57,13 @@ async def calculate_cost(  # todo: can be sync
                 "model": response_data.get("model", "unknown"),
             },
         )
-        return cost_data
+        return MaxCostData(
+            base_msats=0,
+            input_msats=0,
+            output_msats=0,
+            total_msats=0,
+            total_usd=0.0,
+        )
 
     usage_data = response_data["usage"]
 
@@ -103,6 +103,7 @@ async def calculate_cost(  # todo: can be sync
                 input_msats=-1,  # Cost field doesn't break down by token type
                 output_msats=-1,
                 total_msats=cost_in_msats,
+                total_usd=usd_cost,
             )
         except Exception as e:
             logger.warning(
@@ -178,7 +179,12 @@ async def calculate_cost(  # todo: can be sync
                 "model": response_data.get("model", "unknown"),
             },
         )
-        return cost_data
+        return MaxCostData(
+            base_msats=max_cost,
+            input_msats=0,
+            output_msats=0,
+            total_msats=max_cost,
+        )
 
     input_tokens = usage_data.get("prompt_tokens", 0)
     output_tokens = usage_data.get("completion_tokens", 0)
@@ -192,13 +198,22 @@ async def calculate_cost(  # todo: can be sync
     )
 
     # added for response api
-    input_tokens = input_tokens if input_tokens != 0 else  response_data.get("usage", {}).get("input_tokens", 0)
-    output_tokens = output_tokens if output_tokens != 0 else  response_data.get("usage", {}).get("output_tokens", 0)
+    input_tokens = (
+        input_tokens
+        if input_tokens != 0
+        else response_data.get("usage", {}).get("input_tokens", 0)
+    )
+    output_tokens = (
+        output_tokens
+        if output_tokens != 0
+        else response_data.get("usage", {}).get("output_tokens", 0)
+    )
 
     input_msats = round(input_tokens / 1000 * MSATS_PER_1K_INPUT_TOKENS, 3)
 
     output_msats = round(output_tokens / 1000 * MSATS_PER_1K_OUTPUT_TOKENS, 3)
     token_based_cost = math.ceil(input_msats + output_msats)
+    total_usd = (token_based_cost / 1000.0) * sats_usd_price()
 
     logger.info(
         "Calculated token-based cost",
@@ -208,6 +223,7 @@ async def calculate_cost(  # todo: can be sync
             "input_cost_msats": input_msats,
             "output_cost_msats": output_msats,
             "total_cost_msats": token_based_cost,
+            "total_usd": total_usd,
             "model": response_data.get("model", "unknown"),
         },
     )
@@ -217,4 +233,5 @@ async def calculate_cost(  # todo: can be sync
         input_msats=int(input_msats),
         output_msats=int(output_msats),
         total_msats=token_based_cost,
+        total_usd=total_usd,
     )

routstr/payment/lnurl.py

@@ -283,7 +283,7 @@ async def raw_send_to_lnurl(
             f"({min_sendable_sat} - {max_sendable_sat} {unit})"
         )
 
-    estimated_fees_sat = int(max(math.ceil((amount_msat / 1000) * 0.01), 2))
+    estimated_fees_sat = int(max(math.ceil((amount_msat / 1000) * 0.01), 2)) + 1
     estimated_fees_msat = estimated_fees_sat * 1000
     final_amount = amount_msat - estimated_fees_msat
 

routstr/payment/models.py

@@ -253,6 +253,11 @@ async def list_models(
             else 1.01,
         )
         for r in rows
+        if include_disabled
+        or (
+            r.upstream_provider_id in providers_by_id
+            and providers_by_id[r.upstream_provider_id].enabled
+        )
     ]
 
 
@@ -265,6 +270,8 @@ async def get_model_by_id(
     if not row or not row.enabled:
         return None
     provider = await session.get(UpstreamProviderRow, provider_id)
+    if not provider or not provider.enabled:
+        return None
     provider_fee = provider.provider_fee if provider else 1.01
     return _row_to_model(row, apply_provider_fee=True, provider_fee=provider_fee)
 

routstr/payment/price.py

@@ -79,15 +79,29 @@ async def _fetch_btc_usd_price() -> float:
     """Fetch the lowest BTC/USD price from multiple exchanges."""
     async with httpx.AsyncClient(timeout=30.0) as client:
         try:
-            prices = await asyncio.gather(
-                _kraken_btc_usd(client),
-                _coinbase_btc_usd(client),
-                _binance_btc_usdt(client),
-            )
-            valid_prices = [price for price in prices if price is not None]
+            tasks = [
+                asyncio.create_task(_kraken_btc_usd(client)),
+                asyncio.create_task(_coinbase_btc_usd(client)),
+                asyncio.create_task(_binance_btc_usdt(client)),
+            ]
+            valid_prices: list[float] = []
+
+            for future in asyncio.as_completed(tasks):
+                price = await future
+                if price is not None:
+                    valid_prices.append(price)
+
+                if len(valid_prices) >= 2:
+                    break
+
+            for task in tasks:
+                if not task.done():
+                    task.cancel()
+
             if not valid_prices:
                 logger.error("No valid BTC prices obtained from any exchange")
                 raise ValueError("Unable to fetch BTC price from any exchange")
+
             return min(valid_prices)
         except Exception as e:
             logger.error(

routstr/proxy.py

@@ -16,6 +16,7 @@ from .core.db import (
     create_session,
     get_session,
 )
+from .core.exceptions import UpstreamError
 from .payment.helpers import (
     calculate_discounted_max_cost,
     check_token_balance,
@@ -31,7 +32,9 @@ proxy_router = APIRouter()
 
 _upstreams: list[BaseUpstreamProvider] = []
 _model_instances: dict[str, Model] = {}  # All aliases -> Model
-_provider_map: dict[str, BaseUpstreamProvider] = {}  # All aliases -> Provider
+_provider_map: dict[
+    str, list[BaseUpstreamProvider]
+] = {}  # All aliases -> List[Provider]
 _unique_models: dict[str, Model] = {}  # Unique model.id -> Model (no duplicates)
 
 
@@ -68,8 +71,8 @@ def get_model_instance(model_id: str) -> Model | None:
     return _model_instances.get(model_id.lower())
 
 
-def get_provider_for_model(model_id: str) -> BaseUpstreamProvider | None:
-    """Get UpstreamProvider for model ID from global cache."""
+def get_provider_for_model(model_id: str) -> list[BaseUpstreamProvider] | None:
+    """Get UpstreamProvider list for model ID from global cache."""
     return _provider_map.get(model_id.lower())
 
 
@@ -96,6 +99,8 @@ async def refresh_model_maps() -> None:
     disabled_model_ids: set[str] = set()
 
     for provider in provider_rows:
+        if not provider.enabled:
+            continue
         for model in provider.models:
             if model.enabled:
                 overrides_by_id[model.id] = (model, provider.provider_fee)
@@ -152,8 +157,8 @@ async def proxy(
             "invalid_model", f"Model '{model_id}' not found", 400, request=request
         )
 
-    upstream = get_provider_for_model(model_id)
-    if not upstream:
+    upstreams = get_provider_for_model(model_id)
+    if not upstreams:
         return create_error_response(
             "invalid_model",
             f"No provider found for model '{model_id}'",
@@ -161,6 +166,10 @@ async def proxy(
             request=request,
         )
 
+    # todo figure out cost calculation since fallback provider is usually not the same price
+    # Use first provider for initial checks/cost calculation
+    # primary_upstream = upstreams[0]
+
     _max_cost_for_model = await get_max_cost_for_model(
         model=model_id, session=session, model_obj=model_obj
     )
@@ -170,14 +179,31 @@ async def proxy(
     check_token_balance(headers, request_body_dict, max_cost_for_model)
 
     if x_cashu := headers.get("x-cashu", None):
-        if is_responses_api:
-            return await upstream.handle_x_cashu_responses(
-                request, x_cashu, path, max_cost_for_model, model_obj
-            )
-        else:
-            return await upstream.handle_x_cashu(
-                request, x_cashu, path, max_cost_for_model, model_obj
-            )
+        last_error = None
+        for i, upstream in enumerate(upstreams):
+            try:
+                if is_responses_api:
+                    return await upstream.handle_x_cashu_responses(
+                        request, x_cashu, path, max_cost_for_model, model_obj
+                    )
+                else:
+                    return await upstream.handle_x_cashu(
+                        request, x_cashu, path, max_cost_for_model, model_obj
+                    )
+            except UpstreamError as e:
+                logger.warning(
+                    f"Upstream {upstream.provider_type} failed (x-cashu): {e}"
+                )
+                if i == len(upstreams) - 1:
+                    last_error = e
+                continue
+
+        return create_error_response(
+            "upstream_error",
+            str(last_error) if last_error else "All upstreams failed",
+            502,
+            request=request,
+        )
 
     elif auth := headers.get("authorization", None):
         key = await get_bearer_token_key(headers, path, session, auth)
@@ -192,56 +218,152 @@ async def proxy(
             )
 
         logger.debug("Processing unauthenticated GET request", extra={"path": path})
-        headers = upstream.prepare_headers(dict(request.headers))
-        return await upstream.forward_get_request(request, path, headers)
+
+        last_error_response = None
+        for i, upstream in enumerate(upstreams):
+            try:
+                headers = upstream.prepare_headers(dict(request.headers))
+                response = await upstream.forward_get_request(request, path, headers)
+
+                if response.status_code in [502, 429] and i < len(upstreams) - 1:
+                    error_message = ""
+                    try:
+                        if hasattr(response, "body"):
+                            body_bytes = response.body
+                            data = json.loads(body_bytes)
+                            if "error" in data:
+                                error_data = data["error"]
+                                if isinstance(error_data, dict):
+                                    error_message = error_data.get("message", "")
+                                elif isinstance(error_data, str):
+                                    error_message = error_data
+                    except Exception:
+                        pass
+
+                    await upstream.on_upstream_error_redirect(
+                        response.status_code, error_message
+                    )
+
+                    logger.warning(
+                        f"Upstream {upstream.provider_type} returned {response.status_code} (GET), trying next provider",
+                        extra={
+                            "status_code": response.status_code,
+                            "upstream": upstream.provider_type,
+                        },
+                    )
+                    continue
+                return response
+            except UpstreamError as e:
+                logger.warning(f"Upstream {upstream.provider_type} failed (GET): {e}")
+                if i == len(upstreams) - 1:
+                    last_error_response = create_error_response(
+                        "upstream_error", str(e), 502, request=request
+                    )
+                continue
+        return last_error_response or create_error_response(
+            "upstream_error", "All upstreams failed", 502, request=request
+        )
 
     if request_body_dict:
         await pay_for_request(key, max_cost_for_model, session)
 
-    headers = upstream.prepare_headers(dict(request.headers))
+    for i, upstream in enumerate(upstreams):
+        headers = upstream.prepare_headers(dict(request.headers))
 
-    if is_responses_api:
-        response = await upstream.forward_responses_request(
-            request,
-            path,
-            headers,
-            request_body,
-            key,
-            max_cost_for_model,
-            session,
-            model_obj,
-        )
-    else:
-        response = await upstream.forward_request(
-            request,
-            path,
-            headers,
-            request_body,
-            key,
-            max_cost_for_model,
-            session,
-            model_obj,
-        )
+        try:
+            if is_responses_api:
+                response = await upstream.forward_responses_request(
+                    request,
+                    path,
+                    headers,
+                    request_body,
+                    key,
+                    max_cost_for_model,
+                    session,
+                    model_obj,
+                )
+            else:
+                response = await upstream.forward_request(
+                    request,
+                    path,
+                    headers,
+                    request_body,
+                    key,
+                    max_cost_for_model,
+                    session,
+                    model_obj,
+                )
 
-    if response.status_code != 200:
-        await revert_pay_for_request(key, session, max_cost_for_model)
-        logger.warning(
-            "Upstream request failed, revert payment",
-            extra={
-                "status_code": response.status_code,
-                "path": path,
-                "key_hash": key.hashed_key[:8] + "...",
-                "key_balance": key.balance,
-                "max_cost_for_model": max_cost_for_model,
-                "upstream_headers": response.headers
-                if hasattr(response, "headers")
-                else None,
-            },
-        )
-        # Return the mapped error response generated earlier rather than masking with 502
-        return response
+            if response.status_code != 200:
+                # Check if we should retry (502 Upstream Error or 429 Rate Limit)
+                should_retry = response.status_code in [502, 429, 400, 401, 403, 404]
+                if should_retry and i < len(upstreams) - 1:
+                    error_message = ""
+                    try:
+                        if hasattr(response, "body"):
+                            body_bytes = response.body
+                            data = json.loads(body_bytes)
+                            if "error" in data:
+                                error_data = data["error"]
+                                if isinstance(error_data, dict):
+                                    error_message = error_data.get("message", "")
+                                elif isinstance(error_data, str):
+                                    error_message = error_data
+                    except Exception:
+                        pass
 
-    return response
+                    await upstream.on_upstream_error_redirect(
+                        response.status_code, error_message
+                    )
+
+                    logger.warning(
+                        f"Upstream {upstream.provider_type} returned {response.status_code}, trying next provider",
+                        extra={
+                            "status_code": response.status_code,
+                            "upstream": upstream.provider_type,
+                        },
+                    )
+                    continue
+
+                # 4xx error (user error), or other non-retryable error, or last provider failed
+                await revert_pay_for_request(key, session, max_cost_for_model)
+                logger.warning(
+                    "Upstream request failed, revert payment",
+                    extra={
+                        "status_code": response.status_code,
+                        "path": path,
+                        "key_hash": key.hashed_key[:8] + "...",
+                        "key_balance": key.balance,
+                        "max_cost_for_model": max_cost_for_model,
+                        "upstream_headers": response.headers
+                        if hasattr(response, "headers")
+                        else None,
+                    },
+                )
+                return response
+
+            return response
+
+        except UpstreamError as e:
+            logger.warning(
+                f"Upstream {upstream.provider_type} failed: {e}",
+                extra={"retry": i < len(upstreams) - 1},
+            )
+
+            # If this was the last provider
+            if i == len(upstreams) - 1:
+                await revert_pay_for_request(key, session, max_cost_for_model)
+                return create_error_response(
+                    "upstream_error", str(e), 502, request=request
+                )
+
+            # Otherwise loop continues to next provider
+            continue
+
+    # Should not be reached given logic above
+    return create_error_response(
+        "upstream_error", "All upstreams failed", 502, request=request
+    )
 
 
 async def get_bearer_token_key(
@@ -337,7 +459,7 @@ def extract_model_from_responses_request(request_body_dict: dict[str, Any]) -> s
 
     logger.warning(
         "No model found in Responses API request",
-        extra={"body_keys": list(request_body_dict.keys())}
+        extra={"body_keys": list(request_body_dict.keys())},
     )
     return "unknown"
 

routstr/upstream/base.py

@@ -15,6 +15,7 @@ from pydantic import BaseModel
 from ..auth import adjust_payment_for_tokens
 from ..core import get_logger
 from ..core.db import ApiKey, AsyncSession, create_session
+from ..core.exceptions import UpstreamError
 
 if TYPE_CHECKING:
     from ..core.db import UpstreamProviderRow
@@ -234,7 +235,11 @@ class BaseUpstreamProvider:
                     )
 
                 # Handle model in input field (alternative format)
-                if "input" in data and isinstance(data["input"], dict) and "model" in data["input"]:
+                if (
+                    "input" in data
+                    and isinstance(data["input"], dict)
+                    and "model" in data["input"]
+                ):
                     original_model = model_obj.id
                     transformed_model = self.transform_model_name(original_model)
                     data["input"]["model"] = transformed_model
@@ -336,6 +341,20 @@ class BaseUpstreamProvider:
                 message = preview[:500]
         return message, upstream_code
 
+    async def on_upstream_error_redirect(
+        self, status_code: int, error_message: str
+    ) -> None:
+        """Hook called when the proxy redirects to another provider due to an error.
+
+        Subclasses can implement this to perform actions like disabling the provider
+        if it's out of balance.
+
+        Args:
+            status_code: The HTTP status code returned by the upstream
+            error_message: The error message extracted from the upstream response
+        """
+        pass
+
     async def map_upstream_error_response(
         self, request: Request, path: str, upstream_response: httpx.Response
     ) -> Response:
@@ -432,156 +451,111 @@ class BaseUpstreamProvider:
         async def stream_with_cost(
             max_cost_for_model: int,
         ) -> AsyncGenerator[bytes, None]:
-            stored_chunks: list[bytes] = []
             usage_finalized: bool = False
             last_model_seen: str | None = None
+            usage_chunk_data: dict | None = None
+            done_seen: bool = False
 
-            async def finalize_without_usage() -> bytes | None:
+            async def finalize_db_only() -> None:
                 nonlocal usage_finalized
                 if usage_finalized:
-                    return None
+                    return
                 async with create_session() as new_session:
                     fresh_key = await new_session.get(key.__class__, key.hashed_key)
                     if not fresh_key:
-                        return None
+                        return
                     try:
-                        fallback: dict = {
-                            "model": last_model_seen or "unknown",
-                            "usage": None,
-                        }
-                        cost_data = await adjust_payment_for_tokens(
-                            fresh_key, fallback, new_session, max_cost_for_model
+                        await adjust_payment_for_tokens(
+                            fresh_key,
+                            {"model": last_model_seen or "unknown", "usage": None},
+                            new_session,
+                            max_cost_for_model,
                         )
                         usage_finalized = True
-                        logger.info(
-                            "Finalized streaming payment without explicit usage",
-                            extra={
-                                "key_hash": key.hashed_key[:8] + "...",
-                                "cost_data": cost_data,
-                                "balance_after_adjustment": fresh_key.balance,
-                            },
-                        )
-                        return f"data: {json.dumps({'cost': cost_data})}\n\n".encode()
-                    except Exception as cost_error:
-                        logger.error(
-                            "Error finalizing payment without usage",
-                            extra={
-                                "error": str(cost_error),
-                                "error_type": type(cost_error).__name__,
-                                "key_hash": key.hashed_key[:8] + "...",
-                            },
-                        )
-                        return None
-
-            try:
-                async for chunk in response.aiter_bytes():
-                    stored_chunks.append(chunk)
-                    try:
-                        for part in re.split(b"data: ", chunk):
-                            if not part or part.strip() in (b"[DONE]", b""):
-                                continue
-                            try:
-                                obj = json.loads(part)
-                                if isinstance(obj, dict) and obj.get("model"):
-                                    last_model_seen = str(obj.get("model"))
-                            except json.JSONDecodeError:
-                                pass
                     except Exception:
                         pass
 
-                    yield chunk
+            try:
+                async for chunk in response.aiter_bytes():
+                    # Split chunk into SSE events
+                    parts = re.split(b"data: ", chunk)
+                    for i, part in enumerate(parts):
+                        if not part:
+                            continue
 
-                logger.debug(
-                    "Streaming completed, analyzing usage data",
-                    extra={
-                        "key_hash": key.hashed_key[:8] + "...",
-                        "chunks_count": len(stored_chunks),
-                    },
-                )
+                        stripped_part = part.strip()
+                        if not stripped_part:
+                            continue
 
-                for i in range(len(stored_chunks) - 1, -1, -1):
-                    chunk = stored_chunks[i]
-                    if not chunk:
-                        continue
-                    try:
-                        events = re.split(b"data: ", chunk)
-                        for event_data in events:
-                            if not event_data or event_data.strip() in (b"[DONE]", b""):
-                                continue
-                            try:
-                                data = json.loads(event_data)
-                                if isinstance(data, dict) and data.get("model"):
-                                    last_model_seen = str(data.get("model"))
-                                if isinstance(data, dict) and isinstance(
-                                    data.get("usage"), dict
-                                ):
-                                    async with create_session() as new_session:
-                                        fresh_key = await new_session.get(
-                                            key.__class__, key.hashed_key
-                                        )
-                                        if fresh_key:
-                                            try:
-                                                cost_data = (
-                                                    await adjust_payment_for_tokens(
-                                                        fresh_key,
-                                                        data,
-                                                        new_session,
-                                                        max_cost_for_model,
-                                                    )
-                                                )
-                                                usage_finalized = True
-                                                logger.info(
-                                                    "Payment adjustment completed for streaming",
-                                                    extra={
-                                                        "key_hash": key.hashed_key[:8]
-                                                        + "...",
-                                                        "cost_data": cost_data,
-                                                        "model": last_model_seen,
-                                                        "balance_after_adjustment": fresh_key.balance,
-                                                    },
-                                                )
-                                                yield f"data: {json.dumps({'cost': cost_data})}\n\n".encode()
-                                            except Exception as cost_error:
-                                                logger.error(
-                                                    "Error adjusting payment for streaming tokens",
-                                                    extra={
-                                                        "error": str(cost_error),
-                                                        "error_type": type(
-                                                            cost_error
-                                                        ).__name__,
-                                                        "key_hash": key.hashed_key[:8]
-                                                        + "...",
-                                                    },
-                                                )
-                                    break
-                            except json.JSONDecodeError:
-                                continue
-                    except Exception as e:
-                        logger.error(
-                            "Error processing streaming response chunk",
-                            extra={
-                                "error": str(e),
-                                "error_type": type(e).__name__,
-                                "key_hash": key.hashed_key[:8] + "...",
-                            },
+                        if stripped_part == b"[DONE]":
+                            done_seen = True
+                            continue
+
+                        try:
+                            obj = json.loads(part)
+                            if isinstance(obj, dict):
+                                if obj.get("model"):
+                                    last_model_seen = str(obj.get("model"))
+
+                                if isinstance(obj.get("usage"), dict):
+                                    # Hold this chunk back to merge cost later
+                                    usage_chunk_data = obj
+                                    continue
+                        except json.JSONDecodeError:
+                            pass
+
+                        prefix = (
+                            b"data: " if (i > 0 or chunk.startswith(b"data: ")) else b""
                         )
+                        yield prefix + part
+
+                # Stream finished, process usage if found
+                if usage_chunk_data:
+                    async with create_session() as session:
+                        fresh_key = await session.get(key.__class__, key.hashed_key)
+                        if fresh_key:
+                            try:
+                                cost_data = await adjust_payment_for_tokens(
+                                    fresh_key,
+                                    usage_chunk_data,
+                                    session,
+                                    max_cost_for_model,
+                                )
+                                # Merge cost into usage
+                                usage_chunk_data["usage"]["cost"] = cost_data.get(
+                                    "total_usd", 0.0
+                                )
+                                # Keep detailed cost in metadata
+                                usage_chunk_data["metadata"] = usage_chunk_data.get(
+                                    "metadata", {}
+                                )
+                                usage_chunk_data["metadata"]["routstr"] = {
+                                    "cost": cost_data
+                                }
+                                yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
+                                usage_finalized = True
+                            except Exception:
+                                # Fallback: yield original usage chunk if adjustment fails
+                                yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
 
                 if not usage_finalized:
-                    maybe_cost_event = await finalize_without_usage()
-                    if maybe_cost_event is not None:
-                        yield maybe_cost_event
+                    await finalize_db_only()
+
+                if done_seen:
+                    yield b"data: [DONE]\n\n"
 
             except Exception as stream_error:
                 logger.warning(
-                    "Streaming interrupted; finalizing without usage",
+                    "Streaming interrupted; finalizing in background",
                     extra={
                         "error": str(stream_error),
-                        "error_type": type(stream_error).__name__,
                         "key_hash": key.hashed_key[:8] + "...",
                     },
                 )
-                await finalize_without_usage()
                 raise
+            finally:
+                if not usage_finalized:
+                    await finalize_db_only()
 
         # Remove inaccurate encoding headers from upstream response
         response_headers = dict(response.headers)
@@ -621,6 +595,7 @@ class BaseUpstreamProvider:
             },
         )
 
+        content: bytes | None = None
         try:
             content = await response.aread()
             response_json = json.loads(content)
@@ -637,6 +612,14 @@ class BaseUpstreamProvider:
             cost_data = await adjust_payment_for_tokens(
                 key, response_json, session, deducted_max_cost
             )
+
+            # Merge cost into usage for OpenCode
+            if "usage" in response_json:
+                response_json["usage"]["cost"] = cost_data.get("total_usd", 0.0)
+
+            # Keep detailed cost
+            response_json["metadata"] = response_json.get("metadata", {})
+            response_json["metadata"]["routstr"] = {"cost": cost_data}
             response_json["cost"] = cost_data
 
             logger.info(
@@ -722,167 +705,135 @@ class BaseUpstreamProvider:
         async def stream_with_responses_cost(
             max_cost_for_model: int,
         ) -> AsyncGenerator[bytes, None]:
-            stored_chunks: list[bytes] = []
             usage_finalized: bool = False
             last_model_seen: str | None = None
             reasoning_tokens: int = 0
+            usage_chunk_data: dict | None = None
+            done_seen: bool = False
 
-            async def finalize_without_usage() -> bytes | None:
+            async def finalize_db_only() -> None:
                 nonlocal usage_finalized
                 if usage_finalized:
-                    return None
+                    return
                 async with create_session() as new_session:
                     fresh_key = await new_session.get(key.__class__, key.hashed_key)
                     if not fresh_key:
-                        return None
+                        return
                     try:
-                        fallback: dict = {
-                            "model": last_model_seen or "unknown",
-                            "usage": None,
-                        }
-                        cost_data = await adjust_payment_for_tokens(
-                            fresh_key, fallback, new_session, max_cost_for_model
+                        await adjust_payment_for_tokens(
+                            fresh_key,
+                            {"model": last_model_seen or "unknown", "usage": None},
+                            new_session,
+                            max_cost_for_model,
                         )
                         usage_finalized = True
-                        logger.info(
-                            "Finalized Responses API streaming payment without explicit usage",
-                            extra={
-                                "key_hash": key.hashed_key[:8] + "...",
-                                "cost_data": cost_data,
-                                "balance_after_adjustment": fresh_key.balance,
-                            },
-                        )
-                        return f"data: {json.dumps({'cost': cost_data})}\\n\\n".encode()
-                    except Exception as cost_error:
-                        logger.error(
-                            "Error finalizing Responses API payment without usage",
-                            extra={
-                                "error": str(cost_error),
-                                "error_type": type(cost_error).__name__,
-                                "key_hash": key.hashed_key[:8] + "...",
-                            },
-                        )
-                        return None
-
-            try:
-                async for chunk in response.aiter_bytes():
-                    stored_chunks.append(chunk)
-                    try:
-                        for part in re.split(b"data: ", chunk):
-                            if not part or part.strip() in (b"[DONE]", b""):
-                                continue
-                            try:
-                                obj = json.loads(part)
-                                if isinstance(obj, dict):
-                                    if obj.get("model"):
-                                        last_model_seen = str(obj.get("model"))
-
-                                    # Track reasoning tokens for Responses API
-                                    if usage := obj.get("usage", {}):
-                                        if isinstance(usage, dict) and "reasoning_tokens" in usage:
-                                            reasoning_tokens += usage.get("reasoning_tokens", 0)
-                            except json.JSONDecodeError:
-                                pass
                     except Exception:
                         pass
 
-                    yield chunk
+            try:
+                async for chunk in response.aiter_bytes():
+                    # Split chunk into SSE events
+                    parts = re.split(b"data: ", chunk)
+                    for i, part in enumerate(parts):
+                        if not part:
+                            continue
 
-                logger.debug(
-                    "Responses API streaming completed, analyzing usage data",
-                    extra={
-                        "key_hash": key.hashed_key[:8] + "...",
-                        "chunks_count": len(stored_chunks),
-                        "reasoning_tokens": reasoning_tokens,
-                    },
-                )
+                        stripped_part = part.strip()
+                        if not stripped_part:
+                            continue
 
-                # Process final usage data
-                for i in range(len(stored_chunks) - 1, -1, -1):
-                    chunk = stored_chunks[i]
-                    if not chunk:
-                        continue
-                    try:
-                        events = re.split(b"data: ", chunk)
-                        for event_data in events:
-                            if not event_data or event_data.strip() in (b"[DONE]", b""):
-                                continue
-                            try:
-                                data = json.loads(event_data)
-                                if isinstance(data, dict) and data.get("model"):
-                                    last_model_seen = str(data.get("model"))
-                                if isinstance(data, dict) and isinstance(
-                                    data.get("usage"), dict
-                                ):
-                                    # Include reasoning tokens in usage calculation
-                                    async with create_session() as new_session:
-                                        fresh_key = await new_session.get(
-                                            key.__class__, key.hashed_key
+                        if stripped_part == b"[DONE]":
+                            done_seen = True
+                            continue
+
+                        try:
+                            obj = json.loads(part)
+                            if isinstance(obj, dict):
+                                if obj.get("model"):
+                                    last_model_seen = str(obj.get("model"))
+
+                                # Track reasoning tokens for Responses API
+                                if usage := obj.get("usage", {}):
+                                    if (
+                                        isinstance(usage, dict)
+                                        and "reasoning_tokens" in usage
+                                    ):
+                                        reasoning_tokens += usage.get(
+                                            "reasoning_tokens", 0
                                         )
-                                        if fresh_key:
-                                            try:
-                                                cost_data = (
-                                                    await adjust_payment_for_tokens(
-                                                        fresh_key,
-                                                        data,
-                                                        new_session,
-                                                        max_cost_for_model,
-                                                    )
-                                                )
-                                                usage_finalized = True
-                                                logger.info(
-                                                    "Payment adjustment completed for Responses API streaming",
-                                                    extra={
-                                                        "key_hash": key.hashed_key[:8]
-                                                        + "...",
-                                                        "cost_data": cost_data,
-                                                        "model": last_model_seen,
-                                                        "reasoning_tokens": reasoning_tokens,
-                                                        "balance_after_adjustment": fresh_key.balance,
-                                                    },
-                                                )
-                                                yield f"data: {json.dumps({'cost': cost_data})}\\n\\n".encode()
-                                            except Exception as cost_error:
-                                                logger.error(
-                                                    "Error adjusting payment for Responses API streaming tokens",
-                                                    extra={
-                                                        "error": str(cost_error),
-                                                        "error_type": type(
-                                                            cost_error
-                                                        ).__name__,
-                                                        "key_hash": key.hashed_key[:8]
-                                                        + "...",
-                                                    },
-                                                )
-                                    break
-                            except json.JSONDecodeError:
-                                continue
-                    except Exception as e:
-                        logger.error(
-                            "Error processing Responses API streaming response chunk",
-                            extra={
-                                "error": str(e),
-                                "error_type": type(e).__name__,
-                                "key_hash": key.hashed_key[:8] + "...",
-                            },
+
+                                # Responses API usage is in response.completed/incomplete events
+                                chunk_type = obj.get("type", "")
+                                if chunk_type in (
+                                    "response.completed",
+                                    "response.incomplete",
+                                ):
+                                    usage_chunk_data = obj
+                                    continue
+                        except json.JSONDecodeError:
+                            pass
+
+                        prefix = (
+                            b"data: " if (i > 0 or chunk.startswith(b"data: ")) else b""
                         )
+                        yield prefix + part
+
+                # Stream finished, process usage if found
+                if usage_chunk_data:
+                    async with create_session() as session:
+                        fresh_key = await session.get(key.__class__, key.hashed_key)
+                        if fresh_key:
+                            try:
+                                cost_data = await adjust_payment_for_tokens(
+                                    fresh_key,
+                                    usage_chunk_data,
+                                    session,
+                                    max_cost_for_model,
+                                )
+                                # Merge cost into usage chunk
+                                if (
+                                    "response" in usage_chunk_data
+                                    and "usage" in usage_chunk_data["response"]
+                                ):
+                                    usage_chunk_data["response"]["usage"]["cost"] = (
+                                        cost_data.get("total_usd", 0.0)
+                                    )
+                                elif "usage" in usage_chunk_data:
+                                    usage_chunk_data["usage"]["cost"] = cost_data.get(
+                                        "total_usd", 0.0
+                                    )
+
+                                # Keep detailed cost in metadata
+                                usage_chunk_data["metadata"] = usage_chunk_data.get(
+                                    "metadata", {}
+                                )
+                                usage_chunk_data["metadata"]["routstr"] = {
+                                    "cost": cost_data
+                                }
+                                yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
+                                usage_finalized = True
+                            except Exception:
+                                # Fallback: yield original usage chunk if adjustment fails
+                                yield f"data: {json.dumps(usage_chunk_data)}\n\n".encode()
 
                 if not usage_finalized:
-                    maybe_cost_event = await finalize_without_usage()
-                    if maybe_cost_event is not None:
-                        yield maybe_cost_event
+                    await finalize_db_only()
+
+                if done_seen:
+                    yield b"data: [DONE]\n\n"
 
             except Exception as stream_error:
                 logger.warning(
-                    "Responses API streaming interrupted; finalizing without usage",
+                    "Responses API streaming interrupted; finalizing in background",
                     extra={
                         "error": str(stream_error),
-                        "error_type": type(stream_error).__name__,
                         "key_hash": key.hashed_key[:8] + "...",
                     },
                 )
-                await finalize_without_usage()
                 raise
+            finally:
+                if not usage_finalized:
+                    await finalize_db_only()
 
         # Remove inaccurate encoding headers from upstream response
         response_headers = dict(response.headers)
@@ -922,6 +873,7 @@ class BaseUpstreamProvider:
             },
         )
 
+        content: bytes | None = None
         try:
             content = await response.aread()
             response_json = json.loads(content)
@@ -933,14 +885,22 @@ class BaseUpstreamProvider:
                     "model": response_json.get("model", "unknown"),
                     "has_usage": "usage" in response_json,
                     "has_reasoning_tokens": "usage" in response_json
-                        and isinstance(response_json.get("usage"), dict)
-                        and "reasoning_tokens" in response_json["usage"],
+                    and isinstance(response_json.get("usage"), dict)
+                    and "reasoning_tokens" in response_json["usage"],
                 },
             )
 
             cost_data = await adjust_payment_for_tokens(
                 key, response_json, session, deducted_max_cost
             )
+
+            # Merge cost into usage for OpenCode
+            if "usage" in response_json:
+                response_json["usage"]["cost"] = cost_data.get("total_usd", 0.0)
+
+            # Keep detailed cost
+            response_json["metadata"] = response_json.get("metadata", {})
+            response_json["metadata"]["routstr"] = {"cost": cost_data}
             response_json["cost"] = cost_data
 
             logger.info(
@@ -1001,6 +961,44 @@ class BaseUpstreamProvider:
             )
             raise
 
+    async def _finalize_generic_streaming_payment(
+        self, key_hash: str, max_cost: int, path: str
+    ) -> None:
+        """Background task to finalize payment for generic streaming requests."""
+        async with create_session() as session:
+            key = await session.get(ApiKey, key_hash)
+            if not key:
+                logger.warning(
+                    "Key not found during background payment finalization",
+                    extra={"key_hash": key_hash[:8] + "..."},
+                )
+                return
+
+            try:
+                # Finalize with "unknown" model and no usage to release reservation/charge max cost
+                await adjust_payment_for_tokens(
+                    key,
+                    {"model": "unknown", "usage": None},
+                    session,
+                    max_cost,
+                )
+                logger.info(
+                    "Finalized generic streaming payment in background",
+                    extra={
+                        "path": path,
+                        "key_hash": key_hash[:8] + "...",
+                    },
+                )
+            except Exception as e:
+                logger.error(
+                    "Error finalizing generic streaming payment in background",
+                    extra={
+                        "error": str(e),
+                        "key_hash": key_hash[:8] + "...",
+                        "path": path,
+                    },
+                )
+
     async def forward_request(
         self,
         request: Request,
@@ -1085,6 +1083,14 @@ class BaseUpstreamProvider:
             )
 
             if response.status_code != 200:
+                if response.status_code >= 500:
+                    await response.aclose()
+                    await client.aclose()
+                    raise UpstreamError(
+                        f"Upstream returned status {response.status_code}",
+                        status_code=response.status_code,
+                    )
+
                 try:
                     mapped_error = await self.map_upstream_error_response(
                         request, path, response
@@ -1152,6 +1158,12 @@ class BaseUpstreamProvider:
             background_tasks = BackgroundTasks()
             background_tasks.add_task(response.aclose)
             background_tasks.add_task(client.aclose)
+            background_tasks.add_task(
+                self._finalize_generic_streaming_payment,
+                key.hashed_key,
+                max_cost_for_model,
+                path,
+            )
 
             logger.debug(
                 "Streaming non-chat response",
@@ -1169,6 +1181,9 @@ class BaseUpstreamProvider:
                 background=background_tasks,
             )
 
+        except UpstreamError:
+            raise
+
         except httpx.RequestError as exc:
             await client.aclose()
             error_type = type(exc).__name__
@@ -1196,9 +1211,7 @@ class BaseUpstreamProvider:
             else:
                 error_message = f"Error connecting to upstream service: {error_type}"
 
-            return create_error_response(
-                "upstream_error", error_message, 502, request=request
-            )
+            raise UpstreamError(error_message, status_code=502)
 
         except Exception as exc:
             await client.aclose()
@@ -1311,6 +1324,14 @@ class BaseUpstreamProvider:
             )
 
             if response.status_code != 200:
+                if response.status_code >= 500:
+                    await response.aclose()
+                    await client.aclose()
+                    raise UpstreamError(
+                        f"Upstream returned status {response.status_code}",
+                        status_code=response.status_code,
+                    )
+
                 try:
                     mapped_error = await self.map_upstream_error_response(
                         request, path, response
@@ -1355,9 +1376,15 @@ class BaseUpstreamProvider:
             background_tasks = BackgroundTasks()
             background_tasks.add_task(response.aclose)
             background_tasks.add_task(client.aclose)
+            background_tasks.add_task(
+                self._finalize_generic_streaming_payment,
+                key.hashed_key,
+                max_cost_for_model,
+                path,
+            )
 
             logger.debug(
-                "Streaming non-chat response",
+                "Streaming non-Responses API response",
                 extra={
                     "path": path,
                     "status_code": response.status_code,
@@ -1372,6 +1399,9 @@ class BaseUpstreamProvider:
                 background=background_tasks,
             )
 
+        except UpstreamError:
+            raise
+
         except httpx.RequestError as exc:
             await client.aclose()
             error_type = type(exc).__name__
@@ -1399,9 +1429,7 @@ class BaseUpstreamProvider:
             else:
                 error_message = f"Error connecting to upstream service: {error_type}"
 
-            return create_error_response(
-                "upstream_error", error_message, 502, request=request
-            )
+            raise UpstreamError(error_message, status_code=502)
 
         except Exception as exc:
             await client.aclose()
@@ -2503,7 +2531,10 @@ class BaseUpstreamProvider:
                         usage_data = data_json["usage"]
                         model = data_json.get("model")
                         # Track reasoning tokens for Responses API
-                        if isinstance(usage_data, dict) and "reasoning_tokens" in usage_data:
+                        if (
+                            isinstance(usage_data, dict)
+                            and "reasoning_tokens" in usage_data
+                        ):
                             reasoning_tokens = usage_data.get("reasoning_tokens", 0)
                     elif "model" in data_json and not model:
                         model = data_json["model"]

routstr/upstream/gemini.py

@@ -187,11 +187,44 @@ class GeminiUpstreamProvider(BaseUpstreamProvider):
                 )
 
                 async def stream_with_cost() -> AsyncGenerator[bytes, None]:
+                    payment_finalized = False
+
+                    async def finalize_payment() -> None:
+                        nonlocal payment_finalized
+                        if payment_finalized:
+                            return
+                        from ..auth import adjust_payment_for_tokens
+                        from ..core.db import create_session
+
+                        async with create_session() as new_session:
+                            fresh_key = await new_session.get(
+                                key.__class__, key.hashed_key
+                            )
+                            if fresh_key:
+                                try:
+                                    await adjust_payment_for_tokens(
+                                        fresh_key,
+                                        {
+                                            "model": model_obj.id,
+                                            "usage": final_usage_data,
+                                        },
+                                        new_session,
+                                        max_cost_for_model,
+                                    )
+                                    payment_finalized = True
+                                except Exception as cost_error:
+                                    logger.error(
+                                        "Error finalizing Gemini streaming payment in fallback",
+                                        extra={
+                                            "error": str(cost_error),
+                                            "key_hash": key.hashed_key[:8] + "...",
+                                        },
+                                    )
+
                     try:
                         async for chunk in response_generator:
                             sse_data = f"data: {json.dumps(chunk)}\n\n"
                             yield sse_data.encode()
-
                     except Exception as e:
                         logger.error(
                             "Error in Gemini streaming response",
@@ -202,6 +235,9 @@ class GeminiUpstreamProvider(BaseUpstreamProvider):
                             },
                         )
                         raise
+                    finally:
+                        if not payment_finalized:
+                            await finalize_payment()
 
                 return StreamingResponse(
                     stream_with_cost(),

routstr/upstream/helpers.py

@@ -102,6 +102,8 @@ async def get_all_models_with_overrides(
             )
             for row in override_rows
             if row.upstream_provider_id is not None
+            and row.upstream_provider_id in providers_by_id
+            and providers_by_id[row.upstream_provider_id].enabled
         }
 
     all_models: dict[str, Model] = {}

routstr/upstream/ppqai.py

@@ -196,6 +196,37 @@ class PPQAIUpstreamProvider(BaseUpstreamProvider):
             )
             return []
 
+    async def on_upstream_error_redirect(
+        self, status_code: int, error_message: str
+    ) -> None:
+        if "insufficient balance" in error_message.lower():
+            logger.warning(
+                f"Disabling PPQ.AI provider ({self.base_url}) due to insufficient balance",
+                extra={"error": error_message},
+            )
+            from sqlmodel import select
+
+            from ..core.db import UpstreamProviderRow, create_session
+
+            async with create_session() as session:
+                statement = select(UpstreamProviderRow).where(
+                    UpstreamProviderRow.base_url == self.base_url
+                    and UpstreamProviderRow.api_key == self.api_key
+                )
+                result = await session.exec(statement)
+                provider = result.first()
+
+                if provider:
+                    provider.enabled = False
+                    session.add(provider)
+                    await session.commit()
+
+                    # Trigger re-initialization of providers
+                    # Import here to avoid circular dependency
+                    from ..proxy import reinitialize_upstreams
+
+                    await reinitialize_upstreams()
+
     async def create_account(self) -> dict[str, object]:
         """Create a new PPQ.AI account.
 

routstr/wallet.py

@@ -81,7 +81,7 @@ async def swap_to_primary_mint(
         amount_msat = token_amount
     else:
         raise ValueError("Invalid unit")
-    estimated_fee_sat = math.ceil(max(amount_msat // 1000 * 0.01, 2))
+    estimated_fee_sat = math.ceil(max(amount_msat // 1000 * 0.01, 2)) + 1
     amount_msat_after_fee = amount_msat - estimated_fee_sat * 1000
     primary_wallet = await get_wallet(settings.primary_mint, settings.primary_mint_unit)
 

scripts/reproduce_bug_staging.py

@@ -0,0 +1,101 @@
+import asyncio
+
+import httpx
+
+BASE_URL = input("Enter routstr URL: ")
+API_KEY = input("Enter key or token: ")
+
+
+async def get_balance(client: httpx.AsyncClient) -> int:
+    response = await client.get("/v1/balance/info")
+    response.raise_for_status()
+    data = response.json()
+    print(f"Current Balance Info: {data}")
+    return data.get("reserved", 0)
+
+
+async def reproduce() -> None:
+    headers = {"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"}
+
+    async with httpx.AsyncClient(
+        base_url=BASE_URL, headers=headers, timeout=30.0
+    ) as client:
+        print("Checking initial balance...")
+        try:
+            initial_reserved = await get_balance(client)
+        except Exception as e:
+            print(f"Failed to get balance: {e}")
+            return
+
+        print("\nStarting streaming request...")
+        try:
+            # Create a separate client for the stream so we can close it independently if needed,
+            # but usually just breaking the loop and exiting the context manager is enough.
+            # However, to be sure we simulate a harsh disconnect, we can just cancel the task or close the client.
+
+            async with client.stream(
+                "POST",
+                "/v1/chat/completions",
+                json={
+                    "model": "gpt-5-nano",
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": "Write a long poem about the ocean.",
+                        }
+                    ],
+                    "stream": True,
+                },
+            ) as response:
+                print(f"Stream status: {response.status_code}")
+                if response.status_code != 200:
+                    err_bytes = await response.aread()
+                    try:
+                        err_str = err_bytes.decode()
+                    except Exception:
+                        err_str = repr(err_bytes)
+                    print(f"Error: {err_str}")
+                    return
+
+                print("Stream started. Reading a few chunks...")
+                count = 0
+                async for chunk in response.aiter_bytes():
+                    print(f"Received chunk: {len(chunk)} bytes")
+                    count += 1
+                    if count >= 3:
+                        print("Simulating client disconnect (breaking stream)...")
+                        break
+
+        except Exception as e:
+            print(f"Stream interrupted (expected): {e}")
+
+        # Wait a bit for the server to realize we disconnected (though with asyncio it might be immediate or depend on keepalive)
+        print("\nWaiting for server to process disconnect...")
+        await asyncio.sleep(21)
+
+        print("\nChecking final balance...")
+        try:
+            final_reserved = await get_balance(client)
+        except Exception:
+            # Retry once if connection was closed
+            async with httpx.AsyncClient(
+                base_url=BASE_URL, headers=headers, timeout=30.0
+            ) as new_client:
+                final_reserved = await get_balance(new_client)
+
+        if final_reserved > initial_reserved:
+            print(
+                f"\n[FAIL] Bug reproduced! Reserved balance increased: {initial_reserved} -> {final_reserved}"
+            )
+            print(f"Accumulated reserved balance: {final_reserved - initial_reserved}")
+        else:
+            print(
+                f"\n[PASS] Reserved balance released correctly: {initial_reserved} -> {final_reserved}"
+            )
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(reproduce())
+    except KeyboardInterrupt:
+        pass

tests/integration/test_child_keys.py

@@ -0,0 +1,137 @@
+import secrets
+from typing import Any
+
+import pytest
+from fastapi import HTTPException
+from sqlmodel.ext.asyncio.session import AsyncSession
+
+from routstr.auth import adjust_payment_for_tokens, pay_for_request
+from routstr.balance import create_child_key
+from routstr.core.db import ApiKey
+from routstr.core.settings import settings
+
+
+@pytest.mark.asyncio
+async def test_child_key_flow(integration_session: AsyncSession) -> None:
+    # 1. Create a parent key with balance
+    parent_raw = "parent_test_key_" + secrets.token_hex(4)
+    parent_key = ApiKey(
+        hashed_key=parent_raw,
+        balance=10000,  # 10 sats
+    )
+    integration_session.add(parent_key)
+    await integration_session.commit()
+    await integration_session.refresh(parent_key)
+
+    # Mock settings
+    settings.child_key_cost = 1000  # 1 sat
+
+    # 2. Call create_child_key
+    result = await create_child_key(parent_key, integration_session)
+
+    assert "api_key" in result
+    assert result["cost_msats"] == 1000
+    assert result["parent_balance"] == 9000
+
+    child_key_raw = result["api_key"][3:]  # remove sk-
+
+    # 3. Verify child key exists in DB
+    child_key_db = await integration_session.get(ApiKey, child_key_raw)
+    assert child_key_db is not None
+    assert child_key_db.parent_key_hash == parent_key.hashed_key
+    assert child_key_db.balance == 0
+
+    # 4. Test payment with child key
+    cost = 500
+    await pay_for_request(child_key_db, cost, integration_session)
+
+    # Refresh keys
+    await integration_session.refresh(parent_key)
+    await integration_session.refresh(child_key_db)
+
+    # Parent should be charged
+    assert parent_key.reserved_balance == 500
+    assert parent_key.total_requests == 1
+
+    # Child should have total_requests incremented
+    assert child_key_db.total_requests == 1
+
+    # 5. Test adjustment
+    response_data = {"model": "test-model", "usage": {"total_tokens": 10}}
+
+    # Mock calculate_cost
+    import routstr.auth
+    from routstr.payment.cost_calculation import CostData
+
+    async def mock_calculate_cost(*args: Any, **kwargs: Any) -> CostData:
+        return CostData(
+            base_msats=0, input_msats=200, output_msats=200, total_msats=400
+        )
+
+    # Patch calculate_cost
+    original_calculate_cost = routstr.auth.calculate_cost
+    routstr.auth.calculate_cost = mock_calculate_cost
+
+    try:
+        adjustment = await adjust_payment_for_tokens(
+            child_key_db, response_data, integration_session, 500
+        )
+        assert adjustment["total_msats"] == 400
+
+        # Refresh keys
+        await integration_session.refresh(parent_key)
+        await integration_session.refresh(child_key_db)
+
+        # Parent should have updated balance and total_spent
+        assert parent_key.reserved_balance == 0
+        assert parent_key.balance == 9000 - 400
+        assert (
+            parent_key.total_spent == 1400
+        )  # 1000 for child key creation + 400 for request
+
+        # Child should also have total_spent updated
+        assert child_key_db.total_spent == 400
+
+    finally:
+        routstr.auth.calculate_cost = original_calculate_cost
+
+
+@pytest.mark.asyncio
+async def test_child_key_insufficient_balance(
+    integration_session: AsyncSession,
+) -> None:
+    parent_key = ApiKey(
+        hashed_key="poor_parent_" + secrets.token_hex(4),
+        balance=500,
+    )
+    integration_session.add(parent_key)
+    await integration_session.commit()
+    await integration_session.refresh(parent_key)
+
+    settings.child_key_cost = 1000
+
+    with pytest.raises(HTTPException) as exc:
+        await create_child_key(parent_key, integration_session)
+    assert exc.value.status_code == 402
+
+
+@pytest.mark.asyncio
+async def test_child_key_cannot_create_child(integration_session: AsyncSession) -> None:
+    parent_key = ApiKey(
+        hashed_key="parent_" + secrets.token_hex(4),
+        balance=10000,
+    )
+    child_key = ApiKey(
+        hashed_key="child_" + secrets.token_hex(4),
+        balance=0,
+        parent_key_hash=parent_key.hashed_key,
+    )
+    integration_session.add(parent_key)
+    integration_session.add(child_key)
+    await integration_session.commit()
+    await integration_session.refresh(child_key)
+
+    with pytest.raises(HTTPException) as exc:
+        await create_child_key(child_key, integration_session)
+    assert exc.value.status_code == 400
+    assert "Cannot create a child key for another child key" in str(exc.value.detail)

tests/integration/test_example.py

@@ -10,7 +10,6 @@ from httpx import AsyncClient
 
 from .utils import (
     CashuTokenGenerator,
-    PerformanceValidator,
     ResponseValidator,
 )
 
@@ -159,30 +158,7 @@ async def test_error_handling(
     assert response.status_code == 401
 
 
-@pytest.mark.integration
-@pytest.mark.asyncio
-async def test_performance_requirements(integration_client: AsyncClient) -> None:
-    """Test that endpoints meet performance requirements"""
 
-    validator = PerformanceValidator()
-
-    # Test info endpoint performance
-    for i in range(50):
-        start = validator.start_timing("info_endpoint")
-        response = await integration_client.get("/")
-        validator.end_timing("info_endpoint", start)
-        assert response.status_code == 200
-
-    # Validate 95th percentile is under 500ms
-    result = validator.validate_response_time(
-        "info_endpoint", max_duration=0.5, percentile=0.95
-    )
-
-    assert result["valid"], (
-        f"Performance requirement failed: "
-        f"95th percentile was {result['percentile_time']:.3f}s "
-        f"(required < {result['max_allowed']}s)"
-    )
 
 
 @pytest.mark.integration

tests/integration/test_performance_load.py

@@ -9,6 +9,7 @@ import gc
 import statistics
 import time
 from typing import Any, Dict, List
+from unittest.mock import patch
 
 import psutil
 import pytest
@@ -105,42 +106,46 @@ class TestPerformanceBaseline:
             ("GET", "/v1/wallet/info", authenticated_client, None),
         ]
 
-        # Warm up
-        for _ in range(10):
-            await integration_client.get("/")
+        # Enable provider discovery for this test
+        with patch(
+            "routstr.core.settings.settings.providers_refresh_interval_seconds", 300
+        ):
+            # Warm up
+            for _ in range(10):
+                await integration_client.get("/")
 
-        # Test each endpoint
-        for method, path, client, data in endpoints:
-            response_times = []
+            # Test each endpoint
+            for method, path, client, data in endpoints:
+                response_times = []
 
-            for i in range(100):
-                start = time.time()
+                for i in range(100):
+                    start = time.time()
 
-                if method == "GET":
-                    response = await client.get(path)
-                else:
-                    response = await client.post(path, json=data)
+                    if method == "GET":
+                        response = await client.get(path)
+                    else:
+                        response = await client.post(path, json=data)
 
-                duration = time.time() - start
-                response_times.append(duration * 1000)  # Convert to ms
+                    duration = time.time() - start
+                    response_times.append(duration * 1000)  # Convert to ms
 
-                assert response.status_code in [200, 201]
+                    assert response.status_code in [200, 201]
 
-                if i % 10 == 0:
-                    metrics.record_system_metrics()
+                    if i % 10 == 0:
+                        metrics.record_system_metrics()
 
-            # Verify 95th percentile < 500ms
-            p95 = sorted(response_times)[int(len(response_times) * 0.95)]
-            assert p95 < 500, (
-                f"{method} {path} p95 response time {p95}ms exceeds 500ms limit"
-            )
+                # Verify 95th percentile < 500ms
+                p95 = sorted(response_times)[int(len(response_times) * 0.95)]
+                assert p95 < 500, (
+                    f"{method} {path} p95 response time {p95}ms exceeds 500ms limit"
+                )
 
-            print(f"\n{method} {path}:")
-            print(f"  Mean: {statistics.mean(response_times):.2f}ms")
-            print(f"  P95: {p95:.2f}ms")
-            print(
-                f"  P99: {sorted(response_times)[int(len(response_times) * 0.99)]:.2f}ms"
-            )
+                print(f"\n{method} {path}:")
+                print(f"  Mean: {statistics.mean(response_times):.2f}ms")
+                print(f"  P95: {p95:.2f}ms")
+                print(
+                    f"  P99: {sorted(response_times)[int(len(response_times) * 0.99)]:.2f}ms"
+                )
 
 
 @pytest.mark.integration

tests/integration/test_provider_management.py

@@ -3,7 +3,7 @@ Integration tests for provider management functionality.
 Tests GET /v1/providers/ endpoint for listing and managing providers.
 """
 
-from typing import Any
+from typing import Any, Generator
 from unittest.mock import patch
 
 import pytest
@@ -11,7 +11,7 @@ from httpx import AsyncClient
 
 from routstr.discovery import _PROVIDERS_CACHE
 
-from .utils import PerformanceValidator, ResponseValidator
+from .utils import ResponseValidator
 
 
 @pytest.fixture(autouse=True)
@@ -19,6 +19,15 @@ def _clear_providers_cache() -> None:
     _PROVIDERS_CACHE.clear()
 
 
+@pytest.fixture(autouse=True)
+def _enable_provider_discovery() -> Generator[None, Any, Any]:
+    """Enable provider discovery for all tests in this module"""
+    with patch(
+        "routstr.core.settings.settings.providers_refresh_interval_seconds", 300
+    ):
+        yield
+
+
 @pytest.mark.integration
 @pytest.mark.asyncio
 async def test_providers_endpoint_default_response(
@@ -518,46 +527,6 @@ async def test_providers_endpoint_response_format(
             assert isinstance(data_json["providers"], list)
 
 
-@pytest.mark.integration
-@pytest.mark.asyncio
-async def test_providers_endpoint_performance(integration_client: AsyncClient) -> None:
-    """Test providers endpoint meets performance requirements"""
-
-    # Mock quick responses to avoid network delays
-    mock_events: list[dict[str, Any]] = [
-        {
-            "id": f"event{i}",
-            "content": f"Provider: http://provider{i}.onion",
-            "created_at": 1234567890 + i,
-        }
-        for i in range(5)
-    ]
-
-    validator = PerformanceValidator()
-
-    with patch(
-        "routstr.discovery.query_nostr_relay_for_providers", return_value=mock_events
-    ):
-        with patch("routstr.discovery.fetch_provider_health") as mock_fetch:
-            mock_fetch.return_value = {"status_code": 200, "json": {"status": "online"}}
-
-            # Test multiple requests
-            for i in range(10):
-                start = validator.start_timing("providers_endpoint")
-                response = await integration_client.get("/v1/providers/")
-                validator.end_timing("providers_endpoint", start)
-
-                assert response.status_code == 200
-
-    # Validate performance (should be fast with mocked dependencies)
-    perf_result = validator.validate_response_time(
-        "providers_endpoint",
-        max_duration=2.0,  # Allow more time since it involves multiple operations
-        percentile=0.95,
-    )
-    assert perf_result["valid"], f"Performance requirement failed: {perf_result}"
-
-
 @pytest.mark.integration
 @pytest.mark.asyncio
 async def test_providers_endpoint_concurrent_requests(

tests/integration/test_proxy_get_endpoints.py

@@ -18,7 +18,6 @@ from routstr.core.db import ApiKey
 
 from .utils import (
     ConcurrencyTester,
-    PerformanceValidator,
 )
 
 
@@ -551,39 +550,7 @@ async def test_proxy_get_concurrent_requests(
             assert response.status_code == 200
 
 
-@pytest.mark.integration
-@pytest.mark.asyncio
-async def test_proxy_get_performance_requirements(
-    integration_client: AsyncClient, authenticated_client: AsyncClient
-) -> None:
-    """Test that GET proxy requests meet performance requirements"""
 
-    validator = PerformanceValidator()
-
-    with patch("httpx.AsyncClient.request") as mock_request:
-        mock_response = AsyncMock()
-        mock_response.status_code = 200
-        mock_response.headers = {"content-type": "application/json"}
-        mock_response.json = MagicMock(return_value={"performance": "test"})
-        mock_response.text = '{"performance": "test"}'
-        mock_response.iter_bytes = AsyncMock(return_value=[b'{"performance": "test"}'])
-        mock_request.return_value = mock_response
-
-        # Test multiple requests for performance measurement
-        for i in range(20):
-            start = validator.start_timing("proxy_get")
-            response = await authenticated_client.get(f"/v1/perf-test-{i}")
-            validator.end_timing("proxy_get", start)
-
-            assert response.status_code == 200
-
-    # Validate performance requirements
-    perf_result = validator.validate_response_time(
-        "proxy_get",
-        max_duration=1.0,  # Should complete within 1 second
-        percentile=0.95,
-    )
-    assert perf_result["valid"], f"Performance requirement failed: {perf_result}"
 
 
 @pytest.mark.integration

tests/integration/test_proxy_post_endpoints.py

@@ -15,7 +15,6 @@ from httpx import ASGITransport, AsyncClient
 
 from .utils import (
     ConcurrencyTester,
-    PerformanceValidator,
 )
 
 
@@ -290,55 +289,7 @@ async def test_proxy_post_unauthorized_access(integration_client: AsyncClient) -
     assert response.status_code in [400, 401]
 
 
-@pytest.mark.integration
-@pytest.mark.asyncio
-async def test_proxy_post_performance(
-    integration_client: AsyncClient, authenticated_client: AsyncClient
-) -> None:
-    """Test POST endpoint performance requirements"""
 
-    test_payload = {
-        "model": "gpt-3.5-turbo",
-        "messages": [{"role": "user", "content": "Performance test"}],
-    }
-
-    validator = PerformanceValidator()
-
-    with patch("httpx.AsyncClient.send") as mock_send:
-        # Mock fast responses
-        async def mock_iter_bytes(*args: Any, **kwargs: Any) -> Any:
-            yield b'{"choices": [{"message": {"content": "Fast"}}], "usage": {"total_tokens": 5}}'
-
-        mock_response = AsyncMock()
-        mock_response.status_code = 200
-        mock_response.headers = {"content-type": "application/json"}
-        response_data = {
-            "choices": [{"message": {"content": "Fast"}}],
-            "usage": {"total_tokens": 5},
-        }
-        mock_response.text = json.dumps(response_data)
-        mock_response.json = AsyncMock(return_value=response_data)
-        mock_response.iter_bytes = mock_iter_bytes
-        mock_response.aiter_bytes = mock_iter_bytes
-        mock_send.return_value = mock_response
-
-        # Run multiple requests for performance measurement
-        for i in range(20):
-            start = validator.start_timing("proxy_post")
-            response = await authenticated_client.post(
-                "/v1/chat/completions", json=test_payload
-            )
-            validator.end_timing("proxy_post", start)
-
-            assert response.status_code == 200
-
-    # Validate performance
-    perf_result = validator.validate_response_time(
-        "proxy_post",
-        max_duration=1.5,  # Allow slightly more time for POST
-        percentile=0.95,
-    )
-    assert perf_result["valid"], f"Performance requirement failed: {perf_result}"
 
 
 @pytest.mark.integration

tests/integration/test_wallet_information.py

@@ -3,7 +3,7 @@ Integration tests for wallet information retrieval endpoints.
 Tests GET /v1/wallet/ and GET /v1/wallet/info endpoints with various scenarios.
 """
 
-import time
+
 from datetime import datetime, timedelta
 from typing import Any
 
@@ -367,30 +367,4 @@ async def test_wallet_info_with_special_characters_in_headers(
     # Note: Current implementation doesn't return refund_address in response
 
 
-@pytest.mark.integration
-@pytest.mark.asyncio
-@pytest.mark.slow
-async def test_wallet_endpoints_performance(authenticated_client: AsyncClient) -> None:
-    """Test wallet endpoints meet performance requirements"""
 
-    # Warm up
-    await authenticated_client.get("/v1/wallet/")
-
-    # Measure response times
-    response_times = []
-
-    for _ in range(50):
-        start_time = time.time()
-        response = await authenticated_client.get("/v1/wallet/")
-        end_time = time.time()
-
-        assert response.status_code == 200
-        response_times.append(end_time - start_time)
-
-    # Calculate statistics
-    avg_time = sum(response_times) / len(response_times)
-    max_time = max(response_times)
-
-    # Performance assertions
-    assert avg_time < 0.1  # Average should be under 100ms
-    assert max_time < 0.5  # No request should take more than 500ms

tests/integration/test_wallet_refund.py

@@ -537,42 +537,4 @@ async def test_refund_with_expired_key(
         assert response.json()["recipient"] == "expired@ln.address"
 
 
-@pytest.mark.integration
-@pytest.mark.asyncio
-@pytest.mark.slow
-async def test_refund_performance(
-    integration_client: AsyncClient, testmint_wallet: Any
-) -> None:
-    """Test refund endpoint performance"""
 
-    import time
-
-    # Create multiple API keys
-    api_keys = []
-    for i in range(10):
-        token = await testmint_wallet.mint_tokens(100 + i)
-        # Use cashu token as Bearer auth to create API key
-        integration_client.headers["Authorization"] = f"Bearer {token}"
-        response = await integration_client.get("/v1/wallet/info")
-        assert response.status_code == 200
-        api_keys.append(response.json()["api_key"])
-
-    # Measure refund times
-    refund_times = []
-
-    for api_key in api_keys:
-        integration_client.headers["Authorization"] = f"Bearer {api_key}"
-
-        start_time = time.time()
-        response = await integration_client.post("/v1/wallet/refund")
-        end_time = time.time()
-
-        assert response.status_code == 200
-        refund_times.append(end_time - start_time)
-
-    # Performance assertions
-    avg_time = sum(refund_times) / len(refund_times)
-    max_time = max(refund_times)
-
-    assert avg_time < 0.5  # Average under 500ms
-    assert max_time < 1.0  # No refund takes more than 1 second

tests/unit/test_algorithm.py

@@ -10,7 +10,6 @@ os.environ["UPSTREAM_API_KEY"] = "test"
 from routstr.algorithm import (  # noqa: E402
     calculate_model_cost_score,
     get_provider_penalty,
-    should_prefer_model,
 )
 from routstr.payment.models import Architecture, Model, Pricing  # noqa: E402
 
@@ -100,100 +99,3 @@ def test_get_provider_penalty_openrouter() -> None:
     provider = create_test_provider("openrouter", "https://openrouter.ai/api/v1")
     penalty = get_provider_penalty(provider)
     assert penalty == 1.001
-
-
-def test_should_prefer_model_cheaper_wins() -> None:
-    """Test that cheaper model is preferred."""
-    cheap_model = create_test_model("cheap", prompt_price=0.001, completion_price=0.002)
-    expensive_model = create_test_model(
-        "expensive", prompt_price=0.03, completion_price=0.06
-    )
-
-    provider1 = create_test_provider("provider1")
-    provider2 = create_test_provider("provider2")
-
-    # Cheaper model should win
-    assert should_prefer_model(
-        cheap_model, provider1, expensive_model, provider2, "test-alias"
-    )
-
-    # More expensive model should not win
-    assert not should_prefer_model(
-        expensive_model, provider2, cheap_model, provider1, "test-alias"
-    )
-
-
-def test_should_prefer_model_exact_match_wins() -> None:
-    """Test that exact alias match beats cheaper price."""
-    # Make model IDs match the alias differently
-    exact_match = create_test_model(
-        "test-model", prompt_price=0.03, completion_price=0.06
-    )
-    no_match = create_test_model(
-        "other-model", prompt_price=0.001, completion_price=0.002
-    )
-
-    provider1 = create_test_provider("provider1")
-    provider2 = create_test_provider("provider2")
-
-    # Exact match should win even though it's more expensive
-    assert should_prefer_model(
-        exact_match, provider1, no_match, provider2, "test-model"
-    )
-
-
-def test_should_prefer_model_openrouter_slight_penalty() -> None:
-    """Test that OpenRouter has slight penalty compared to other providers."""
-    model1 = create_test_model("model1", prompt_price=0.001, completion_price=0.002)
-    model2 = create_test_model("model2", prompt_price=0.001, completion_price=0.002)
-
-    regular_provider = create_test_provider("regular", "http://provider.com")
-    openrouter_provider = create_test_provider(
-        "openrouter", "https://openrouter.ai/api/v1"
-    )
-
-    # Regular provider should be preferred over OpenRouter at same cost
-    assert should_prefer_model(
-        model1, regular_provider, model2, openrouter_provider, "test-alias"
-    )
-
-    # OpenRouter should not replace regular provider at same cost
-    assert not should_prefer_model(
-        model2, openrouter_provider, model1, regular_provider, "test-alias"
-    )
-
-
-def test_should_prefer_model_openrouter_can_win_if_cheaper() -> None:
-    """Test that OpenRouter can still win if significantly cheaper."""
-    cheap_model = create_test_model(
-        "cheap", prompt_price=0.0001, completion_price=0.0002
-    )
-    expensive_model = create_test_model(
-        "expensive", prompt_price=0.03, completion_price=0.06
-    )
-
-    regular_provider = create_test_provider("regular", "http://provider.com")
-    openrouter_provider = create_test_provider(
-        "openrouter", "https://openrouter.ai/api/v1"
-    )
-
-    # OpenRouter should win if it's much cheaper (even with penalty)
-    assert should_prefer_model(
-        cheap_model,
-        openrouter_provider,
-        expensive_model,
-        regular_provider,
-        "test-alias",
-    )
-
-
-def test_should_prefer_model_same_cost_first_wins() -> None:
-    """Test that when costs are identical, current model is kept."""
-    model1 = create_test_model("model1", prompt_price=0.001, completion_price=0.002)
-    model2 = create_test_model("model2", prompt_price=0.001, completion_price=0.002)
-
-    provider1 = create_test_provider("provider1")
-    provider2 = create_test_provider("provider2")
-
-    # When costs are equal, should not replace
-    assert not should_prefer_model(model2, provider2, model1, provider1, "test-alias")

ui/Dockerfile

@@ -4,17 +4,22 @@ FROM base AS deps
 RUN apk add --no-cache libc6-compat
 WORKDIR /app
 
-COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* .npmrc* ./
-RUN npm i
+RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
+
+COPY package.json pnpm-lock.yaml* ./
+RUN pnpm install --frozen-lockfile
 
 FROM base AS builder
 WORKDIR /app
+
+RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
+
 COPY --from=deps /app/node_modules ./node_modules
 COPY . .
 
 ENV NEXT_TELEMETRY_DISABLED=1
 
-RUN npm run build
+RUN pnpm run build
 FROM base AS runner
 WORKDIR /app
 

ui/Dockerfile.build

@@ -6,12 +6,14 @@ RUN apk add --no-cache libc6-compat
 WORKDIR /app
 
 # Copy package files
-COPY package.json package-lock.json* pnpm-lock.yaml* ./
-RUN npm ci
+COPY package.json pnpm-lock.yaml* ./
+RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
+RUN pnpm install --frozen-lockfile
 
 # Build the UI
 FROM base AS builder
 WORKDIR /app
+RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
 COPY --from=deps /app/node_modules ./node_modules
 COPY . .
 
@@ -27,7 +29,7 @@ ENV NODE_ENV=production
 ENV NEXT_TELEMETRY_DISABLED=1
 
 # Build the application
-RUN npm run build && \
+RUN pnpm run build && \
     echo "UI build completed at $(date)"
 
 # Use the builder stage as the final stage

ui/app/logs/log-filters.tsx

@@ -21,7 +21,17 @@ import {
   PopoverTrigger,
 } from '@/components/ui/popover';
 import { Calendar } from '@/components/ui/calendar';
-import { CalendarIcon, Filter, X } from 'lucide-react';
+import { Badge } from '@/components/ui/badge';
+import {
+  Command,
+  CommandEmpty,
+  CommandGroup,
+  CommandInput,
+  CommandItem,
+  CommandList,
+} from '@/components/ui/command';
+import { Checkbox } from '@/components/ui/checkbox';
+import { CalendarIcon, Filter, X, Plus } from 'lucide-react';
 import { useState, useEffect } from 'react';
 import { format } from 'date-fns';
 import { cn } from '@/lib/utils';
@@ -31,11 +41,17 @@ interface LogFiltersProps {
   selectedLevel: string;
   requestId: string;
   searchText: string;
+  selectedStatusCodes: string[];
+  selectedMethods: string[];
+  selectedEndpoints: string[];
   limit: number;
   onDateChange: (date: string) => void;
   onLevelChange: (level: string) => void;
   onRequestIdChange: (requestId: string) => void;
   onSearchTextChange: (searchText: string) => void;
+  onStatusCodesChange: (statusCodes: string[]) => void;
+  onMethodsChange: (methods: string[]) => void;
+  onEndpointsChange: (endpoints: string[]) => void;
   onLimitChange: (limit: number) => void;
   onClearFilters: () => void;
 }
@@ -43,16 +59,87 @@ interface LogFiltersProps {
 const LOG_LEVELS = ['TRACE', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'];
 const PRESET_LIMITS = ['25', '50', '100', '200', '500', '1000'];
 
+const STATUS_CODE_OPTIONS = [
+  '200',
+  '201',
+  '204',
+  '400',
+  '401',
+  '402',
+  '403',
+  '404',
+  '422',
+  '429',
+  '500',
+  '502',
+  '503',
+  '504',
+];
+
+const METHOD_OPTIONS = [
+  'GET',
+  'POST',
+  'PUT',
+  'DELETE',
+  'PATCH',
+  'OPTIONS',
+  'HEAD',
+];
+
+const ENDPOINT_OPTIONS = [
+  '/chat/completions',
+  '/v1/chat/completions',
+  '/models',
+  '/v1/models',
+  '/responses',
+  '/v1/responses',
+  'v1/embeddings/models',
+  '/embeddings/models',
+];
+
+interface FilterBadgeProps {
+  value: string;
+  onRemove: (value: string) => void;
+}
+
+function FilterBadge({ value, onRemove }: FilterBadgeProps) {
+  return (
+    <Badge
+      variant='secondary'
+      className='flex items-center gap-1 px-1 font-normal'
+    >
+      {value}
+      <button
+        type='button'
+        onClick={(e) => {
+          e.preventDefault();
+          e.stopPropagation();
+          onRemove(value);
+        }}
+        className='hover:bg-muted-foreground/20 rounded-full'
+      >
+        <X className='h-3 w-3' />
+      </button>
+    </Badge>
+  );
+}
+
 export function LogFilters({
   selectedDate,
   selectedLevel,
   requestId,
   searchText,
+  selectedStatusCodes,
+  selectedMethods,
+  selectedEndpoints,
   limit,
   onDateChange,
   onLevelChange,
   onRequestIdChange,
   onSearchTextChange,
+  onStatusCodesChange,
+  onMethodsChange,
+  onEndpointsChange,
   onLimitChange,
   onClearFilters,
 }: LogFiltersProps) {
@@ -68,6 +155,10 @@ export function LogFilters({
       : undefined
   );
 
+  const [statusSearch, setStatusSearch] = useState('');
+  const [methodSearch, setMethodSearch] = useState('');
+  const [endpointSearch, setEndpointSearch] = useState('');
+
   useEffect(() => {
     const currentIsPreset = PRESET_LIMITS.includes(limit.toString());
     setIsCustom(!currentIsPreset);
@@ -129,6 +220,31 @@ export function LogFilters({
     }
   };
 
+  const toggleSelection = (
+    current: string[],
+    value: string,
+    onChange: (val: string[]) => void
+  ) => {
+    if (current.includes(value)) {
+      onChange(current.filter((v) => v !== value));
+    } else {
+      onChange([...current, value]);
+    }
+  };
+
+  const handleQuickStatusCode = (range: '4xx' | '5xx') => {
+    const codes = STATUS_CODE_OPTIONS.filter((c) => c.startsWith(range[0]));
+    const newSelection = new Set([...selectedStatusCodes]);
+    const allIncluded = codes.every((c) => selectedStatusCodes.includes(c));
+
+    if (allIncluded) {
+      codes.forEach((c) => newSelection.delete(c));
+    } else {
+      codes.forEach((c) => newSelection.add(c));
+    }
+    onStatusCodesChange(Array.from(newSelection));
+  };
+
   return (
     <Card className='mb-6'>
       <CardHeader>
@@ -137,7 +253,8 @@ export function LogFilters({
           Filters
         </CardTitle>
         <CardDescription>
-          Filter logs by date, level, request ID, text search, and limit
+          Filter logs by date, level, request ID, text search, status code,
+          method, endpoint and limit
         </CardDescription>
       </CardHeader>
       <CardContent>
@@ -197,6 +314,337 @@ export function LogFilters({
             </Select>
           </div>
 
+          <div className='space-y-2'>
+            <Label>Status Codes</Label>
+            <Popover>
+              <PopoverTrigger asChild>
+                <Button
+                  variant='outline'
+                  className='w-full justify-start text-left font-normal'
+                >
+                  <div className='flex flex-wrap gap-1'>
+                    {selectedStatusCodes.length > 0 ? (
+                      selectedStatusCodes.map((code) => (
+                        <FilterBadge
+                          key={code}
+                          value={code}
+                          onRemove={(val) =>
+                            toggleSelection(
+                              selectedStatusCodes,
+                              val,
+                              onStatusCodesChange
+                            )
+                          }
+                        />
+                      ))
+                    ) : (
+                      <span className='text-muted-foreground'>All codes</span>
+                    )}
+                  </div>
+                </Button>
+              </PopoverTrigger>
+              <PopoverContent className='w-64 p-0' align='start'>
+                <Command>
+                  <CommandInput
+                    placeholder='Search or add status code...'
+                    value={statusSearch}
+                    onValueChange={setStatusSearch}
+                  />
+                  <CommandList>
+                    {selectedStatusCodes.length > 0 && (
+                      <CommandGroup heading='Selected'>
+                        {selectedStatusCodes.map((code) => (
+                          <CommandItem
+                            key={`selected-${code}`}
+                            onSelect={() =>
+                              toggleSelection(
+                                selectedStatusCodes,
+                                code,
+                                onStatusCodesChange
+                              )
+                            }
+                          >
+                            <Checkbox checked={true} className='mr-2' />
+                            {code}
+                          </CommandItem>
+                        ))}
+                      </CommandGroup>
+                    )}
+                    {statusSearch &&
+                      !STATUS_CODE_OPTIONS.includes(statusSearch) &&
+                      !selectedStatusCodes.includes(statusSearch) && (
+                        <CommandGroup heading='Custom'>
+                          <CommandItem
+                            onSelect={() => {
+                              if (/^\d+$/.test(statusSearch)) {
+                                toggleSelection(
+                                  selectedStatusCodes,
+                                  statusSearch,
+                                  onStatusCodesChange
+                                );
+                                setStatusSearch('');
+                              }
+                            }}
+                          >
+                            <Plus className='mr-2 h-4 w-4' />
+                            Add &quot;{statusSearch}&quot;
+                          </CommandItem>
+                        </CommandGroup>
+                      )}
+                    <CommandEmpty>No results found.</CommandEmpty>
+                    <CommandGroup heading='Quick Filters'>
+                      <CommandItem
+                        onSelect={() => handleQuickStatusCode('4xx')}
+                      >
+                        <Checkbox
+                          checked={STATUS_CODE_OPTIONS.filter((c) =>
+                            c.startsWith('4')
+                          ).every((c) => selectedStatusCodes.includes(c))}
+                          className='mr-2'
+                        />
+                        4xx Errors
+                      </CommandItem>
+                      <CommandItem
+                        onSelect={() => handleQuickStatusCode('5xx')}
+                      >
+                        <Checkbox
+                          checked={STATUS_CODE_OPTIONS.filter((c) =>
+                            c.startsWith('5')
+                          ).every((c) => selectedStatusCodes.includes(c))}
+                          className='mr-2'
+                        />
+                        5xx Errors
+                      </CommandItem>
+                    </CommandGroup>
+                    <CommandGroup heading='Common Codes'>
+                      {STATUS_CODE_OPTIONS.filter(
+                        (code) => !selectedStatusCodes.includes(code)
+                      ).map((code) => (
+                        <CommandItem
+                          key={code}
+                          onSelect={() =>
+                            toggleSelection(
+                              selectedStatusCodes,
+                              code,
+                              onStatusCodesChange
+                            )
+                          }
+                        >
+                          <Checkbox checked={false} className='mr-2' />
+                          {code}
+                        </CommandItem>
+                      ))}
+                    </CommandGroup>
+                  </CommandList>
+                </Command>
+              </PopoverContent>
+            </Popover>
+          </div>
+
+          <div className='space-y-2'>
+            <Label>HTTP Methods</Label>
+            <Popover>
+              <PopoverTrigger asChild>
+                <Button
+                  variant='outline'
+                  className='w-full justify-start text-left font-normal'
+                >
+                  <div className='flex flex-wrap gap-1'>
+                    {selectedMethods.length > 0 ? (
+                      selectedMethods.map((method) => (
+                        <FilterBadge
+                          key={method}
+                          value={method}
+                          onRemove={(val) =>
+                            toggleSelection(
+                              selectedMethods,
+                              val,
+                              onMethodsChange
+                            )
+                          }
+                        />
+                      ))
+                    ) : (
+                      <span className='text-muted-foreground'>All methods</span>
+                    )}
+                  </div>
+                </Button>
+              </PopoverTrigger>
+              <PopoverContent className='w-64 p-0' align='start'>
+                <Command>
+                  <CommandInput
+                    placeholder='Search or add method...'
+                    value={methodSearch}
+                    onValueChange={setMethodSearch}
+                  />
+                  <CommandList>
+                    {selectedMethods.length > 0 && (
+                      <CommandGroup heading='Selected'>
+                        {selectedMethods.map((method) => (
+                          <CommandItem
+                            key={`selected-${method}`}
+                            onSelect={() =>
+                              toggleSelection(
+                                selectedMethods,
+                                method,
+                                onMethodsChange
+                              )
+                            }
+                          >
+                            <Checkbox checked={true} className='mr-2' />
+                            {method}
+                          </CommandItem>
+                        ))}
+                      </CommandGroup>
+                    )}
+                    {methodSearch &&
+                      !METHOD_OPTIONS.includes(methodSearch.toUpperCase()) &&
+                      !selectedMethods.includes(methodSearch.toUpperCase()) && (
+                        <CommandGroup heading='Custom'>
+                          <CommandItem
+                            onSelect={() => {
+                              toggleSelection(
+                                selectedMethods,
+                                methodSearch.toUpperCase(),
+                                onMethodsChange
+                              );
+                              setMethodSearch('');
+                            }}
+                          >
+                            <Plus className='mr-2 h-4 w-4' />
+                            Add &quot;{methodSearch.toUpperCase()}&quot;
+                          </CommandItem>
+                        </CommandGroup>
+                      )}
+                    <CommandEmpty>No results found.</CommandEmpty>
+                    <CommandGroup>
+                      {METHOD_OPTIONS.filter(
+                        (method) => !selectedMethods.includes(method)
+                      ).map((method) => (
+                        <CommandItem
+                          key={method}
+                          onSelect={() =>
+                            toggleSelection(
+                              selectedMethods,
+                              method,
+                              onMethodsChange
+                            )
+                          }
+                        >
+                          <Checkbox checked={false} className='mr-2' />
+                          {method}
+                        </CommandItem>
+                      ))}
+                    </CommandGroup>
+                  </CommandList>
+                </Command>
+              </PopoverContent>
+            </Popover>
+          </div>
+
+          <div className='space-y-2'>
+            <Label>Endpoints</Label>
+            <Popover>
+              <PopoverTrigger asChild>
+                <Button
+                  variant='outline'
+                  className='w-full justify-start text-left font-normal'
+                >
+                  <div className='flex flex-wrap gap-1 overflow-hidden'>
+                    {selectedEndpoints.length > 0 ? (
+                      selectedEndpoints.map((endpoint) => (
+                        <FilterBadge
+                          key={endpoint}
+                          value={endpoint}
+                          onRemove={(val) =>
+                            toggleSelection(
+                              selectedEndpoints,
+                              val,
+                              onEndpointsChange
+                            )
+                          }
+                        />
+                      ))
+                    ) : (
+                      <span className='text-muted-foreground'>
+                        All endpoints
+                      </span>
+                    )}
+                  </div>
+                </Button>
+              </PopoverTrigger>
+              <PopoverContent className='w-80 p-0' align='start'>
+                <Command>
+                  <CommandInput
+                    placeholder='Search or add endpoint pattern...'
+                    value={endpointSearch}
+                    onValueChange={setEndpointSearch}
+                  />
+                  <CommandList>
+                    {selectedEndpoints.length > 0 && (
+                      <CommandGroup heading='Selected'>
+                        {selectedEndpoints.map((endpoint) => (
+                          <CommandItem
+                            key={`selected-${endpoint}`}
+                            onSelect={() =>
+                              toggleSelection(
+                                selectedEndpoints,
+                                endpoint,
+                                onEndpointsChange
+                              )
+                            }
+                          >
+                            <Checkbox checked={true} className='mr-2' />
+                            {endpoint}
+                          </CommandItem>
+                        ))}
+                      </CommandGroup>
+                    )}
+                    {endpointSearch &&
+                      !ENDPOINT_OPTIONS.includes(endpointSearch) &&
+                      !selectedEndpoints.includes(endpointSearch) && (
+                        <CommandGroup heading='Custom'>
+                          <CommandItem
+                            onSelect={() => {
+                              toggleSelection(
+                                selectedEndpoints,
+                                endpointSearch,
+                                onEndpointsChange
+                              );
+                              setEndpointSearch('');
+                            }}
+                          >
+                            <Plus className='mr-2 h-4 w-4' />
+                            Add &quot;{endpointSearch}&quot;
+                          </CommandItem>
+                        </CommandGroup>
+                      )}
+                    <CommandEmpty>No results found.</CommandEmpty>
+                    <CommandGroup heading='Common Endpoints'>
+                      {ENDPOINT_OPTIONS.filter(
+                        (endpoint) => !selectedEndpoints.includes(endpoint)
+                      ).map((endpoint) => (
+                        <CommandItem
+                          key={endpoint}
+                          onSelect={() =>
+                            toggleSelection(
+                              selectedEndpoints,
+                              endpoint,
+                              onEndpointsChange
+                            )
+                          }
+                        >
+                          <Checkbox checked={false} className='mr-2' />
+                          {endpoint}
+                        </CommandItem>
+                      ))}
+                    </CommandGroup>
+                  </CommandList>
+                </Command>
+              </PopoverContent>
+            </Popover>
+          </div>
+
           <div className='space-y-2'>
             <Label htmlFor='request-id'>Request ID</Label>
             <Input

ui/app/logs/page.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { useState } from 'react';
+import { useState, useEffect } from 'react';
 import { useQuery } from '@tanstack/react-query';
 import { AppSidebar } from '@/components/app-sidebar';
 import { SiteHeader } from '@/components/site-header';
@@ -22,15 +22,66 @@ import { LogFilters } from './log-filters';
 import { LogEntryCard } from './log-entry-card';
 import { LogDetailsDialog } from './log-details-dialog';
 
+const STORAGE_KEY = 'routstr-log-filters';
+
 export default function LogsPage() {
   const [selectedDate, setSelectedDate] = useState<string>('all');
   const [selectedLevel, setSelectedLevel] = useState<string>('all');
   const [requestId, setRequestId] = useState<string>('');
   const [searchText, setSearchText] = useState<string>('');
+  const [selectedStatusCodes, setSelectedStatusCodes] = useState<string[]>([]);
+  const [selectedMethods, setSelectedMethods] = useState<string[]>([]);
+  const [selectedEndpoints, setSelectedEndpoints] = useState<string[]>([]);
   const [limit, setLimit] = useState<number>(100);
   const [selectedLog, setSelectedLog] = useState<LogEntry | null>(null);
   const [isDialogOpen, setIsDialogOpen] = useState<boolean>(false);
 
+  // Load filters from localStorage on mount
+  useEffect(() => {
+    const saved = localStorage.getItem(STORAGE_KEY);
+    if (saved) {
+      try {
+        const parsed = JSON.parse(saved);
+        if (parsed.selectedDate) setSelectedDate(parsed.selectedDate);
+        if (parsed.selectedLevel) setSelectedLevel(parsed.selectedLevel);
+        if (parsed.requestId) setRequestId(parsed.requestId);
+        if (parsed.searchText) setSearchText(parsed.searchText);
+        if (parsed.selectedStatusCodes)
+          setSelectedStatusCodes(parsed.selectedStatusCodes);
+        if (parsed.selectedMethods) setSelectedMethods(parsed.selectedMethods);
+        if (parsed.selectedEndpoints)
+          setSelectedEndpoints(parsed.selectedEndpoints);
+        if (parsed.limit) setLimit(parsed.limit);
+      } catch (e) {
+        console.error('Failed to load filters from localStorage', e);
+      }
+    }
+  }, []);
+
+  // Save filters to localStorage whenever they change
+  useEffect(() => {
+    const filters = {
+      selectedDate,
+      selectedLevel,
+      requestId,
+      searchText,
+      selectedStatusCodes,
+      selectedMethods,
+      selectedEndpoints,
+      limit,
+    };
+    localStorage.setItem(STORAGE_KEY, JSON.stringify(filters));
+  }, [
+    selectedDate,
+    selectedLevel,
+    requestId,
+    searchText,
+    selectedStatusCodes,
+    selectedMethods,
+    selectedEndpoints,
+    limit,
+  ]);
+
   const {
     data: logsData,
     refetch: refetchLogs,
@@ -42,6 +93,9 @@ export default function LogsPage() {
       selectedLevel,
       requestId,
       searchText,
+      selectedStatusCodes,
+      selectedMethods,
+      selectedEndpoints,
       limit,
     ],
     queryFn: () =>
@@ -50,6 +104,16 @@ export default function LogsPage() {
         level: selectedLevel === 'all' ? undefined : selectedLevel,
         request_id: requestId || undefined,
         search: searchText || undefined,
+        status_codes:
+          selectedStatusCodes.length > 0
+            ? selectedStatusCodes.join(',')
+            : undefined,
+        methods:
+          selectedMethods.length > 0 ? selectedMethods.join(',') : undefined,
+        endpoints:
+          selectedEndpoints.length > 0
+            ? selectedEndpoints.join(',')
+            : undefined,
         limit: limit,
       }),
     refetchInterval: 30000,
@@ -60,6 +124,9 @@ export default function LogsPage() {
     setSelectedLevel('all');
     setRequestId('');
     setSearchText('');
+    setSelectedStatusCodes([]);
+    setSelectedMethods([]);
+    setSelectedEndpoints([]);
     setLimit(100);
   };
 
@@ -100,11 +167,17 @@ export default function LogsPage() {
             selectedLevel={selectedLevel}
             requestId={requestId}
             searchText={searchText}
+            selectedStatusCodes={selectedStatusCodes}
+            selectedMethods={selectedMethods}
+            selectedEndpoints={selectedEndpoints}
             limit={limit}
             onDateChange={setSelectedDate}
             onLevelChange={setSelectedLevel}
             onRequestIdChange={setRequestId}
             onSearchTextChange={setSearchText}
+            onStatusCodesChange={setSelectedStatusCodes}
+            onMethodsChange={setSelectedMethods}
+            onEndpointsChange={setSelectedEndpoints}
             onLimitChange={setLimit}
             onClearFilters={handleClearFilters}
           />
@@ -122,13 +195,22 @@ export default function LogsPage() {
               {(selectedDate !== 'all' ||
                 selectedLevel !== 'all' ||
                 requestId ||
-                searchText) && (
+                searchText ||
+                selectedStatusCodes.length > 0 ||
+                selectedMethods.length > 0 ||
+                selectedEndpoints.length > 0) && (
                 <CardDescription className='text-xs sm:text-sm'>
                   Showing logs
                   {selectedDate !== 'all' && ` for ${selectedDate}`}
                   {selectedLevel !== 'all' && ` with level ${selectedLevel}`}
                   {requestId && ` with request ID ${requestId}`}
                   {searchText && ` matching "${searchText}"`}
+                  {selectedStatusCodes.length > 0 &&
+                    ` with status ${selectedStatusCodes.join(', ')}`}
+                  {selectedMethods.length > 0 &&
+                    ` with method ${selectedMethods.join(', ')}`}
+                  {selectedEndpoints.length > 0 &&
+                    ` with endpoint ${selectedEndpoints.join(', ')}`}
                 </CardDescription>
               )}
             </CardHeader>

ui/app/logs/types.ts

@@ -17,6 +17,9 @@ export interface LogsResponse {
   level: string | null;
   request_id: string | null;
   search: string | null;
+  status_codes: string | null;
+  methods: string | null;
+  endpoints: string | null;
   limit: number;
 }
 

ui/app/providers/page.tsx

@@ -208,7 +208,12 @@ function ProviderBalance({
     return <Skeleton className='h-9 w-24' />;
   }
 
-  if (error || !balanceData?.ok || !balanceData.balance_data) {
+  if (
+    error ||
+    !balanceData?.ok ||
+    balanceData.balance_data === undefined ||
+    balanceData.balance_data === null
+  ) {
     return null;
   }
 

ui/components/temporary-balances.tsx

@@ -60,7 +60,11 @@ export function TemporaryBalances({
     let totalRequests = 0;
 
     balances.forEach((balance) => {
-      totalBalance += balance.balance || 0;
+      // Only count parents for total balance to avoid double counting
+      // since child keys use parent balance
+      if (!balance.parent_key_hash) {
+        totalBalance += balance.balance || 0;
+      }
       totalSpent += balance.total_spent || 0;
       totalRequests += balance.total_requests || 0;
     });
@@ -72,6 +76,34 @@ export function TemporaryBalances({
     ? calculateTotals(data)
     : { totalBalance: 0, totalSpent: 0, totalRequests: 0 };
 
+  // Group parents and children
+  const hierarchicalData = (() => {
+    if (!data) return [];
+
+    const parents = filteredData.filter((item) => !item.parent_key_hash);
+    const result: (TemporaryBalance & { isChild?: boolean })[] = [];
+
+    parents.forEach((parent) => {
+      result.push(parent);
+      const children = data.filter(
+        (item) => item.parent_key_hash === parent.hashed_key
+      );
+      children.forEach((child) => {
+        result.push({ ...child, isChild: true });
+      });
+    });
+
+    // Add children whose parents didn't match the search or aren't in the list
+    const orphans = filteredData.filter(
+      (item) =>
+        item.parent_key_hash &&
+        !result.some((r) => r.hashed_key === item.hashed_key)
+    );
+    result.push(...orphans.map((o) => ({ ...o, isChild: true })));
+
+    return result;
+  })();
+
   return (
     <>
       <Card className='h-full w-full shadow-sm'>
@@ -182,22 +214,37 @@ export function TemporaryBalances({
                   <div className='text-right'>Expiry Time</div>
                 </div>
 
-                {filteredData.length > 0 ? (
-                  filteredData.map((balance, index) => (
+                {hierarchicalData.length > 0 ? (
+                  hierarchicalData.map((balance, index) => (
                     <div
                       key={index}
                       className={cn(
                         'hover:bg-muted/50 border-t p-3 text-sm transition-colors',
-                        balance.balance === 0 && 'opacity-60'
+                        balance.balance === 0 &&
+                          !balance.isChild &&
+                          'opacity-60',
+                        balance.isChild &&
+                          'ml-4 border-l-2 border-l-blue-200 bg-blue-50/30'
                       )}
                     >
                       {/* Desktop Layout */}
                       <div className='hidden grid-cols-6 gap-2 md:grid'>
-                        <div className='max-w-32 truncate font-mono text-xs break-all'>
+                        <div className='flex max-w-48 items-center gap-2 truncate font-mono text-xs break-all'>
+                          {balance.isChild && (
+                            <span className='rounded bg-blue-100 px-1 py-0.5 text-[10px] font-bold text-blue-700 uppercase'>
+                              Child
+                            </span>
+                          )}
                           {balance.hashed_key}
                         </div>
                         <div className='text-right font-mono'>
-                          {formatBalance(balance.balance)}
+                          {balance.isChild ? (
+                            <span className='text-muted-foreground italic'>
+                              (Parent)
+                            </span>
+                          ) : (
+                            formatBalance(balance.balance)
+                          )}
                         </div>
                         <div className='text-right font-mono'>
                           {formatBalance(balance.total_spent)}
@@ -226,13 +273,20 @@ export function TemporaryBalances({
 
                       {/* Mobile Layout */}
                       <div className='space-y-3 md:hidden'>
-                        <div className='space-y-1'>
-                          <span className='text-muted-foreground text-xs font-medium'>
-                            Key
-                          </span>
-                          <div className='font-mono text-xs break-all'>
-                            {balance.hashed_key}
+                        <div className='flex items-center justify-between'>
+                          <div className='space-y-1'>
+                            <span className='text-muted-foreground text-xs font-medium'>
+                              {balance.isChild ? 'Child Key' : 'Key'}
+                            </span>
+                            <div className='font-mono text-xs break-all'>
+                              {balance.hashed_key}
+                            </div>
                           </div>
+                          {balance.isChild && (
+                            <span className='rounded bg-blue-100 px-1.5 py-0.5 text-[10px] font-bold text-blue-700 uppercase'>
+                              Child
+                            </span>
+                          )}
                         </div>
 
                         <div className='grid grid-cols-2 gap-3'>
@@ -241,7 +295,13 @@ export function TemporaryBalances({
                               Balance
                             </div>
                             <div className='truncate font-mono text-sm'>
-                              {formatBalance(balance.balance)}
+                              {balance.isChild ? (
+                                <span className='text-muted-foreground text-xs italic'>
+                                  (Uses Parent)
+                                </span>
+                              ) : (
+                                formatBalance(balance.balance)
+                              )}
                             </div>
                           </div>
                           <div className='space-y-1'>

ui/lib/api/services/admin.ts

@@ -926,6 +926,7 @@ export const TemporaryBalanceSchema = z.object({
   total_requests: z.number(),
   refund_address: z.string().nullable(),
   key_expiry_time: z.number().nullable(),
+  parent_key_hash: z.string().nullable().optional(),
 });
 
 export type TemporaryBalance = z.infer<typeof TemporaryBalanceSchema>;

ui/package.json

@@ -15,6 +15,7 @@
     "@dnd-kit/core": "^6.3.1",
     "@dnd-kit/modifiers": "^9.0.0",
     "@dnd-kit/sortable": "^10.0.0",
+    "@dnd-kit/utilities": "^3.2.2",
     "@hookform/resolvers": "^5.0.1",
     "@radix-ui/react-alert-dialog": "^1.1.10",
     "@radix-ui/react-avatar": "^1.1.6",
@@ -40,17 +41,17 @@
     "@radix-ui/react-toggle": "^1.1.6",
     "@radix-ui/react-toggle-group": "^1.1.6",
     "@radix-ui/react-tooltip": "^1.2.3",
-    "@tanstack/react-query": "^5.74.4",
+    "@tanstack/react-query": "^5.90.16",
     "@tanstack/react-table": "^8.21.3",
     "axios": "^1.13.2",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "cmdk": "^1.1.1",
-    "date-fns": "^3.6.0",
+    "date-fns": "^4.1.0",
     "embla-carousel-react": "^8.6.0",
     "input-otp": "^1.4.2",
-    "lucide-react": "^0.501.0",
-    "next": "15.3.1",
+    "lucide-react": "^0.562.0",
+    "next": "15.5.9",
     "next-themes": "^0.4.6",
     "qrcode": "^1.5.4",
     "qrcode.react": "^4.2.0",
@@ -67,21 +68,21 @@
     "zustand": "^5.0.3"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3",
-    "@tailwindcss/postcss": "^4",
-    "@tanstack/react-query-devtools": "^5.74.4",
+    "@eslint/eslintrc": "^3.3.3",
+    "@tailwindcss/postcss": "^4.1.18",
+    "@tanstack/react-query-devtools": "^5.91.2",
     "@types/node": "^20",
     "@types/qrcode": "^1.5.6",
     "@types/react": "^19",
     "@types/react-dom": "^19",
     "eslint": "^9.25.0",
-    "eslint-config-next": "15.3.1",
+    "eslint-config-next": "15.5.9",
     "eslint-config-prettier": "^10.1.2",
     "eslint-plugin-prettier": "^5.2.6",
     "eslint-plugin-react": "^7.37.5",
     "prettier": "^3.5.3",
     "prettier-plugin-tailwindcss": "^0.6.11",
-    "tailwindcss": "^4",
+    "tailwindcss": "^4.1.18",
     "typescript": "^5"
   }
 }

ui/tsconfig.json

@@ -22,6 +22,12 @@
       "@/*": ["./*"]
     }
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts",
+    ".next/dev/types/**/*.ts"
+  ],
   "exclude": ["node_modules"]
 }

uv.lock

@@ -1878,7 +1878,7 @@ wheels = [
 
 [[package]]
 name = "routstr"
-version = "0.2.1"
+version = "0.3.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiosqlite" },