Merge branch 'fix-x-cashu-misscalculation' into refresh-models

# Conflicts:
#	routstr/core/admin.py

routstr/core/admin.py

@@ -9,6 +9,7 @@ from pydantic import BaseModel
 from sqlmodel import select
 
 from ..payment.models import Model, get_model_by_id, list_models
+from ..payment.models import sync_models_with_api
 from ..wallet import (
     fetch_all_balances,
     get_proofs_per_mint_and_unit,
@@ -501,6 +502,64 @@ async def dashboard(request: Request) -> str:
                     window.location.href = `/admin/logs/${requestId}`;
                 }
 
+                function openSyncModelsModal() {
+                    const modal = document.getElementById('sync-models-modal');
+                    modal.style.display = 'block';
+                }
+
+                function closeSyncModelsModal() {
+                    const modal = document.getElementById('sync-models-modal');
+                    modal.style.display = 'none';
+                }
+
+                async function syncModels() {
+                    const deleteRemoved = document.getElementById('delete-removed-models').checked;
+                    const button = document.getElementById('sync-models-btn');
+                    const resultDiv = document.getElementById('sync-models-result');
+                    
+                    button.disabled = true;
+                    button.textContent = 'Syncing...';
+                    resultDiv.style.display = 'none';
+                    
+                    try {
+                        const response = await fetch('/admin/api/sync_models', {
+                            method: 'POST',
+                            headers: {
+                                'Content-Type': 'application/json',
+                            },
+                            credentials: 'same-origin',
+                            body: JSON.stringify({
+                                delete_removed: deleteRemoved
+                            })
+                        });
+                        
+                        if (response.ok) {
+                            const data = await response.json();
+                            resultDiv.innerHTML = `<strong>✅ Success!</strong><br>${data.message}`;
+                            resultDiv.style.display = 'block';
+                            resultDiv.style.backgroundColor = '#d4edda';
+                            resultDiv.style.borderColor = '#c3e6cb';
+                            resultDiv.style.color = '#155724';
+                        } else {
+                            const errorData = await response.json();
+                            resultDiv.innerHTML = `<strong>❌ Error:</strong><br>${errorData.detail || 'Unknown error'}`;
+                            resultDiv.style.display = 'block';
+                            resultDiv.style.backgroundColor = '#f8d7da';
+                            resultDiv.style.borderColor = '#f5c6cb';
+                            resultDiv.style.color = '#721c24';
+                        }
+                    } catch (error) {
+                        resultDiv.innerHTML = `<strong>❌ Error:</strong><br>${error.message}`;
+                        resultDiv.style.display = 'block';
+                        resultDiv.style.backgroundColor = '#f8d7da';
+                        resultDiv.style.borderColor = '#f5c6cb';
+                        resultDiv.style.color = '#721c24';
+                    } finally {
+                        button.disabled = false;
+                        button.textContent = 'Sync Models';
+                    }
+                }
+
                 async function openSettingsModal() {
                     const modal = document.getElementById('settings-modal');
                     const textarea = document.getElementById('settings-json');
@@ -582,12 +641,15 @@ async def dashboard(request: Request) -> str:
                     const withdrawModal = document.getElementById('withdraw-modal');
                     const investigateModal = document.getElementById('investigate-modal');
                     const settingsModal = document.getElementById('settings-modal');
+                    const syncModelsModal = document.getElementById('sync-models-modal');
                     if (event.target == withdrawModal) {
                         closeWithdrawModal();
                     } else if (event.target == investigateModal) {
                         closeInvestigateModal();
                     } else if (event.target == settingsModal) {
                         closeSettingsModal();
+                    } else if (event.target == syncModelsModal) {
+                        closeSyncModelsModal();
                     }
                 }
             </script>
@@ -619,6 +681,29 @@ async def dashboard(request: Request) -> str:
             <button onclick="openSettingsModal()">
                 ⚙️ Settings
             </button>
+            <button onclick="openSyncModelsModal()">
+                🔄 Sync Models
+            </button>
+            
+            <div id="sync-models-modal" class="modal">
+                <div class="modal-content">
+                    <span class="close" onclick="closeSyncModelsModal()">&times;</span>
+                    <h3>Sync Models from API</h3>
+                    <p>This will fetch the latest models from OpenRouter and update the database.</p>
+                    <div style="margin: 15px 0;">
+                        <label style="display: flex; align-items: center; gap: 10px; cursor: pointer;">
+                            <input type="checkbox" id="delete-removed-models" style="width: auto; margin: 0;">
+                            <span>Delete models that are no longer available</span>
+                        </label>
+                        <p style="font-size: 0.85rem; color: #718096; margin-top: 5px; margin-left: 30px;">
+                            ⚠️ Warning: This will permanently remove models that are no longer in the API.
+                        </p>
+                    </div>
+                    <div id="sync-models-result" style="display: none; padding: 12px; border-radius: 6px; margin: 15px 0; border: 1px solid;"></div>
+                    <button id="sync-models-btn" onclick="syncModels()">🔄 Sync Models</button>
+                    <button onclick="closeSyncModelsModal()" style="background-color: #718096;">Cancel</button>
+                </div>
+            </div>
             
             <div id="withdraw-modal" class="modal">
                 <div class="modal-content">
@@ -808,7 +893,6 @@ async def view_logs(request: Request, request_id: str) -> str:
 async def withdraw(
     request: Request, withdraw_request: WithdrawRequest
 ) -> dict[str, str]:
-    # Get wallet and check balance
     from .settings import settings as global_settings
 
     wallet = await get_wallet(
@@ -836,6 +920,35 @@ async def withdraw(
     )
     return {"token": token}
 
+class SyncModelsRequest(BaseModel):
+    delete_removed: bool = False
+
+
+@admin_router.post("/api/sync_models", dependencies=[Depends(require_admin_api)])
+async def sync_models(request: Request, sync_request: SyncModelsRequest) -> dict:
+    try:
+        src = settings.source or None
+        source_filter = src if src and src.strip() else None
+    except Exception:
+        source_filter = None
+
+    logger.info(
+        "Manual models sync triggered",
+        extra={"delete_removed": sync_request.delete_removed},
+    )
+
+    counts = await sync_models_with_api(
+        source_filter=source_filter, delete_removed=sync_request.delete_removed
+    )
+
+    return {
+        "success": True,
+        "inserted": counts["inserted"],
+        "updated": counts["updated"],
+        "deleted": counts["deleted"],
+        "message": f"Synced: {counts['inserted']} inserted, {counts['updated']} updated, {counts['deleted']} deleted",
+    }
+
 
 DASHBOARD_MODELS_JS: str = """<!--html-->
     <script>

routstr/core/main.py

@@ -67,8 +67,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
 
         await ensure_models_bootstrapped()
         pricing_task = asyncio.create_task(update_sats_pricing())
-        if global_settings.models_refresh_interval_seconds > 0:
-            models_refresh_task = asyncio.create_task(refresh_models_periodically())
+        models_refresh_task = asyncio.create_task(refresh_models_periodically())
         payout_task = asyncio.create_task(periodic_payout())
         nip91_task = asyncio.create_task(announce_provider())
         providers_task = asyncio.create_task(providers_cache_refresher())

routstr/core/settings.py

@@ -64,10 +64,11 @@ class Settings(BaseSettings):
         default=120, env="PRICING_REFRESH_INTERVAL_SECONDS"
     )
     models_refresh_interval_seconds: int = Field(
-        default=0, env="MODELS_REFRESH_INTERVAL_SECONDS"
+        default=30, env="MODELS_REFRESH_INTERVAL_SECONDS"
     )
     enable_pricing_refresh: bool = Field(default=True, env="ENABLE_PRICING_REFRESH")
     enable_models_refresh: bool = Field(default=True, env="ENABLE_MODELS_REFRESH")
+    delete_removed_models: bool = Field(default=False, env="DELETE_REMOVED_MODELS")
     refund_cache_ttl_seconds: int = Field(default=3600, env="REFUND_CACHE_TTL_SECONDS")
 
     # Logging

routstr/payment/helpers.py

@@ -11,7 +11,7 @@ from ..core import get_logger
 from ..core.db import ModelRow
 from ..core.settings import settings
 from ..wallet import deserialize_token_from_string
-from .models import Pricing
+from .models import Pricing, compute_effective_max_cost_msats
 
 logger = get_logger(__name__)
 
@@ -72,7 +72,8 @@ def check_token_balance(headers: dict, body: dict, max_cost_for_model: int) -> N
         token_obj.amount if token_obj.unit == "msat" else token_obj.amount * 1000
     )
 
-    if max_cost_for_model > amount_msat:
+    fee_buffer = 60
+    if max_cost_for_model > amount_msat + fee_buffer:
         raise HTTPException(
             status_code=413,
             detail={
@@ -133,16 +134,17 @@ async def get_max_cost_for_model(
     row = await session.get(ModelRow, model)
     if row and row.sats_pricing:
         try:
-            sats = Pricing(**json.loads(row.sats_pricing))  # type: ignore
-            max_cost = sats.max_cost * 1000 * (1 - settings.tolerance_percentage / 100)
-            logger.debug(
-                "Found model-specific max cost",
-                extra={"model": model, "max_cost_msats": max_cost},
-            )
-            calculated_msats = int(max_cost)
-            return max(settings.min_request_msat, calculated_msats)
+            sats_dict = json.loads(row.sats_pricing)
         except Exception:
-            pass
+            sats_dict = None
+        if isinstance(sats_dict, dict):
+            effective_msats = compute_effective_max_cost_msats(sats_dict)
+            if effective_msats > 0:
+                logger.debug(
+                    "Found model-specific max cost",
+                    extra={"model": model, "max_cost_msats": effective_msats},
+                )
+                return effective_msats
 
     logger.warning(
         "Model pricing not found, using fixed cost",
@@ -201,6 +203,8 @@ async def calculate_discounted_max_cost(
             else:
                 adjusted = adjusted + math.ceil(-estimated_completion_delta_sats * 1000)
 
+    adjusted = min(max_cost_for_model, adjusted)
+
     logger.debug(
         "Discounted max cost computed",
         extra={

routstr/payment/models.py

@@ -58,6 +58,39 @@ class Model(BaseModel):
     top_provider: TopProvider | None = None
 
 
+def compute_effective_max_cost_msats(
+    pricing: Pricing | dict | None,
+) -> int:
+    if pricing is None:
+        try:
+            return max(1, int(settings.min_request_msat))
+        except Exception:
+            return 1
+
+    pricing_obj = (
+        pricing if isinstance(pricing, Pricing) else Pricing.parse_obj(pricing)  # type: ignore[arg-type]
+    )
+
+    try:
+        tolerance = float(getattr(settings, "tolerance_percentage", 0.0))
+    except Exception:
+        tolerance = 0.0
+
+    tolerance_factor = max(0.0, 1.0 - tolerance / 100.0)
+
+    try:
+        min_request_msat = max(1, int(settings.min_request_msat))
+    except Exception:
+        min_request_msat = 1
+
+    base_msats = int(float(pricing_obj.max_cost or 0.0) * 1000.0 * tolerance_factor)
+
+    if base_msats <= 0:
+        return min_request_msat
+
+    return max(min_request_msat, base_msats)
+
+
 def fetch_openrouter_models(source_filter: str | None = None) -> list[dict]:
     """Fetches model information from OpenRouter API."""
     base_url = "https://openrouter.ai/api/v1"
@@ -177,6 +210,10 @@ def _row_to_model(row: ModelRow) -> Model:
     except Exception:
         pass
 
+    if isinstance(sats_pricing, dict):
+        effective_msats = compute_effective_max_cost_msats(sats_pricing)
+        sats_pricing["max_cost"] = effective_msats / 1000.0
+
     return Model(
         id=row.id,
         name=row.name,
@@ -400,11 +437,76 @@ async def update_sats_pricing() -> None:
             break
 
 
+async def sync_models_with_api(
+    source_filter: str | None = None, delete_removed: bool = False
+) -> dict[str, int]:
+    """Fetch models from OpenRouter and sync with database.
+
+    Args:
+        source_filter: Optional source filter (e.g., 'anthropic')
+        delete_removed: If True, delete models that no longer exist in API
+
+    Returns:
+        Dict with counts: inserted, updated, deleted
+    """
+    models = fetch_openrouter_models(source_filter=source_filter)
+    if not models:
+        return {"inserted": 0, "updated": 0, "deleted": 0}
+
+    async with create_session() as s:
+        result = await s.exec(select(ModelRow))  # type: ignore
+        existing_rows = {row.id: row for row in result.all()}
+
+        fetched_ids = set()
+        inserted = 0
+        updated = 0
+
+        for m in models:
+            try:
+                model = Model(**m)  # type: ignore
+            except Exception:
+                continue
+
+            fetched_ids.add(model.id)
+            payload = _model_to_row_payload(model)
+
+            if model.id not in existing_rows:
+                try:
+                    s.add(ModelRow(**payload))  # type: ignore
+                    inserted += 1
+                except Exception:
+                    pass
+            else:
+                existing_row = existing_rows[model.id]
+                changed = False
+                for key, value in payload.items():
+                    if getattr(existing_row, key) != value:
+                        setattr(existing_row, key, value)
+                        changed = True
+                if changed:
+                    s.add(existing_row)
+                    updated += 1
+
+        deleted = 0
+        if delete_removed:
+            for existing_id in existing_rows:
+                if existing_id not in fetched_ids:
+                    row_to_delete = existing_rows[existing_id]
+                    await s.delete(row_to_delete)
+                    deleted += 1
+
+        if inserted or updated or deleted:
+            await s.commit()
+
+        return {"inserted": inserted, "updated": updated, "deleted": deleted}
+
+
 async def refresh_models_periodically() -> None:
-    """Background task: periodically fetch OpenRouter models and insert new ones.
+    """Background task: periodically fetch OpenRouter models and sync with database.
 
     - Respects optional SOURCE filter from settings
-    - Does not overwrite existing rows
+    - Updates existing models with new information
+    - Optionally deletes models no longer in API (if settings.delete_removed_models)
     - Sleeps according to settings.models_refresh_interval_seconds; disabled when 0
     """
     interval = getattr(settings, "models_refresh_interval_seconds", 0)
@@ -429,33 +531,24 @@ async def refresh_models_periodically() -> None:
             except Exception:
                 source_filter = None
 
-            models = fetch_openrouter_models(source_filter=source_filter)
-            if not models:
-                await asyncio.sleep(interval)
-                continue
+            try:
+                delete_removed = getattr(settings, "delete_removed_models", False)
+            except Exception:
+                delete_removed = False
 
-            async with create_session() as s:
-                result = await s.exec(select(ModelRow.id))  # type: ignore
-                existing_ids = {
-                    row[0] if isinstance(row, tuple) else row for row in result.all()
-                }
-                inserted = 0
-                for m in models:
-                    try:
-                        model = Model(**m)  # type: ignore
-                    except Exception:
-                        continue
-                    if model.id in existing_ids:
-                        continue
-                    payload = _model_to_row_payload(model)
-                    try:
-                        s.add(ModelRow(**payload))  # type: ignore
-                    except Exception:
-                        pass
-                    inserted += 1
-                if inserted:
-                    await s.commit()
-                    logger.info(f"Inserted {inserted} new models from OpenRouter")
+            counts = await sync_models_with_api(
+                source_filter=source_filter, delete_removed=delete_removed
+            )
+
+            if counts["inserted"] or counts["updated"] or counts["deleted"]:
+                logger.info(
+                    "Models synced",
+                    extra={
+                        "inserted": counts["inserted"],
+                        "updated": counts["updated"],
+                        "deleted": counts["deleted"],
+                    },
+                )
         except asyncio.CancelledError:
             break
         except Exception as e:

tests/unit/test_payment_helpers.py

@@ -1,12 +1,20 @@
 import os
 from unittest.mock import AsyncMock, Mock, patch
 
+import pytest
+from fastapi import HTTPException
+
 # Set required env vars before importing
 os.environ["UPSTREAM_BASE_URL"] = "http://test"
 os.environ["UPSTREAM_API_KEY"] = "test"
 
 from routstr.core.settings import settings  # noqa: E402
-from routstr.payment.helpers import get_max_cost_for_model  # noqa: E402
+from routstr.payment.helpers import (  # noqa: E402
+    calculate_discounted_max_cost,
+    check_token_balance,
+    get_max_cost_for_model,
+)
+from routstr.payment.models import Pricing  # noqa: E402
 
 
 async def test_get_max_cost_for_model_known() -> None:
@@ -73,3 +81,85 @@ async def test_get_max_cost_for_model_tolerance() -> None:
         with patch.object(settings, "tolerance_percentage", 10):
             cost = await get_max_cost_for_model("gpt-4", session=mock_session)
             assert cost == 450000  # 500 sats * 1000 * 0.9 = 450000
+
+
+async def test_calculate_discounted_max_cost_no_session() -> None:
+    with patch.object(settings, "fixed_pricing", False):
+        result = await calculate_discounted_max_cost(123, {}, session=None)
+        assert result == 123
+
+
+async def test_calculate_discounted_max_cost_clamped(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    mock_session = AsyncMock()
+    mock_exec_result = Mock()
+    mock_exec_result.all = Mock(return_value=[("gpt-4",)])
+    mock_session.exec.return_value = mock_exec_result
+
+    pricing = {
+        "prompt": 0.0,
+        "completion": 0.0,
+        "request": 0.0,
+        "image": 0.0,
+        "web_search": 0.0,
+        "internal_reasoning": 0.0,
+        "max_prompt_cost": 100.0,
+        "max_completion_cost": 200.0,
+        "max_cost": 300.0,
+    }
+
+    async def mock_get_model_cost_info(*args, **kwargs):  # type: ignore
+        return Pricing(**pricing)
+
+    monkeypatch.setattr(
+        "routstr.payment.helpers.get_model_cost_info", mock_get_model_cost_info
+    )
+
+    with patch.object(settings, "fixed_pricing", False):
+        with patch.object(settings, "tolerance_percentage", 0):
+            result = await calculate_discounted_max_cost(
+                320000,
+                {"model": "gpt-4", "max_tokens": 10, "messages": ["one"]},
+                session=mock_session,
+            )
+            assert 0 <= result <= 320000
+
+
+async def test_check_token_balance_with_fee_buffer(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    class Token:
+        unit = "msat"
+        amount = 320400
+
+    def mock_deserialize(_value: str) -> Token:
+        return Token()
+
+    monkeypatch.setattr(
+        "routstr.payment.helpers.deserialize_token_from_string", mock_deserialize
+    )
+    headers = {"x-cashu": "token"}
+    body = {"model": "gpt-4"}
+    check_token_balance(headers, body, max_cost_for_model=320450)
+
+
+def test_check_token_balance_insufficient(monkeypatch: pytest.MonkeyPatch) -> None:
+    class Token:
+        unit = "msat"
+        amount = 320200
+
+    def mock_deserialize(_value: str) -> Token:
+        return Token()
+
+    monkeypatch.setattr(
+        "routstr.payment.helpers.deserialize_token_from_string", mock_deserialize
+    )
+    headers = {"x-cashu": "token"}
+    body = {"model": "gpt-4"}
+    with pytest.raises(HTTPException) as exc:
+        check_token_balance(headers, body, max_cost_for_model=320450)
+    assert exc.value.status_code == 413
+    detail = exc.value.detail  # type: ignore[assignment]
+    assert isinstance(detail, dict)
+    assert detail.get("type") == "minimum_balance_required"