mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-07-22 12:22:20 +00:00
Compare commits
27 Commits
completion
...
embedding-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
525476ccfa | ||
|
|
b54812cb04 | ||
|
|
ee508cbb3a | ||
|
|
0c61fdee07 | ||
|
|
b9418db31f | ||
|
|
71e7c2171b | ||
|
|
5b8e56f590 | ||
|
|
a6d0bd1a19 | ||
|
|
9594e9fb52 | ||
|
|
b01c7b2e56 | ||
|
|
dd4ed7541f | ||
|
|
cc42534a97 | ||
|
|
d203370f01 | ||
|
|
e39742c429 | ||
|
|
301dd81215 | ||
|
|
5416cefd87 | ||
|
|
43e97326e0 | ||
|
|
06770a0702 | ||
|
|
547365894d | ||
|
|
c0176a5274 | ||
|
|
329d22363f | ||
|
|
9438bc957f | ||
|
|
5d2219880d | ||
|
|
195da0c9da | ||
|
|
8df0c17bc3 | ||
|
|
7bc9ee0653 | ||
|
|
355f8601c1 |
37
migrations/versions/b9667ffc5701_alias_ids.py
Normal file
37
migrations/versions/b9667ffc5701_alias_ids.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""alias-ids
|
||||
|
||||
Revision ID: b9667ffc5701
|
||||
Revises: lightning_invoices
|
||||
Create Date: 2025-12-25 19:30:44.673350
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "b9667ffc5701"
|
||||
down_revision = "lightning_invoices"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic ###
|
||||
op.add_column(
|
||||
"models",
|
||||
sa.Column("canonical_slug", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"models",
|
||||
sa.Column("alias_ids", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
)
|
||||
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column("models", "alias_ids")
|
||||
op.drop_column("models", "canonical_slug")
|
||||
# ### end Alembic commands ###
|
||||
@@ -206,19 +206,20 @@ def create_model_mappings(
|
||||
alias: str, model: "Model", provider: "BaseUpstreamProvider"
|
||||
) -> None:
|
||||
"""Set alias to model/provider if not set or if new model is preferred."""
|
||||
existing_model = model_instances.get(alias)
|
||||
alias_lower = alias.lower()
|
||||
existing_model = model_instances.get(alias_lower)
|
||||
if not existing_model:
|
||||
# No existing mapping, set it
|
||||
model_instances[alias] = model
|
||||
provider_map[alias] = provider
|
||||
model_instances[alias_lower] = model
|
||||
provider_map[alias_lower] = provider
|
||||
else:
|
||||
# Check if candidate should replace existing
|
||||
existing_provider = provider_map[alias]
|
||||
existing_provider = provider_map[alias_lower]
|
||||
if should_prefer_model(
|
||||
model, provider, existing_model, existing_provider, alias
|
||||
):
|
||||
model_instances[alias] = model
|
||||
provider_map[alias] = provider
|
||||
model_instances[alias_lower] = model
|
||||
provider_map[alias_lower] = provider
|
||||
|
||||
def process_provider_models(
|
||||
upstream: "BaseUpstreamProvider", is_openrouter: bool = False
|
||||
|
||||
@@ -1493,20 +1493,8 @@ class ModelCreate(BaseModel):
|
||||
per_request_limits: dict[str, object] | None = None
|
||||
top_provider: dict[str, object] | None = None
|
||||
upstream_provider_id: int | None = None
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class ModelUpdate(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
description: str
|
||||
created: int
|
||||
context_length: int
|
||||
architecture: dict[str, object]
|
||||
pricing: dict[str, object]
|
||||
per_request_limits: dict[str, object] | None = None
|
||||
top_provider: dict[str, object] | None = None
|
||||
upstream_provider_id: int | None = None
|
||||
canonical_slug: str | None = None
|
||||
alias_ids: list[str] | None = None
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
@@ -2416,44 +2404,80 @@ async def admin_upstream_providers(request: Request) -> str:
|
||||
"/api/upstream-providers/{provider_id}/models",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
)
|
||||
async def create_provider_model(
|
||||
async def upsert_provider_model(
|
||||
provider_id: int, payload: ModelCreate
|
||||
) -> dict[str, object]:
|
||||
print(payload)
|
||||
logger.info(
|
||||
f"UPSERT_PROVIDER_MODEL called: provider_id={provider_id}, model_id={payload.id}"
|
||||
)
|
||||
async with create_session() as session:
|
||||
provider = await session.get(UpstreamProviderRow, provider_id)
|
||||
if not provider:
|
||||
raise HTTPException(status_code=404, detail="Provider not found")
|
||||
|
||||
exists = await session.get(ModelRow, (payload.id, provider_id))
|
||||
if exists:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Model with this ID already exists for this provider",
|
||||
)
|
||||
# Try to get existing model
|
||||
existing_row = await session.get(ModelRow, (payload.id, provider_id))
|
||||
|
||||
row = ModelRow(
|
||||
id=payload.id,
|
||||
name=payload.name,
|
||||
description=payload.description,
|
||||
created=int(payload.created),
|
||||
context_length=int(payload.context_length),
|
||||
architecture=json.dumps(payload.architecture),
|
||||
pricing=json.dumps(payload.pricing),
|
||||
sats_pricing=None,
|
||||
per_request_limits=(
|
||||
if existing_row:
|
||||
# Update existing model
|
||||
logger.info(f"Updating existing model: {payload.id}")
|
||||
existing_row.name = payload.name
|
||||
existing_row.description = payload.description
|
||||
existing_row.created = int(payload.created)
|
||||
existing_row.context_length = int(payload.context_length)
|
||||
existing_row.architecture = json.dumps(payload.architecture)
|
||||
existing_row.pricing = json.dumps(payload.pricing)
|
||||
existing_row.sats_pricing = None
|
||||
existing_row.per_request_limits = (
|
||||
json.dumps(payload.per_request_limits)
|
||||
if payload.per_request_limits is not None
|
||||
else None
|
||||
),
|
||||
top_provider=(
|
||||
)
|
||||
existing_row.top_provider = (
|
||||
json.dumps(payload.top_provider) if payload.top_provider else None
|
||||
),
|
||||
upstream_provider_id=provider_id,
|
||||
enabled=payload.enabled,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
)
|
||||
existing_row.canonical_slug = payload.canonical_slug
|
||||
existing_row.alias_ids = (
|
||||
json.dumps(payload.alias_ids) if payload.alias_ids else None
|
||||
)
|
||||
existing_row.enabled = payload.enabled
|
||||
|
||||
session.add(existing_row)
|
||||
await session.commit()
|
||||
await session.refresh(existing_row)
|
||||
row = existing_row
|
||||
|
||||
else:
|
||||
# Create new model
|
||||
logger.info(f"Creating new model: {payload.id}")
|
||||
row = ModelRow(
|
||||
id=payload.id,
|
||||
name=payload.name,
|
||||
description=payload.description,
|
||||
created=int(payload.created),
|
||||
context_length=int(payload.context_length),
|
||||
architecture=json.dumps(payload.architecture),
|
||||
pricing=json.dumps(payload.pricing),
|
||||
sats_pricing=None,
|
||||
per_request_limits=(
|
||||
json.dumps(payload.per_request_limits)
|
||||
if payload.per_request_limits is not None
|
||||
else None
|
||||
),
|
||||
top_provider=(
|
||||
json.dumps(payload.top_provider) if payload.top_provider else None
|
||||
),
|
||||
canonical_slug=payload.canonical_slug,
|
||||
alias_ids=(
|
||||
json.dumps(payload.alias_ids) if payload.alias_ids else None
|
||||
),
|
||||
upstream_provider_id=provider_id,
|
||||
enabled=payload.enabled,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
await refresh_model_maps()
|
||||
return _row_to_model(
|
||||
@@ -2461,6 +2485,20 @@ async def create_provider_model(
|
||||
).dict() # type: ignore
|
||||
|
||||
|
||||
@admin_router.patch(
|
||||
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
)
|
||||
async def update_provider_model_legacy(
|
||||
provider_id: int, model_id: str, payload: ModelCreate
|
||||
) -> dict[str, object]:
|
||||
"""Legacy PATCH endpoint - redirects to upsert POST endpoint for backward compatibility."""
|
||||
logger.info(
|
||||
f"LEGACY_PATCH_UPDATE called: provider_id={provider_id}, model_id={model_id}"
|
||||
)
|
||||
return await upsert_provider_model(provider_id, payload)
|
||||
|
||||
|
||||
@admin_router.get(
|
||||
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
@@ -2481,76 +2519,6 @@ async def get_provider_model(provider_id: int, model_id: str) -> dict[str, objec
|
||||
).dict() # type: ignore
|
||||
|
||||
|
||||
@admin_router.patch(
|
||||
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
)
|
||||
async def update_provider_model(
|
||||
provider_id: int, model_id: str, payload: ModelUpdate
|
||||
) -> dict[str, object]:
|
||||
if payload.id != model_id:
|
||||
raise HTTPException(status_code=400, detail="Path id does not match payload id")
|
||||
|
||||
async with create_session() as session:
|
||||
provider = await session.get(UpstreamProviderRow, provider_id)
|
||||
if not provider:
|
||||
raise HTTPException(status_code=404, detail="Provider not found")
|
||||
|
||||
row = await session.get(ModelRow, (model_id, provider_id))
|
||||
if not row:
|
||||
raise HTTPException(
|
||||
status_code=404, detail="Model not found for this provider"
|
||||
)
|
||||
|
||||
row.name = payload.name
|
||||
row.description = payload.description
|
||||
row.created = int(payload.created)
|
||||
row.context_length = int(payload.context_length)
|
||||
row.architecture = json.dumps(payload.architecture)
|
||||
row.pricing = json.dumps(payload.pricing)
|
||||
row.sats_pricing = None
|
||||
row.per_request_limits = (
|
||||
json.dumps(payload.per_request_limits)
|
||||
if payload.per_request_limits is not None
|
||||
else None
|
||||
)
|
||||
row.top_provider = (
|
||||
json.dumps(payload.top_provider) if payload.top_provider else None
|
||||
)
|
||||
was_disabled = not row.enabled
|
||||
row.enabled = payload.enabled
|
||||
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
|
||||
if was_disabled and payload.enabled:
|
||||
from ..payment.models import _cleanup_enabled_models_once
|
||||
|
||||
try:
|
||||
await _cleanup_enabled_models_once()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to run model cleanup after enabling: {e}",
|
||||
extra={"model_id": model_id, "error": str(e)},
|
||||
)
|
||||
|
||||
await refresh_model_maps()
|
||||
return _row_to_model(
|
||||
row, apply_provider_fee=True, provider_fee=provider.provider_fee
|
||||
).dict() # type: ignore
|
||||
|
||||
|
||||
@admin_router.put(
|
||||
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
)
|
||||
async def update_provider_model_put(
|
||||
provider_id: int, model_id: str, payload: ModelUpdate
|
||||
) -> dict[str, object]:
|
||||
return await update_provider_model(provider_id, model_id, payload)
|
||||
|
||||
|
||||
@admin_router.delete(
|
||||
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
|
||||
dependencies=[Depends(require_admin_api)],
|
||||
|
||||
@@ -68,6 +68,10 @@ class ModelRow(SQLModel, table=True): # type: ignore
|
||||
sats_pricing: str | None = Field(default=None)
|
||||
per_request_limits: str | None = Field(default=None)
|
||||
top_provider: str | None = Field(default=None)
|
||||
canonical_slug: str | None = Field(default=None, description="Canonical model slug")
|
||||
alias_ids: str | None = Field(
|
||||
default=None, description="JSON array of model alias IDs"
|
||||
)
|
||||
enabled: bool = Field(default=True, description="Whether this model is enabled")
|
||||
upstream_provider: "UpstreamProviderRow" = Relationship(back_populates="models")
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from ..balance import balance_router, deprecated_wallet_router
|
||||
from ..discovery import providers_cache_refresher, providers_router
|
||||
from ..nip91 import announce_provider
|
||||
from ..payment.models import (
|
||||
cleanup_enabled_models_periodically,
|
||||
models_router,
|
||||
update_sats_pricing,
|
||||
)
|
||||
@@ -49,7 +48,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
nip91_task = None
|
||||
providers_task = None
|
||||
models_refresh_task = None
|
||||
models_cleanup_task = None
|
||||
model_maps_refresh_task = None
|
||||
|
||||
try:
|
||||
@@ -86,7 +84,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
models_refresh_task = asyncio.create_task(
|
||||
refresh_upstreams_models_periodically(get_upstreams())
|
||||
)
|
||||
models_cleanup_task = asyncio.create_task(cleanup_enabled_models_periodically())
|
||||
model_maps_refresh_task = asyncio.create_task(refresh_model_maps_periodically())
|
||||
payout_task = asyncio.create_task(periodic_payout())
|
||||
if global_settings.nsec:
|
||||
@@ -125,8 +122,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
providers_task.cancel()
|
||||
if models_refresh_task is not None:
|
||||
models_refresh_task.cancel()
|
||||
if models_cleanup_task is not None:
|
||||
models_cleanup_task.cancel()
|
||||
if model_maps_refresh_task is not None:
|
||||
model_maps_refresh_task.cancel()
|
||||
|
||||
@@ -144,8 +139,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
tasks_to_wait.append(providers_task)
|
||||
if models_refresh_task is not None:
|
||||
tasks_to_wait.append(models_refresh_task)
|
||||
if models_cleanup_task is not None:
|
||||
tasks_to_wait.append(models_cleanup_task)
|
||||
if model_maps_refresh_task is not None:
|
||||
tasks_to_wait.append(model_maps_refresh_task)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from pydantic.v1 import BaseModel
|
||||
from ..core import get_logger
|
||||
from ..core.db import AsyncSession
|
||||
from ..core.settings import settings
|
||||
from .price import sats_usd_price
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -64,6 +65,56 @@ async def calculate_cost( # todo: can be sync
|
||||
)
|
||||
return cost_data
|
||||
|
||||
usage_data = response_data["usage"]
|
||||
|
||||
usd_cost = 0.0
|
||||
|
||||
# Prioritize cost_details.upstream_inference_cost
|
||||
if "cost_details" in usage_data:
|
||||
usd_cost = float(
|
||||
usage_data["cost_details"].get("upstream_inference_cost", 0) or 0
|
||||
)
|
||||
|
||||
# Fallback to cost field if upstream_inference_cost is 0
|
||||
if usd_cost == 0 and "cost" in usage_data:
|
||||
try:
|
||||
usd_cost = float(usage_data.get("cost", 0) or 0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if usd_cost > 0:
|
||||
try:
|
||||
sats_per_usd = 1.0 / sats_usd_price()
|
||||
cost_in_sats = usd_cost * sats_per_usd
|
||||
cost_in_msats = math.ceil(cost_in_sats * 1000)
|
||||
|
||||
logger.info(
|
||||
"Using cost from usage data/details",
|
||||
extra={
|
||||
"usd_cost": usd_cost,
|
||||
"cost_in_sats": cost_in_sats,
|
||||
"cost_in_msats": cost_in_msats,
|
||||
"model": response_data.get("model", "unknown"),
|
||||
},
|
||||
)
|
||||
|
||||
return CostData(
|
||||
base_msats=-1,
|
||||
input_msats=-1, # Cost field doesn't break down by token type
|
||||
output_msats=-1,
|
||||
total_msats=cost_in_msats,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Error calculating cost from usage data",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"usd_cost": usd_cost,
|
||||
"model": response_data.get("model", "unknown"),
|
||||
},
|
||||
)
|
||||
# Fall through to token-based calculation
|
||||
|
||||
MSATS_PER_1K_INPUT_TOKENS: float = (
|
||||
float(settings.fixed_per_1k_input_tokens) * 1000.0
|
||||
)
|
||||
@@ -129,10 +180,19 @@ async def calculate_cost( # todo: can be sync
|
||||
)
|
||||
return cost_data
|
||||
|
||||
input_tokens = response_data.get("usage", {}).get("prompt_tokens", 0)
|
||||
output_tokens = response_data.get("usage", {}).get("completion_tokens", 0)
|
||||
input_tokens = usage_data.get("prompt_tokens", 0)
|
||||
output_tokens = usage_data.get("completion_tokens", 0)
|
||||
|
||||
# added for response api
|
||||
input_tokens = (
|
||||
input_tokens if input_tokens != 0 else usage_data.get("input_tokens", 0)
|
||||
)
|
||||
output_tokens = (
|
||||
output_tokens if output_tokens != 0 else usage_data.get("output_tokens", 0)
|
||||
)
|
||||
|
||||
input_msats = round(input_tokens / 1000 * MSATS_PER_1K_INPUT_TOKENS, 3)
|
||||
|
||||
output_msats = round(output_tokens / 1000 * MSATS_PER_1K_OUTPUT_TOKENS, 3)
|
||||
token_based_cost = math.ceil(input_msats + output_msats)
|
||||
|
||||
|
||||
@@ -5,10 +5,9 @@ import random
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic.v1 import BaseModel
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..core.db import ModelRow, create_session, get_session
|
||||
from ..core.db import ModelRow, get_session
|
||||
from ..core.logging import get_logger
|
||||
from ..core.settings import settings
|
||||
from .price import sats_usd_price
|
||||
@@ -93,12 +92,32 @@ async def async_fetch_openrouter_models(source_filter: str | None = None) -> lis
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"{base_url}/models", timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
models_response, embeddings_response = await asyncio.gather(
|
||||
client.get(f"{base_url}/models", timeout=30),
|
||||
client.get(f"{base_url}/embeddings/models", timeout=30),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
def process_models_response(
|
||||
response: httpx.Response | BaseException,
|
||||
) -> list[dict]:
|
||||
if not isinstance(response, BaseException):
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return [
|
||||
model
|
||||
for model in data.get("data", [])
|
||||
if ":free" not in model.get("id", "").lower()
|
||||
]
|
||||
return []
|
||||
|
||||
models_data: list[dict] = []
|
||||
for model in data.get("data", []):
|
||||
models_data.extend(process_models_response(models_response))
|
||||
models_data.extend(process_models_response(embeddings_response))
|
||||
|
||||
# Apply source filter and exclusions
|
||||
filtered_models = []
|
||||
for model in models_data:
|
||||
model_id = model.get("id", "")
|
||||
|
||||
if source_filter:
|
||||
@@ -116,9 +135,9 @@ async def async_fetch_openrouter_models(source_filter: str | None = None) -> lis
|
||||
if not _has_valid_pricing(model):
|
||||
continue
|
||||
|
||||
models_data.append(model)
|
||||
filtered_models.append(model)
|
||||
|
||||
return models_data
|
||||
return filtered_models
|
||||
except Exception as e:
|
||||
logger.error(f"Error (async) fetching models from OpenRouter API: {e}")
|
||||
return []
|
||||
@@ -165,6 +184,7 @@ def _row_to_model(
|
||||
enabled=row.enabled,
|
||||
upstream_provider_id=row.upstream_provider_id,
|
||||
canonical_slug=getattr(row, "canonical_slug", None),
|
||||
alias_ids=json.loads(row.alias_ids) if row.alias_ids else None,
|
||||
)
|
||||
|
||||
if apply_provider_fee:
|
||||
@@ -413,114 +433,6 @@ async def update_sats_pricing() -> None:
|
||||
logger.error(f"Error updating sats pricing: {e}")
|
||||
|
||||
|
||||
async def cleanup_enabled_models_periodically() -> None:
|
||||
"""Background task to clean up enabled models that match upstream pricing.
|
||||
|
||||
When model is enabled (enabled=True), remove it from DB if it matches upstream pricing.
|
||||
Keep it in DB only if pricing differs from upstream or if it's disabled.
|
||||
"""
|
||||
interval = getattr(
|
||||
settings, "models_cleanup_interval_seconds", 300
|
||||
) # 5 minutes default
|
||||
if not interval or interval <= 0:
|
||||
return
|
||||
|
||||
while True:
|
||||
try:
|
||||
await _cleanup_enabled_models_once()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Error during enabled models cleanup",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
)
|
||||
|
||||
try:
|
||||
jitter = max(0.0, float(interval) * 0.1)
|
||||
await asyncio.sleep(interval + random.uniform(0, jitter))
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
|
||||
|
||||
async def _cleanup_enabled_models_once() -> None:
|
||||
"""Clean up enabled models that match upstream pricing."""
|
||||
from ..proxy import get_upstreams
|
||||
|
||||
async with create_session() as session:
|
||||
# Get all enabled models from DB
|
||||
result = await session.exec(
|
||||
select(ModelRow).where(
|
||||
ModelRow.enabled, # Only enabled models
|
||||
)
|
||||
)
|
||||
db_models = result.all()
|
||||
|
||||
if not db_models:
|
||||
return
|
||||
|
||||
upstreams = get_upstreams()
|
||||
models_to_remove = []
|
||||
|
||||
for db_model in db_models:
|
||||
# Find corresponding upstream model
|
||||
upstream_model = None
|
||||
for upstream in upstreams:
|
||||
upstream_model = upstream.get_cached_model_by_id(db_model.id)
|
||||
if upstream_model:
|
||||
break
|
||||
|
||||
if not upstream_model:
|
||||
continue
|
||||
|
||||
# Compare pricing to see if they match
|
||||
db_pricing = json.loads(db_model.pricing)
|
||||
upstream_pricing = upstream_model.pricing.dict()
|
||||
|
||||
# Check if pricing matches (with small tolerance for float comparison)
|
||||
pricing_matches = _pricing_matches(db_pricing, upstream_pricing)
|
||||
|
||||
if pricing_matches:
|
||||
models_to_remove.append(db_model)
|
||||
logger.info(
|
||||
f"Removing enabled model {db_model.id} - matches upstream pricing",
|
||||
extra={"model_id": db_model.id},
|
||||
)
|
||||
|
||||
# Remove models that match upstream pricing
|
||||
for model in models_to_remove:
|
||||
await session.delete(model)
|
||||
|
||||
if models_to_remove:
|
||||
await session.commit()
|
||||
logger.info(
|
||||
f"Cleaned up {len(models_to_remove)} enabled models that match upstream pricing"
|
||||
)
|
||||
|
||||
|
||||
def _pricing_matches(
|
||||
db_pricing: dict, upstream_pricing: dict, tolerance: float = 0.0
|
||||
) -> bool:
|
||||
"""Check if pricing dictionaries match within tolerance."""
|
||||
keys_to_compare = [
|
||||
"prompt",
|
||||
"completion",
|
||||
"request",
|
||||
"image",
|
||||
"web_search",
|
||||
"internal_reasoning",
|
||||
]
|
||||
|
||||
for key in keys_to_compare:
|
||||
db_val = int(float(db_pricing.get(key, 0.0)) * 1000000)
|
||||
upstream_val = int(float(upstream_pricing.get(key, 0.0)) * 1000000)
|
||||
|
||||
if abs(db_val - upstream_val) > tolerance:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@models_router.get("/v1/models")
|
||||
@models_router.get("/models", include_in_schema=False)
|
||||
async def models(session: AsyncSession = Depends(get_session)) -> dict:
|
||||
|
||||
@@ -65,12 +65,12 @@ def get_upstreams() -> list[BaseUpstreamProvider]:
|
||||
|
||||
def get_model_instance(model_id: str) -> Model | None:
|
||||
"""Get Model instance by ID from global cache."""
|
||||
return _model_instances.get(model_id)
|
||||
return _model_instances.get(model_id.lower())
|
||||
|
||||
|
||||
def get_provider_for_model(model_id: str) -> BaseUpstreamProvider | None:
|
||||
"""Get UpstreamProvider for model ID from global cache."""
|
||||
return _provider_map.get(model_id)
|
||||
return _provider_map.get(model_id.lower())
|
||||
|
||||
|
||||
def get_unique_models() -> list[Model]:
|
||||
|
||||
@@ -734,51 +734,53 @@ class BaseUpstreamProvider:
|
||||
await client.aclose()
|
||||
return mapped_error
|
||||
|
||||
if path.endswith("chat/completions"):
|
||||
client_wants_streaming = False
|
||||
if request_body:
|
||||
try:
|
||||
request_data = json.loads(request_body)
|
||||
client_wants_streaming = request_data.get("stream", False)
|
||||
logger.debug(
|
||||
"Chat completion request analysis",
|
||||
extra={
|
||||
"client_wants_streaming": client_wants_streaming,
|
||||
"model": request_data.get("model", "unknown"),
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
},
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(
|
||||
"Failed to parse request body JSON for streaming detection"
|
||||
)
|
||||
if path.endswith("chat/completions") or path.endswith("embeddings"):
|
||||
if path.endswith("chat/completions"):
|
||||
client_wants_streaming = False
|
||||
if request_body:
|
||||
try:
|
||||
request_data = json.loads(request_body)
|
||||
client_wants_streaming = request_data.get("stream", False)
|
||||
logger.debug(
|
||||
"Chat completion request analysis",
|
||||
extra={
|
||||
"client_wants_streaming": client_wants_streaming,
|
||||
"model": request_data.get("model", "unknown"),
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
},
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(
|
||||
"Failed to parse request body JSON for streaming detection"
|
||||
)
|
||||
|
||||
content_type = response.headers.get("content-type", "")
|
||||
upstream_is_streaming = "text/event-stream" in content_type
|
||||
is_streaming = client_wants_streaming and upstream_is_streaming
|
||||
content_type = response.headers.get("content-type", "")
|
||||
upstream_is_streaming = "text/event-stream" in content_type
|
||||
is_streaming = client_wants_streaming and upstream_is_streaming
|
||||
|
||||
logger.debug(
|
||||
"Response type analysis",
|
||||
extra={
|
||||
"is_streaming": is_streaming,
|
||||
"client_wants_streaming": client_wants_streaming,
|
||||
"upstream_is_streaming": upstream_is_streaming,
|
||||
"content_type": content_type,
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
},
|
||||
)
|
||||
|
||||
if is_streaming and response.status_code == 200:
|
||||
result = await self.handle_streaming_chat_completion(
|
||||
response, key, max_cost_for_model
|
||||
logger.debug(
|
||||
"Response type analysis",
|
||||
extra={
|
||||
"is_streaming": is_streaming,
|
||||
"client_wants_streaming": client_wants_streaming,
|
||||
"upstream_is_streaming": upstream_is_streaming,
|
||||
"content_type": content_type,
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
},
|
||||
)
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
background_tasks.add_task(client.aclose)
|
||||
result.background = background_tasks
|
||||
return result
|
||||
|
||||
elif response.status_code == 200:
|
||||
if is_streaming and response.status_code == 200:
|
||||
result = await self.handle_streaming_chat_completion(
|
||||
response, key, max_cost_for_model
|
||||
)
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
background_tasks.add_task(client.aclose)
|
||||
result.background = background_tasks
|
||||
return result
|
||||
|
||||
# Handle both non-streaming chat completions and embeddings
|
||||
if response.status_code == 200:
|
||||
try:
|
||||
return await self.handle_non_streaming_chat_completion(
|
||||
response, key, session, max_cost_for_model
|
||||
@@ -1519,9 +1521,9 @@ class BaseUpstreamProvider:
|
||||
error_response.headers["X-Cashu"] = refund_token
|
||||
return error_response
|
||||
|
||||
if path.endswith("chat/completions"):
|
||||
if path.endswith("chat/completions") or path.endswith("embeddings"):
|
||||
logger.debug(
|
||||
"Processing chat completion response",
|
||||
"Processing completion/embeddings response",
|
||||
extra={"path": path, "amount": amount, "unit": unit},
|
||||
)
|
||||
|
||||
@@ -1770,15 +1772,32 @@ class BaseUpstreamProvider:
|
||||
async def _fetch_openrouter_models(self) -> list[dict]:
|
||||
"""Fetch models from OpenRouter API."""
|
||||
url = "https://openrouter.ai/api/v1/models"
|
||||
embeddings_url = "https://openrouter.ai/api/v1/embeddings/models"
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
models = response.json()
|
||||
return [
|
||||
model
|
||||
for model in models.get("data", [])
|
||||
if ":free" not in model.get("id", "").lower()
|
||||
]
|
||||
models_response, embeddings_response = await asyncio.gather(
|
||||
client.get(url), client.get(embeddings_url), return_exceptions=True
|
||||
)
|
||||
|
||||
all_models = []
|
||||
|
||||
def process_models_response(
|
||||
response: httpx.Response | BaseException,
|
||||
) -> list[dict]:
|
||||
if not isinstance(response, BaseException):
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return [
|
||||
model
|
||||
for model in data.get("data", [])
|
||||
if ":free" not in model.get("id", "").lower()
|
||||
]
|
||||
return []
|
||||
|
||||
all_models.extend(process_models_response(models_response))
|
||||
all_models.extend(process_models_response(embeddings_response))
|
||||
|
||||
return all_models
|
||||
|
||||
async def _fetch_provider_models(self) -> dict:
|
||||
"""Fetch models from provider's API."""
|
||||
|
||||
@@ -50,7 +50,13 @@ class OpenRouterUpstreamProvider(BaseUpstreamProvider):
|
||||
async def fetch_models(self) -> list[Model]:
|
||||
"""Fetch all OpenRouter models."""
|
||||
models_data = await async_fetch_openrouter_models()
|
||||
return [Model(**model) for model in models_data] # type: ignore
|
||||
models = [Model(**model) for model in models_data] # type: ignore
|
||||
# manual alias for openai/text-embedding-ada-002 due to openrouter api bug
|
||||
for model in models:
|
||||
if model.id == "openai/text-embedding-ada-002":
|
||||
model.alias_ids = ["text-embedding-ada-002-v2"]
|
||||
break
|
||||
return models
|
||||
|
||||
async def get_balance(self) -> float | None:
|
||||
"""Get the current account balance from OpenRouter.
|
||||
|
||||
97
tests/integration/test_embeddings.py
Normal file
97
tests/integration/test_embeddings.py
Normal file
@@ -0,0 +1,97 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_embeddings_endpoint(authenticated_client: AsyncClient) -> None:
|
||||
"""Test the embeddings endpoint proxy functionality"""
|
||||
|
||||
test_payload = {
|
||||
"model": "text-embedding-ada-002",
|
||||
"input": "The quick brown fox",
|
||||
}
|
||||
|
||||
mock_response_data = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"object": "embedding", "embedding": [0.0023, -0.0012, 0.0045], "index": 0}
|
||||
],
|
||||
"model": "text-embedding-ada-002",
|
||||
"usage": {"prompt_tokens": 5, "total_tokens": 5},
|
||||
}
|
||||
|
||||
with patch("httpx.AsyncClient.send") as mock_send:
|
||||
# Create a proper async generator for iter_bytes
|
||||
async def mock_iter_bytes(*args: Any, **kwargs: Any) -> Any:
|
||||
yield json.dumps(mock_response_data).encode()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {"content-type": "application/json"}
|
||||
mock_response.text = json.dumps(mock_response_data)
|
||||
# Use MagicMock for synchronous .json() method
|
||||
mock_response.json = MagicMock(return_value=mock_response_data)
|
||||
mock_response.iter_bytes = mock_iter_bytes
|
||||
mock_response.aiter_bytes = mock_iter_bytes
|
||||
mock_send.return_value = mock_response
|
||||
|
||||
# Make POST request to embeddings endpoint
|
||||
response = await authenticated_client.post("/v1/embeddings", json=test_payload)
|
||||
|
||||
assert response.status_code == 200
|
||||
response_data = response.json()
|
||||
assert response_data["object"] == "list"
|
||||
assert len(response_data["data"]) == 1
|
||||
assert response_data["data"][0]["object"] == "embedding"
|
||||
|
||||
# Verify request was forwarded
|
||||
mock_send.assert_called_once()
|
||||
forwarded_request = mock_send.call_args[0][0]
|
||||
# Verify the path ends with embeddings
|
||||
# Note: forwarded path might be full URL
|
||||
assert str(forwarded_request.url).endswith("embeddings")
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_case_insensitivity(authenticated_client: AsyncClient) -> None:
|
||||
"""Test that model lookups are case insensitive"""
|
||||
|
||||
# We'll use a mixed-case model ID that should match the lowercase one in the system
|
||||
# We assume 'gpt-3.5-turbo' is available in the mock env/database
|
||||
|
||||
test_payload = {
|
||||
"model": "GPT-3.5-TURBO",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
}
|
||||
|
||||
with patch("httpx.AsyncClient.send") as mock_send:
|
||||
mock_response_data = {
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"choices": [{"message": {"content": "Hi"}}],
|
||||
"usage": {"total_tokens": 10},
|
||||
}
|
||||
|
||||
async def mock_iter_bytes(*args: Any, **kwargs: Any) -> Any:
|
||||
yield json.dumps(mock_response_data).encode()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {"content-type": "application/json"}
|
||||
mock_response.text = json.dumps(mock_response_data)
|
||||
mock_response.json = MagicMock(return_value=mock_response_data)
|
||||
mock_response.iter_bytes = mock_iter_bytes
|
||||
mock_response.aiter_bytes = mock_iter_bytes
|
||||
mock_send.return_value = mock_response
|
||||
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions", json=test_payload
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -41,6 +41,10 @@ export function ModelSearchFilter({
|
||||
model.name.toLowerCase().includes(query) ||
|
||||
model.full_name.toLowerCase().includes(query) ||
|
||||
model.provider.toLowerCase().includes(query) ||
|
||||
(model.alias_ids &&
|
||||
model.alias_ids.some((alias) =>
|
||||
alias.toLowerCase().includes(query)
|
||||
)) ||
|
||||
(model.description &&
|
||||
model.description.toLowerCase().includes(query)) ||
|
||||
model.modelType.toLowerCase().includes(query)
|
||||
|
||||
@@ -542,6 +542,7 @@ export function ModelSelector({
|
||||
top_provider: null,
|
||||
upstream_provider_id: providerId,
|
||||
enabled: model.isEnabled,
|
||||
alias_ids: model.alias_ids || null,
|
||||
};
|
||||
|
||||
setModelDialogState({
|
||||
@@ -585,6 +586,7 @@ export function ModelSelector({
|
||||
top_provider: null,
|
||||
upstream_provider_id: providerId,
|
||||
enabled: model.isEnabled,
|
||||
alias_ids: model.alias_ids || null,
|
||||
};
|
||||
|
||||
setModelDialogState({
|
||||
|
||||
@@ -31,6 +31,7 @@ export const ModelSchema = z.object({
|
||||
// API key type indicators
|
||||
has_own_api_key: z.boolean(),
|
||||
api_key_type: z.string(), // "individual" or "group"
|
||||
alias_ids: z.array(z.string()).nullable().optional(),
|
||||
});
|
||||
|
||||
// Schema for a model with additional provider-specific settings
|
||||
|
||||
@@ -121,6 +121,7 @@ export interface AdminModelAsModel {
|
||||
soft_deleted?: boolean;
|
||||
has_own_api_key: boolean;
|
||||
api_key_type: string;
|
||||
alias_ids?: string[] | null;
|
||||
}
|
||||
|
||||
export interface AdminModelGroup {
|
||||
@@ -207,6 +208,7 @@ export class AdminService {
|
||||
soft_deleted: !adminModel.enabled,
|
||||
has_own_api_key: false,
|
||||
api_key_type: 'group',
|
||||
alias_ids: adminModel.alias_ids,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -354,8 +356,9 @@ export class AdminService {
|
||||
original: data.pricing,
|
||||
converted: payload.pricing,
|
||||
});
|
||||
const model = await apiClient.patch<AdminModel>(
|
||||
`/admin/api/upstream-providers/${providerId}/models/${encodeURIComponent(modelId)}`,
|
||||
// Use the same POST endpoint for both create and update (upsert)
|
||||
const model = await apiClient.post<AdminModel>(
|
||||
`/admin/api/upstream-providers/${providerId}/models`,
|
||||
payload
|
||||
);
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user