Compare commits

...

27 Commits

Author SHA1 Message Date
Shroominic
525476ccfa fix alias not displaying correctly 2025-12-25 22:34:52 +01:00
Shroominic
b54812cb04 fix: alias_id not displayed correctly by dashboard 2025-12-25 22:28:26 +01:00
Shroominic
ee508cbb3a rm model cleanup 2025-12-25 21:50:09 +01:00
Shroominic
0c61fdee07 ruff format 2025-12-25 21:48:20 +01:00
Shroominic
b9418db31f rm model cleanup task 2025-12-25 21:47:17 +01:00
Shroominic
71e7c2171b add missing migration 2025-12-25 19:36:26 +01:00
9qeklajc
5b8e56f590 Merge branch 'v0.2.1' into embedding-with-aliases
# Conflicts:
#	routstr/upstream/base.py
2025-12-23 23:43:46 +01:00
9qeklajc
a6d0bd1a19 update aliases 2025-12-23 23:43:16 +01:00
shroominic
9594e9fb52 Merge pull request #251 from Routstr/embeddings-integration
Embeddings integration
2025-12-23 20:53:26 +01:00
9qeklajc
b01c7b2e56 Merge branch 'v0.2.1' into embeddings 2025-12-22 21:04:28 +01:00
Shroominic
dd4ed7541f undo 43e9732 2025-12-22 11:31:19 +01:00
Shroominic
cc42534a97 manual alias due to openrouter api bug 2025-12-22 11:30:51 +01:00
shroominic
d203370f01 Merge pull request #265 from Routstr/update-cost-calculation
use cost field if available
2025-12-22 10:20:35 +01:00
Shroominic
e39742c429 rm completion_image pricing with manual overrides 2025-12-22 09:35:23 +01:00
Shroominic
301dd81215 include openrouters upstream_inference_cost + manually add gemini image completion cost 2025-12-21 09:55:25 +01:00
9qeklajc
5416cefd87 use cost field if available 2025-12-20 16:16:23 +01:00
9qeklajc
43e97326e0 fix model naming issue in response 2025-12-12 20:14:43 +01:00
9qeklajc
06770a0702 Merge branch 'v0.2.1' into embeddings-integration 2025-12-12 19:44:45 +01:00
Shroominic
547365894d add simple test 2025-12-11 13:58:57 +08:00
Shroominic
c0176a5274 Merge branch 'v0.2.1' into embeddings-integration 2025-12-11 13:06:20 +08:00
Shroominic
329d22363f Merge branch 'v0.2.1' into embeddings-integration 2025-12-11 12:59:41 +08:00
9qeklajc
9438bc957f clean up 2025-12-03 22:24:43 +01:00
9qeklajc
5d2219880d embedding integration 2025-12-03 22:23:03 +01:00
9qeklajc
195da0c9da embedding integration 2025-12-03 22:22:57 +01:00
9qeklajc
8df0c17bc3 embedding integration 2025-12-03 22:20:31 +01:00
9qeklajc
7bc9ee0653 Merge branch 'gemini-upstream' into embeddings 2025-12-03 22:01:39 +01:00
9qeklajc
355f8601c1 embedding integration 2025-12-03 21:58:31 +01:00
15 changed files with 404 additions and 297 deletions

View File

@@ -0,0 +1,37 @@
"""alias-ids
Revision ID: b9667ffc5701
Revises: lightning_invoices
Create Date: 2025-12-25 19:30:44.673350
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
# revision identifiers, used by Alembic.
revision = "b9667ffc5701"
down_revision = "lightning_invoices"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic ###
op.add_column(
"models",
sa.Column("canonical_slug", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
)
op.add_column(
"models",
sa.Column("alias_ids", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("models", "alias_ids")
op.drop_column("models", "canonical_slug")
# ### end Alembic commands ###

View File

@@ -206,19 +206,20 @@ def create_model_mappings(
alias: str, model: "Model", provider: "BaseUpstreamProvider"
) -> None:
"""Set alias to model/provider if not set or if new model is preferred."""
existing_model = model_instances.get(alias)
alias_lower = alias.lower()
existing_model = model_instances.get(alias_lower)
if not existing_model:
# No existing mapping, set it
model_instances[alias] = model
provider_map[alias] = provider
model_instances[alias_lower] = model
provider_map[alias_lower] = provider
else:
# Check if candidate should replace existing
existing_provider = provider_map[alias]
existing_provider = provider_map[alias_lower]
if should_prefer_model(
model, provider, existing_model, existing_provider, alias
):
model_instances[alias] = model
provider_map[alias] = provider
model_instances[alias_lower] = model
provider_map[alias_lower] = provider
def process_provider_models(
upstream: "BaseUpstreamProvider", is_openrouter: bool = False

View File

@@ -1493,20 +1493,8 @@ class ModelCreate(BaseModel):
per_request_limits: dict[str, object] | None = None
top_provider: dict[str, object] | None = None
upstream_provider_id: int | None = None
enabled: bool = True
class ModelUpdate(BaseModel):
id: str
name: str
description: str
created: int
context_length: int
architecture: dict[str, object]
pricing: dict[str, object]
per_request_limits: dict[str, object] | None = None
top_provider: dict[str, object] | None = None
upstream_provider_id: int | None = None
canonical_slug: str | None = None
alias_ids: list[str] | None = None
enabled: bool = True
@@ -2416,44 +2404,80 @@ async def admin_upstream_providers(request: Request) -> str:
"/api/upstream-providers/{provider_id}/models",
dependencies=[Depends(require_admin_api)],
)
async def create_provider_model(
async def upsert_provider_model(
provider_id: int, payload: ModelCreate
) -> dict[str, object]:
print(payload)
logger.info(
f"UPSERT_PROVIDER_MODEL called: provider_id={provider_id}, model_id={payload.id}"
)
async with create_session() as session:
provider = await session.get(UpstreamProviderRow, provider_id)
if not provider:
raise HTTPException(status_code=404, detail="Provider not found")
exists = await session.get(ModelRow, (payload.id, provider_id))
if exists:
raise HTTPException(
status_code=409,
detail="Model with this ID already exists for this provider",
)
# Try to get existing model
existing_row = await session.get(ModelRow, (payload.id, provider_id))
row = ModelRow(
id=payload.id,
name=payload.name,
description=payload.description,
created=int(payload.created),
context_length=int(payload.context_length),
architecture=json.dumps(payload.architecture),
pricing=json.dumps(payload.pricing),
sats_pricing=None,
per_request_limits=(
if existing_row:
# Update existing model
logger.info(f"Updating existing model: {payload.id}")
existing_row.name = payload.name
existing_row.description = payload.description
existing_row.created = int(payload.created)
existing_row.context_length = int(payload.context_length)
existing_row.architecture = json.dumps(payload.architecture)
existing_row.pricing = json.dumps(payload.pricing)
existing_row.sats_pricing = None
existing_row.per_request_limits = (
json.dumps(payload.per_request_limits)
if payload.per_request_limits is not None
else None
),
top_provider=(
)
existing_row.top_provider = (
json.dumps(payload.top_provider) if payload.top_provider else None
),
upstream_provider_id=provider_id,
enabled=payload.enabled,
)
session.add(row)
await session.commit()
await session.refresh(row)
)
existing_row.canonical_slug = payload.canonical_slug
existing_row.alias_ids = (
json.dumps(payload.alias_ids) if payload.alias_ids else None
)
existing_row.enabled = payload.enabled
session.add(existing_row)
await session.commit()
await session.refresh(existing_row)
row = existing_row
else:
# Create new model
logger.info(f"Creating new model: {payload.id}")
row = ModelRow(
id=payload.id,
name=payload.name,
description=payload.description,
created=int(payload.created),
context_length=int(payload.context_length),
architecture=json.dumps(payload.architecture),
pricing=json.dumps(payload.pricing),
sats_pricing=None,
per_request_limits=(
json.dumps(payload.per_request_limits)
if payload.per_request_limits is not None
else None
),
top_provider=(
json.dumps(payload.top_provider) if payload.top_provider else None
),
canonical_slug=payload.canonical_slug,
alias_ids=(
json.dumps(payload.alias_ids) if payload.alias_ids else None
),
upstream_provider_id=provider_id,
enabled=payload.enabled,
)
session.add(row)
await session.commit()
await session.refresh(row)
await refresh_model_maps()
return _row_to_model(
@@ -2461,6 +2485,20 @@ async def create_provider_model(
).dict() # type: ignore
@admin_router.patch(
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
dependencies=[Depends(require_admin_api)],
)
async def update_provider_model_legacy(
provider_id: int, model_id: str, payload: ModelCreate
) -> dict[str, object]:
"""Legacy PATCH endpoint - redirects to upsert POST endpoint for backward compatibility."""
logger.info(
f"LEGACY_PATCH_UPDATE called: provider_id={provider_id}, model_id={model_id}"
)
return await upsert_provider_model(provider_id, payload)
@admin_router.get(
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
dependencies=[Depends(require_admin_api)],
@@ -2481,76 +2519,6 @@ async def get_provider_model(provider_id: int, model_id: str) -> dict[str, objec
).dict() # type: ignore
@admin_router.patch(
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
dependencies=[Depends(require_admin_api)],
)
async def update_provider_model(
provider_id: int, model_id: str, payload: ModelUpdate
) -> dict[str, object]:
if payload.id != model_id:
raise HTTPException(status_code=400, detail="Path id does not match payload id")
async with create_session() as session:
provider = await session.get(UpstreamProviderRow, provider_id)
if not provider:
raise HTTPException(status_code=404, detail="Provider not found")
row = await session.get(ModelRow, (model_id, provider_id))
if not row:
raise HTTPException(
status_code=404, detail="Model not found for this provider"
)
row.name = payload.name
row.description = payload.description
row.created = int(payload.created)
row.context_length = int(payload.context_length)
row.architecture = json.dumps(payload.architecture)
row.pricing = json.dumps(payload.pricing)
row.sats_pricing = None
row.per_request_limits = (
json.dumps(payload.per_request_limits)
if payload.per_request_limits is not None
else None
)
row.top_provider = (
json.dumps(payload.top_provider) if payload.top_provider else None
)
was_disabled = not row.enabled
row.enabled = payload.enabled
session.add(row)
await session.commit()
await session.refresh(row)
if was_disabled and payload.enabled:
from ..payment.models import _cleanup_enabled_models_once
try:
await _cleanup_enabled_models_once()
except Exception as e:
logger.warning(
f"Failed to run model cleanup after enabling: {e}",
extra={"model_id": model_id, "error": str(e)},
)
await refresh_model_maps()
return _row_to_model(
row, apply_provider_fee=True, provider_fee=provider.provider_fee
).dict() # type: ignore
@admin_router.put(
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
dependencies=[Depends(require_admin_api)],
)
async def update_provider_model_put(
provider_id: int, model_id: str, payload: ModelUpdate
) -> dict[str, object]:
return await update_provider_model(provider_id, model_id, payload)
@admin_router.delete(
"/api/upstream-providers/{provider_id}/models/{model_id:path}",
dependencies=[Depends(require_admin_api)],

View File

@@ -68,6 +68,10 @@ class ModelRow(SQLModel, table=True): # type: ignore
sats_pricing: str | None = Field(default=None)
per_request_limits: str | None = Field(default=None)
top_provider: str | None = Field(default=None)
canonical_slug: str | None = Field(default=None, description="Canonical model slug")
alias_ids: str | None = Field(
default=None, description="JSON array of model alias IDs"
)
enabled: bool = Field(default=True, description="Whether this model is enabled")
upstream_provider: "UpstreamProviderRow" = Relationship(back_populates="models")

View File

@@ -14,7 +14,6 @@ from ..balance import balance_router, deprecated_wallet_router
from ..discovery import providers_cache_refresher, providers_router
from ..nip91 import announce_provider
from ..payment.models import (
cleanup_enabled_models_periodically,
models_router,
update_sats_pricing,
)
@@ -49,7 +48,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
nip91_task = None
providers_task = None
models_refresh_task = None
models_cleanup_task = None
model_maps_refresh_task = None
try:
@@ -86,7 +84,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
models_refresh_task = asyncio.create_task(
refresh_upstreams_models_periodically(get_upstreams())
)
models_cleanup_task = asyncio.create_task(cleanup_enabled_models_periodically())
model_maps_refresh_task = asyncio.create_task(refresh_model_maps_periodically())
payout_task = asyncio.create_task(periodic_payout())
if global_settings.nsec:
@@ -125,8 +122,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
providers_task.cancel()
if models_refresh_task is not None:
models_refresh_task.cancel()
if models_cleanup_task is not None:
models_cleanup_task.cancel()
if model_maps_refresh_task is not None:
model_maps_refresh_task.cancel()
@@ -144,8 +139,6 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
tasks_to_wait.append(providers_task)
if models_refresh_task is not None:
tasks_to_wait.append(models_refresh_task)
if models_cleanup_task is not None:
tasks_to_wait.append(models_cleanup_task)
if model_maps_refresh_task is not None:
tasks_to_wait.append(model_maps_refresh_task)

View File

@@ -5,6 +5,7 @@ from pydantic.v1 import BaseModel
from ..core import get_logger
from ..core.db import AsyncSession
from ..core.settings import settings
from .price import sats_usd_price
logger = get_logger(__name__)
@@ -64,6 +65,56 @@ async def calculate_cost( # todo: can be sync
)
return cost_data
usage_data = response_data["usage"]
usd_cost = 0.0
# Prioritize cost_details.upstream_inference_cost
if "cost_details" in usage_data:
usd_cost = float(
usage_data["cost_details"].get("upstream_inference_cost", 0) or 0
)
# Fallback to cost field if upstream_inference_cost is 0
if usd_cost == 0 and "cost" in usage_data:
try:
usd_cost = float(usage_data.get("cost", 0) or 0)
except Exception:
pass
if usd_cost > 0:
try:
sats_per_usd = 1.0 / sats_usd_price()
cost_in_sats = usd_cost * sats_per_usd
cost_in_msats = math.ceil(cost_in_sats * 1000)
logger.info(
"Using cost from usage data/details",
extra={
"usd_cost": usd_cost,
"cost_in_sats": cost_in_sats,
"cost_in_msats": cost_in_msats,
"model": response_data.get("model", "unknown"),
},
)
return CostData(
base_msats=-1,
input_msats=-1, # Cost field doesn't break down by token type
output_msats=-1,
total_msats=cost_in_msats,
)
except Exception as e:
logger.warning(
"Error calculating cost from usage data",
extra={
"error": str(e),
"usd_cost": usd_cost,
"model": response_data.get("model", "unknown"),
},
)
# Fall through to token-based calculation
MSATS_PER_1K_INPUT_TOKENS: float = (
float(settings.fixed_per_1k_input_tokens) * 1000.0
)
@@ -129,10 +180,19 @@ async def calculate_cost( # todo: can be sync
)
return cost_data
input_tokens = response_data.get("usage", {}).get("prompt_tokens", 0)
output_tokens = response_data.get("usage", {}).get("completion_tokens", 0)
input_tokens = usage_data.get("prompt_tokens", 0)
output_tokens = usage_data.get("completion_tokens", 0)
# added for response api
input_tokens = (
input_tokens if input_tokens != 0 else usage_data.get("input_tokens", 0)
)
output_tokens = (
output_tokens if output_tokens != 0 else usage_data.get("output_tokens", 0)
)
input_msats = round(input_tokens / 1000 * MSATS_PER_1K_INPUT_TOKENS, 3)
output_msats = round(output_tokens / 1000 * MSATS_PER_1K_OUTPUT_TOKENS, 3)
token_based_cost = math.ceil(input_msats + output_msats)

View File

@@ -5,10 +5,9 @@ import random
import httpx
from fastapi import APIRouter, Depends
from pydantic.v1 import BaseModel
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from ..core.db import ModelRow, create_session, get_session
from ..core.db import ModelRow, get_session
from ..core.logging import get_logger
from ..core.settings import settings
from .price import sats_usd_price
@@ -93,12 +92,32 @@ async def async_fetch_openrouter_models(source_filter: str | None = None) -> lis
try:
async with httpx.AsyncClient() as client:
response = await client.get(f"{base_url}/models", timeout=30)
response.raise_for_status()
data = response.json()
models_response, embeddings_response = await asyncio.gather(
client.get(f"{base_url}/models", timeout=30),
client.get(f"{base_url}/embeddings/models", timeout=30),
return_exceptions=True,
)
def process_models_response(
response: httpx.Response | BaseException,
) -> list[dict]:
if not isinstance(response, BaseException):
response.raise_for_status()
data = response.json()
return [
model
for model in data.get("data", [])
if ":free" not in model.get("id", "").lower()
]
return []
models_data: list[dict] = []
for model in data.get("data", []):
models_data.extend(process_models_response(models_response))
models_data.extend(process_models_response(embeddings_response))
# Apply source filter and exclusions
filtered_models = []
for model in models_data:
model_id = model.get("id", "")
if source_filter:
@@ -116,9 +135,9 @@ async def async_fetch_openrouter_models(source_filter: str | None = None) -> lis
if not _has_valid_pricing(model):
continue
models_data.append(model)
filtered_models.append(model)
return models_data
return filtered_models
except Exception as e:
logger.error(f"Error (async) fetching models from OpenRouter API: {e}")
return []
@@ -165,6 +184,7 @@ def _row_to_model(
enabled=row.enabled,
upstream_provider_id=row.upstream_provider_id,
canonical_slug=getattr(row, "canonical_slug", None),
alias_ids=json.loads(row.alias_ids) if row.alias_ids else None,
)
if apply_provider_fee:
@@ -413,114 +433,6 @@ async def update_sats_pricing() -> None:
logger.error(f"Error updating sats pricing: {e}")
async def cleanup_enabled_models_periodically() -> None:
"""Background task to clean up enabled models that match upstream pricing.
When model is enabled (enabled=True), remove it from DB if it matches upstream pricing.
Keep it in DB only if pricing differs from upstream or if it's disabled.
"""
interval = getattr(
settings, "models_cleanup_interval_seconds", 300
) # 5 minutes default
if not interval or interval <= 0:
return
while True:
try:
await _cleanup_enabled_models_once()
except asyncio.CancelledError:
break
except Exception as e:
logger.error(
"Error during enabled models cleanup",
extra={"error": str(e), "error_type": type(e).__name__},
)
try:
jitter = max(0.0, float(interval) * 0.1)
await asyncio.sleep(interval + random.uniform(0, jitter))
except asyncio.CancelledError:
break
async def _cleanup_enabled_models_once() -> None:
"""Clean up enabled models that match upstream pricing."""
from ..proxy import get_upstreams
async with create_session() as session:
# Get all enabled models from DB
result = await session.exec(
select(ModelRow).where(
ModelRow.enabled, # Only enabled models
)
)
db_models = result.all()
if not db_models:
return
upstreams = get_upstreams()
models_to_remove = []
for db_model in db_models:
# Find corresponding upstream model
upstream_model = None
for upstream in upstreams:
upstream_model = upstream.get_cached_model_by_id(db_model.id)
if upstream_model:
break
if not upstream_model:
continue
# Compare pricing to see if they match
db_pricing = json.loads(db_model.pricing)
upstream_pricing = upstream_model.pricing.dict()
# Check if pricing matches (with small tolerance for float comparison)
pricing_matches = _pricing_matches(db_pricing, upstream_pricing)
if pricing_matches:
models_to_remove.append(db_model)
logger.info(
f"Removing enabled model {db_model.id} - matches upstream pricing",
extra={"model_id": db_model.id},
)
# Remove models that match upstream pricing
for model in models_to_remove:
await session.delete(model)
if models_to_remove:
await session.commit()
logger.info(
f"Cleaned up {len(models_to_remove)} enabled models that match upstream pricing"
)
def _pricing_matches(
db_pricing: dict, upstream_pricing: dict, tolerance: float = 0.0
) -> bool:
"""Check if pricing dictionaries match within tolerance."""
keys_to_compare = [
"prompt",
"completion",
"request",
"image",
"web_search",
"internal_reasoning",
]
for key in keys_to_compare:
db_val = int(float(db_pricing.get(key, 0.0)) * 1000000)
upstream_val = int(float(upstream_pricing.get(key, 0.0)) * 1000000)
if abs(db_val - upstream_val) > tolerance:
return False
return True
@models_router.get("/v1/models")
@models_router.get("/models", include_in_schema=False)
async def models(session: AsyncSession = Depends(get_session)) -> dict:

View File

@@ -65,12 +65,12 @@ def get_upstreams() -> list[BaseUpstreamProvider]:
def get_model_instance(model_id: str) -> Model | None:
"""Get Model instance by ID from global cache."""
return _model_instances.get(model_id)
return _model_instances.get(model_id.lower())
def get_provider_for_model(model_id: str) -> BaseUpstreamProvider | None:
"""Get UpstreamProvider for model ID from global cache."""
return _provider_map.get(model_id)
return _provider_map.get(model_id.lower())
def get_unique_models() -> list[Model]:

View File

@@ -734,51 +734,53 @@ class BaseUpstreamProvider:
await client.aclose()
return mapped_error
if path.endswith("chat/completions"):
client_wants_streaming = False
if request_body:
try:
request_data = json.loads(request_body)
client_wants_streaming = request_data.get("stream", False)
logger.debug(
"Chat completion request analysis",
extra={
"client_wants_streaming": client_wants_streaming,
"model": request_data.get("model", "unknown"),
"key_hash": key.hashed_key[:8] + "...",
},
)
except json.JSONDecodeError:
logger.warning(
"Failed to parse request body JSON for streaming detection"
)
if path.endswith("chat/completions") or path.endswith("embeddings"):
if path.endswith("chat/completions"):
client_wants_streaming = False
if request_body:
try:
request_data = json.loads(request_body)
client_wants_streaming = request_data.get("stream", False)
logger.debug(
"Chat completion request analysis",
extra={
"client_wants_streaming": client_wants_streaming,
"model": request_data.get("model", "unknown"),
"key_hash": key.hashed_key[:8] + "...",
},
)
except json.JSONDecodeError:
logger.warning(
"Failed to parse request body JSON for streaming detection"
)
content_type = response.headers.get("content-type", "")
upstream_is_streaming = "text/event-stream" in content_type
is_streaming = client_wants_streaming and upstream_is_streaming
content_type = response.headers.get("content-type", "")
upstream_is_streaming = "text/event-stream" in content_type
is_streaming = client_wants_streaming and upstream_is_streaming
logger.debug(
"Response type analysis",
extra={
"is_streaming": is_streaming,
"client_wants_streaming": client_wants_streaming,
"upstream_is_streaming": upstream_is_streaming,
"content_type": content_type,
"key_hash": key.hashed_key[:8] + "...",
},
)
if is_streaming and response.status_code == 200:
result = await self.handle_streaming_chat_completion(
response, key, max_cost_for_model
logger.debug(
"Response type analysis",
extra={
"is_streaming": is_streaming,
"client_wants_streaming": client_wants_streaming,
"upstream_is_streaming": upstream_is_streaming,
"content_type": content_type,
"key_hash": key.hashed_key[:8] + "...",
},
)
background_tasks = BackgroundTasks()
background_tasks.add_task(response.aclose)
background_tasks.add_task(client.aclose)
result.background = background_tasks
return result
elif response.status_code == 200:
if is_streaming and response.status_code == 200:
result = await self.handle_streaming_chat_completion(
response, key, max_cost_for_model
)
background_tasks = BackgroundTasks()
background_tasks.add_task(response.aclose)
background_tasks.add_task(client.aclose)
result.background = background_tasks
return result
# Handle both non-streaming chat completions and embeddings
if response.status_code == 200:
try:
return await self.handle_non_streaming_chat_completion(
response, key, session, max_cost_for_model
@@ -1519,9 +1521,9 @@ class BaseUpstreamProvider:
error_response.headers["X-Cashu"] = refund_token
return error_response
if path.endswith("chat/completions"):
if path.endswith("chat/completions") or path.endswith("embeddings"):
logger.debug(
"Processing chat completion response",
"Processing completion/embeddings response",
extra={"path": path, "amount": amount, "unit": unit},
)
@@ -1770,15 +1772,32 @@ class BaseUpstreamProvider:
async def _fetch_openrouter_models(self) -> list[dict]:
"""Fetch models from OpenRouter API."""
url = "https://openrouter.ai/api/v1/models"
embeddings_url = "https://openrouter.ai/api/v1/embeddings/models"
async with httpx.AsyncClient(timeout=30.0) as client:
response = await client.get(url)
response.raise_for_status()
models = response.json()
return [
model
for model in models.get("data", [])
if ":free" not in model.get("id", "").lower()
]
models_response, embeddings_response = await asyncio.gather(
client.get(url), client.get(embeddings_url), return_exceptions=True
)
all_models = []
def process_models_response(
response: httpx.Response | BaseException,
) -> list[dict]:
if not isinstance(response, BaseException):
response.raise_for_status()
data = response.json()
return [
model
for model in data.get("data", [])
if ":free" not in model.get("id", "").lower()
]
return []
all_models.extend(process_models_response(models_response))
all_models.extend(process_models_response(embeddings_response))
return all_models
async def _fetch_provider_models(self) -> dict:
"""Fetch models from provider's API."""

View File

@@ -50,7 +50,13 @@ class OpenRouterUpstreamProvider(BaseUpstreamProvider):
async def fetch_models(self) -> list[Model]:
"""Fetch all OpenRouter models."""
models_data = await async_fetch_openrouter_models()
return [Model(**model) for model in models_data] # type: ignore
models = [Model(**model) for model in models_data] # type: ignore
# manual alias for openai/text-embedding-ada-002 due to openrouter api bug
for model in models:
if model.id == "openai/text-embedding-ada-002":
model.alias_ids = ["text-embedding-ada-002-v2"]
break
return models
async def get_balance(self) -> float | None:
"""Get the current account balance from OpenRouter.

View File

@@ -0,0 +1,97 @@
import json
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from httpx import AsyncClient
@pytest.mark.integration
@pytest.mark.asyncio
async def test_proxy_embeddings_endpoint(authenticated_client: AsyncClient) -> None:
"""Test the embeddings endpoint proxy functionality"""
test_payload = {
"model": "text-embedding-ada-002",
"input": "The quick brown fox",
}
mock_response_data = {
"object": "list",
"data": [
{"object": "embedding", "embedding": [0.0023, -0.0012, 0.0045], "index": 0}
],
"model": "text-embedding-ada-002",
"usage": {"prompt_tokens": 5, "total_tokens": 5},
}
with patch("httpx.AsyncClient.send") as mock_send:
# Create a proper async generator for iter_bytes
async def mock_iter_bytes(*args: Any, **kwargs: Any) -> Any:
yield json.dumps(mock_response_data).encode()
mock_response = AsyncMock()
mock_response.status_code = 200
mock_response.headers = {"content-type": "application/json"}
mock_response.text = json.dumps(mock_response_data)
# Use MagicMock for synchronous .json() method
mock_response.json = MagicMock(return_value=mock_response_data)
mock_response.iter_bytes = mock_iter_bytes
mock_response.aiter_bytes = mock_iter_bytes
mock_send.return_value = mock_response
# Make POST request to embeddings endpoint
response = await authenticated_client.post("/v1/embeddings", json=test_payload)
assert response.status_code == 200
response_data = response.json()
assert response_data["object"] == "list"
assert len(response_data["data"]) == 1
assert response_data["data"][0]["object"] == "embedding"
# Verify request was forwarded
mock_send.assert_called_once()
forwarded_request = mock_send.call_args[0][0]
# Verify the path ends with embeddings
# Note: forwarded path might be full URL
assert str(forwarded_request.url).endswith("embeddings")
@pytest.mark.integration
@pytest.mark.asyncio
async def test_model_case_insensitivity(authenticated_client: AsyncClient) -> None:
"""Test that model lookups are case insensitive"""
# We'll use a mixed-case model ID that should match the lowercase one in the system
# We assume 'gpt-3.5-turbo' is available in the mock env/database
test_payload = {
"model": "GPT-3.5-TURBO",
"messages": [{"role": "user", "content": "Hello"}],
}
with patch("httpx.AsyncClient.send") as mock_send:
mock_response_data = {
"id": "chatcmpl-123",
"object": "chat.completion",
"choices": [{"message": {"content": "Hi"}}],
"usage": {"total_tokens": 10},
}
async def mock_iter_bytes(*args: Any, **kwargs: Any) -> Any:
yield json.dumps(mock_response_data).encode()
mock_response = AsyncMock()
mock_response.status_code = 200
mock_response.headers = {"content-type": "application/json"}
mock_response.text = json.dumps(mock_response_data)
mock_response.json = MagicMock(return_value=mock_response_data)
mock_response.iter_bytes = mock_iter_bytes
mock_response.aiter_bytes = mock_iter_bytes
mock_send.return_value = mock_response
response = await authenticated_client.post(
"/v1/chat/completions", json=test_payload
)
assert response.status_code == 200

View File

@@ -41,6 +41,10 @@ export function ModelSearchFilter({
model.name.toLowerCase().includes(query) ||
model.full_name.toLowerCase().includes(query) ||
model.provider.toLowerCase().includes(query) ||
(model.alias_ids &&
model.alias_ids.some((alias) =>
alias.toLowerCase().includes(query)
)) ||
(model.description &&
model.description.toLowerCase().includes(query)) ||
model.modelType.toLowerCase().includes(query)

View File

@@ -542,6 +542,7 @@ export function ModelSelector({
top_provider: null,
upstream_provider_id: providerId,
enabled: model.isEnabled,
alias_ids: model.alias_ids || null,
};
setModelDialogState({
@@ -585,6 +586,7 @@ export function ModelSelector({
top_provider: null,
upstream_provider_id: providerId,
enabled: model.isEnabled,
alias_ids: model.alias_ids || null,
};
setModelDialogState({

View File

@@ -31,6 +31,7 @@ export const ModelSchema = z.object({
// API key type indicators
has_own_api_key: z.boolean(),
api_key_type: z.string(), // "individual" or "group"
alias_ids: z.array(z.string()).nullable().optional(),
});
// Schema for a model with additional provider-specific settings

View File

@@ -121,6 +121,7 @@ export interface AdminModelAsModel {
soft_deleted?: boolean;
has_own_api_key: boolean;
api_key_type: string;
alias_ids?: string[] | null;
}
export interface AdminModelGroup {
@@ -207,6 +208,7 @@ export class AdminService {
soft_deleted: !adminModel.enabled,
has_own_api_key: false,
api_key_type: 'group',
alias_ids: adminModel.alias_ids,
};
}
@@ -354,8 +356,9 @@ export class AdminService {
original: data.pricing,
converted: payload.pricing,
});
const model = await apiClient.patch<AdminModel>(
`/admin/api/upstream-providers/${providerId}/models/${encodeURIComponent(modelId)}`,
// Use the same POST endpoint for both create and update (upsert)
const model = await apiClient.post<AdminModel>(
`/admin/api/upstream-providers/${providerId}/models`,
payload
);
return {