mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-07-22 12:22:20 +00:00
Compare commits
8 Commits
fix-docker
...
fix-swappi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d9ab46b3bb | ||
|
|
9d5e14903b | ||
|
|
a07e6723d9 | ||
|
|
1db37cf084 | ||
|
|
27ec052c17 | ||
|
|
00b813f6a8 | ||
|
|
91e5198a94 | ||
|
|
164ed775c8 |
@@ -337,21 +337,26 @@ async def refund_wallet_endpoint(
|
||||
)
|
||||
|
||||
# --- MINT: balance is locked at zero, safe to create the refund token ---
|
||||
# Proofs from untrusted mints are swapped to primary_mint on receive.
|
||||
# Use primary_mint unless key.refund_mint_url is an explicitly trusted mint.
|
||||
effective_refund_mint = (
|
||||
key.refund_mint_url
|
||||
if key.refund_mint_url and key.refund_mint_url in settings.cashu_mints
|
||||
else settings.primary_mint
|
||||
)
|
||||
try:
|
||||
if key.refund_address:
|
||||
from .core.settings import settings as global_settings
|
||||
|
||||
await send_to_lnurl(
|
||||
remaining_balance,
|
||||
key.refund_currency or "sat",
|
||||
key.refund_mint_url or global_settings.primary_mint,
|
||||
effective_refund_mint,
|
||||
key.refund_address,
|
||||
)
|
||||
result = {"recipient": key.refund_address}
|
||||
else:
|
||||
refund_currency = key.refund_currency or "sat"
|
||||
token = await send_token(
|
||||
remaining_balance, refund_currency, key.refund_mint_url
|
||||
remaining_balance, refund_currency, effective_refund_mint
|
||||
)
|
||||
result = {"token": token}
|
||||
|
||||
@@ -379,15 +384,26 @@ async def refund_wallet_endpoint(
|
||||
# Minting failed — restore the debited balance
|
||||
await _restore_balance(session, key.hashed_key, pre_debit_balance, pre_debit_reserved)
|
||||
error_msg = str(e)
|
||||
logger.error(
|
||||
"refund_wallet_endpoint: mint/send failed",
|
||||
extra={
|
||||
"error": error_msg,
|
||||
"error_type": type(e).__name__,
|
||||
"hashed_key": key.hashed_key,
|
||||
"remaining_balance": remaining_balance,
|
||||
"refund_currency": key.refund_currency,
|
||||
"refund_mint_url": key.refund_mint_url,
|
||||
"has_refund_address": bool(key.refund_address),
|
||||
},
|
||||
)
|
||||
if (
|
||||
"mint" in error_msg.lower()
|
||||
or "connection" in error_msg.lower()
|
||||
or isinstance(e, Exception)
|
||||
and "ConnectError" in str(type(e))
|
||||
or "ConnectError" in str(type(e))
|
||||
):
|
||||
raise HTTPException(status_code=503, detail="Mint service unavailable")
|
||||
raise HTTPException(status_code=503, detail=f"Mint service unavailable: {error_msg}")
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail="Refund failed")
|
||||
raise HTTPException(status_code=500, detail=f"Refund failed: {error_msg}")
|
||||
|
||||
await _refund_cache_set(bearer_value, result)
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ from ..payment.models import (
|
||||
from ..payment.price import sats_usd_price
|
||||
from ..wallet import recieve_token, send_token
|
||||
from . import messages_dispatch
|
||||
from .count_tokens import count_tokens_locally
|
||||
from .litellm_routing import detect_litellm_prefix
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -2196,6 +2197,12 @@ class BaseUpstreamProvider:
|
||||
"""
|
||||
path = self.normalize_request_path(path, model_obj)
|
||||
|
||||
if (
|
||||
path.endswith("messages/count_tokens")
|
||||
and not self.supports_anthropic_messages
|
||||
):
|
||||
return count_tokens_locally(request_body, model_obj)
|
||||
|
||||
if (
|
||||
path.endswith("messages")
|
||||
and not path.endswith("count_tokens")
|
||||
@@ -3394,6 +3401,12 @@ class BaseUpstreamProvider:
|
||||
|
||||
request_body = await request.body()
|
||||
|
||||
if (
|
||||
path.endswith("messages/count_tokens")
|
||||
and not self.supports_anthropic_messages
|
||||
):
|
||||
return count_tokens_locally(request_body, model_obj)
|
||||
|
||||
if (
|
||||
path.endswith("messages")
|
||||
and not path.endswith("count_tokens")
|
||||
|
||||
108
routstr/upstream/count_tokens.py
Normal file
108
routstr/upstream/count_tokens.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""Local handling of Anthropic ``/v1/messages/count_tokens`` for upstreams
|
||||
that do not natively expose the endpoint.
|
||||
|
||||
Most non-Anthropic upstreams (OpenAI-compat, Gemini OpenAI-compat,
|
||||
OpenRouter chat-completions, generic providers) return 400/404 when asked
|
||||
to ``POST /messages/count_tokens``. Claude Code and other Anthropic SDK
|
||||
clients call this endpoint before each turn to size context windows and
|
||||
trigger compaction, so a failure breaks the whole chat.
|
||||
|
||||
We answer locally. ``litellm.token_counter`` understands the Anthropic
|
||||
message shape and the per-model tokenizers, so we prefer it. If it raises
|
||||
(unknown model, encoding lookup failure, ...), we fall back to the
|
||||
project's own ``estimate_tokens`` heuristic, which is always defined and
|
||||
never raises.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from fastapi.responses import Response
|
||||
|
||||
from ..core import get_logger
|
||||
from ..payment.helpers import estimate_tokens
|
||||
from ..payment.models import Model
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _parse_request_body(request_body: bytes | None) -> dict[str, Any]:
|
||||
if not request_body:
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(request_body)
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
def _count_with_litellm(model: str, body: dict[str, Any]) -> int:
|
||||
messages = body.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
messages = []
|
||||
|
||||
system = body.get("system")
|
||||
if isinstance(system, str) and system:
|
||||
messages = [{"role": "system", "content": system}, *messages]
|
||||
elif isinstance(system, list):
|
||||
text = "".join(
|
||||
block.get("text", "")
|
||||
for block in system
|
||||
if isinstance(block, dict) and block.get("type") == "text"
|
||||
)
|
||||
if text:
|
||||
messages = [{"role": "system", "content": text}, *messages]
|
||||
|
||||
tools = body.get("tools") if isinstance(body.get("tools"), list) else None
|
||||
|
||||
return int(
|
||||
litellm.token_counter(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_tokens_locally(
|
||||
request_body: bytes | None,
|
||||
model_obj: Model | None,
|
||||
) -> Response:
|
||||
"""Return an Anthropic-compatible count_tokens response without
|
||||
touching the upstream. Always returns 200; never raises."""
|
||||
body = _parse_request_body(request_body)
|
||||
|
||||
model_name = ""
|
||||
if model_obj is not None:
|
||||
model_name = model_obj.forwarded_model_id or model_obj.id or ""
|
||||
if not model_name:
|
||||
body_model = body.get("model")
|
||||
if isinstance(body_model, str):
|
||||
model_name = body_model
|
||||
|
||||
input_tokens: int
|
||||
try:
|
||||
input_tokens = _count_with_litellm(model_name, body)
|
||||
except Exception as exc:
|
||||
messages = body.get("messages")
|
||||
fallback_messages = messages if isinstance(messages, list) else []
|
||||
input_tokens = estimate_tokens(fallback_messages)
|
||||
logger.debug(
|
||||
"litellm token_counter failed; using local estimator",
|
||||
extra={
|
||||
"model": model_name,
|
||||
"error": str(exc),
|
||||
"error_type": type(exc).__name__,
|
||||
"estimated_tokens": input_tokens,
|
||||
},
|
||||
)
|
||||
|
||||
payload = {"input_tokens": max(0, int(input_tokens))}
|
||||
return Response(
|
||||
content=json.dumps(payload).encode(),
|
||||
status_code=200,
|
||||
media_type="application/json",
|
||||
)
|
||||
@@ -56,17 +56,56 @@ async def recieve_token(
|
||||
|
||||
async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int, str]:
|
||||
"""Internal send function - returns amount and serialized token"""
|
||||
wallet: Wallet = await get_wallet(mint_url or settings.primary_mint, unit)
|
||||
proofs = get_proofs_per_mint_and_unit(
|
||||
wallet, mint_url or settings.primary_mint, unit
|
||||
effective_mint_url = mint_url or settings.primary_mint
|
||||
wallet: Wallet = await get_wallet(effective_mint_url, unit)
|
||||
proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
|
||||
proofs_for_mint = sum(p.amount for p in proofs)
|
||||
|
||||
# Fallback: proofs from untrusted source mints are swapped to primary_mint
|
||||
# during receive, so the user's preferred refund_mint_url may have no proofs
|
||||
# even though the global wallet has the balance.
|
||||
if proofs_for_mint < amount and effective_mint_url != settings.primary_mint:
|
||||
logger.info(
|
||||
f"send: insufficient proofs at {effective_mint_url} "
|
||||
f"(have {proofs_for_mint}, need {amount}), falling back to primary_mint={settings.primary_mint}"
|
||||
)
|
||||
effective_mint_url = settings.primary_mint
|
||||
wallet = await get_wallet(effective_mint_url, unit)
|
||||
proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
|
||||
proofs_for_mint = sum(p.amount for p in proofs)
|
||||
|
||||
all_mint_urls = list({k.mint_url for k in wallet.keysets.values()})
|
||||
proof_summary = {
|
||||
f"{k.mint_url}/{k.unit.name}": sum(p.amount for p in wallet.proofs if p.id == k.id)
|
||||
for k in wallet.keysets.values()
|
||||
}
|
||||
# Show ALL proofs in DB by keyset_id, regardless of whether the loaded wallet
|
||||
# knows about that keyset. This reveals proofs orphaned under stale keysets.
|
||||
raw_proofs_by_keyset: dict[str, int] = {}
|
||||
for p in wallet.proofs:
|
||||
raw_proofs_by_keyset[p.id] = raw_proofs_by_keyset.get(p.id, 0) + p.amount
|
||||
logger.info(
|
||||
f"send: proof inventory | mint={effective_mint_url} unit={unit} amount={amount} "
|
||||
f"primary_mint={settings.primary_mint} proofs_for_mint={proofs_for_mint} "
|
||||
f"all_mints={all_mint_urls} by_keyset={proof_summary} "
|
||||
f"raw_proofs_by_keyset_id={raw_proofs_by_keyset} "
|
||||
f"total_wallet_proofs={sum(p.amount for p in wallet.proofs)}"
|
||||
)
|
||||
|
||||
# Reserve proofs only after serialization succeeds — if serialize_proofs or
|
||||
# swap_to_send fails mid-way, proofs stay unreserved so dashboard balance
|
||||
# doesn't go negative.
|
||||
send_proofs, _ = await wallet.select_to_send(
|
||||
proofs, amount, set_reserved=True, include_fees=False
|
||||
)
|
||||
token = await wallet.serialize_proofs(
|
||||
send_proofs, include_dleq=False, legacy=False, memo=None
|
||||
proofs, amount, set_reserved=False, include_fees=False
|
||||
)
|
||||
try:
|
||||
token = await wallet.serialize_proofs(
|
||||
send_proofs, include_dleq=False, legacy=False, memo=None
|
||||
)
|
||||
except Exception:
|
||||
await wallet.set_reserved_for_send(send_proofs, reserved=False)
|
||||
raise
|
||||
await wallet.set_reserved_for_send(send_proofs, reserved=True)
|
||||
return amount, token
|
||||
|
||||
|
||||
@@ -81,10 +120,11 @@ async def _calculate_swap_amount(
|
||||
token_mint_url: str,
|
||||
token_wallet: Wallet,
|
||||
primary_wallet: Wallet,
|
||||
proofs: list,
|
||||
) -> int:
|
||||
"""
|
||||
Calculate the amount to mint on the primary mint after accounting for
|
||||
potential swap fees (melt fees) on the foreign mint.
|
||||
melt fees and NUT-02 input fees on the foreign mint.
|
||||
"""
|
||||
if settings.primary_mint_unit == "sat":
|
||||
receive_amount = amount_msat // 1000
|
||||
@@ -111,10 +151,11 @@ async def _calculate_swap_amount(
|
||||
dummy_melt_quote = await token_wallet.melt_quote(dummy_mint_quote.request)
|
||||
|
||||
fee_reserve = dummy_melt_quote.fee_reserve
|
||||
input_fees = token_wallet.get_fees_for_proofs(proofs)
|
||||
if token_unit == "sat":
|
||||
fee_msat = fee_reserve * 1000
|
||||
fee_msat = (fee_reserve + input_fees) * 1000
|
||||
else:
|
||||
fee_msat = fee_reserve
|
||||
fee_msat = fee_reserve + input_fees
|
||||
|
||||
amount_msat_after_fee = amount_msat - fee_msat
|
||||
|
||||
@@ -124,13 +165,14 @@ async def _calculate_swap_amount(
|
||||
minted_amount = int(amount_msat_after_fee)
|
||||
|
||||
if minted_amount <= 0:
|
||||
raise ValueError(f"Fees ({fee_reserve} {token_unit}) exceed token amount")
|
||||
raise ValueError(f"Fees ({fee_reserve + input_fees} {token_unit}) exceed token amount")
|
||||
|
||||
logger.info(
|
||||
"swap_to_primary_mint: fee estimation result",
|
||||
extra={
|
||||
"token_amount_sat": amount_msat // 1000,
|
||||
"estimated_fee_sat": fee_msat // 1000,
|
||||
"input_fees": input_fees,
|
||||
"minted_amount": minted_amount,
|
||||
"minted_unit": settings.primary_mint_unit,
|
||||
},
|
||||
@@ -191,6 +233,7 @@ async def swap_to_primary_mint(
|
||||
token_obj.mint,
|
||||
token_wallet,
|
||||
primary_wallet,
|
||||
token_obj.proofs,
|
||||
)
|
||||
|
||||
mint_quote = await primary_wallet.request_mint(minted_amount)
|
||||
@@ -200,13 +243,15 @@ async def swap_to_primary_mint(
|
||||
)
|
||||
|
||||
melt_quote = await token_wallet.melt_quote(mint_quote.request)
|
||||
total_needed = melt_quote.amount + melt_quote.fee_reserve
|
||||
input_fees = token_wallet.get_fees_for_proofs(token_obj.proofs)
|
||||
total_needed = melt_quote.amount + melt_quote.fee_reserve + input_fees
|
||||
logger.info(
|
||||
"swap_to_primary_mint: melt quote received",
|
||||
extra={
|
||||
"melt_quote_id": melt_quote.quote,
|
||||
"melt_amount": melt_quote.amount,
|
||||
"melt_fee_reserve": melt_quote.fee_reserve,
|
||||
"input_fees": input_fees,
|
||||
"total_needed": total_needed,
|
||||
"token_amount": token_amount,
|
||||
},
|
||||
@@ -219,6 +264,7 @@ async def swap_to_primary_mint(
|
||||
"token_amount": token_amount,
|
||||
"melt_amount": melt_quote.amount,
|
||||
"melt_fee_reserve": melt_quote.fee_reserve,
|
||||
"input_fees": input_fees,
|
||||
"total_needed": total_needed,
|
||||
"shortfall": total_needed - token_amount,
|
||||
},
|
||||
@@ -226,7 +272,7 @@ async def swap_to_primary_mint(
|
||||
raise ValueError(
|
||||
f"Token amount ({token_amount} {token_obj.unit}) is insufficient to cover "
|
||||
f"melt fees. Needed: {total_needed} {token_obj.unit} "
|
||||
f"(amount: {melt_quote.amount} + fee: {melt_quote.fee_reserve})"
|
||||
f"(amount: {melt_quote.amount} + fee: {melt_quote.fee_reserve} + input_fees: {input_fees})"
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -257,19 +303,66 @@ async def swap_to_primary_mint(
|
||||
extra={"minted_amount": minted_amount, "mint_quote_id": mint_quote.quote},
|
||||
)
|
||||
|
||||
await primary_wallet.load_proofs(reload=True)
|
||||
pre_mint_balance = primary_wallet.available_balance.amount
|
||||
try:
|
||||
_ = await primary_wallet.mint(minted_amount, quote_id=mint_quote.quote)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"swap_to_primary_mint: mint on primary failed after successful melt",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"minted_amount": minted_amount,
|
||||
"mint_quote_id": mint_quote.quote,
|
||||
},
|
||||
)
|
||||
raise
|
||||
if "11003" in str(e) or "outputs already signed" in str(e).lower():
|
||||
# Previous mint call signed outputs at the mint but failed before
|
||||
# bump_secret_derivation ran locally. Recover orphaned proofs and
|
||||
# advance the counter so the next request derives fresh secrets.
|
||||
logger.warning(
|
||||
"swap_to_primary_mint: outputs already signed — recovering orphaned proofs",
|
||||
extra={"mint_quote_id": mint_quote.quote, "minted_amount": minted_amount},
|
||||
)
|
||||
try:
|
||||
for keyset_id in primary_wallet.keysets:
|
||||
await primary_wallet.restore_tokens_for_keyset(keyset_id, to=1, batch=25)
|
||||
await primary_wallet.load_proofs(reload=True)
|
||||
post_recovery_balance = primary_wallet.available_balance.amount
|
||||
balance_gained = post_recovery_balance - pre_mint_balance
|
||||
logger.info(
|
||||
"swap_to_primary_mint: recovery scan completed",
|
||||
extra={
|
||||
"pre_mint_balance": pre_mint_balance,
|
||||
"post_recovery_balance": post_recovery_balance,
|
||||
"balance_gained": balance_gained,
|
||||
"expected": minted_amount,
|
||||
},
|
||||
)
|
||||
if balance_gained < minted_amount:
|
||||
# Recovery scan ran but did NOT restore the orphaned proofs
|
||||
# (mint reports them as spent — they're stuck). Refuse to
|
||||
# credit the API key balance for proofs we don't actually hold.
|
||||
raise ValueError(
|
||||
f"Swap recovery failed: mint signed outputs but proofs are "
|
||||
f"unrecoverable (mint reports them spent). "
|
||||
f"Expected {minted_amount}, recovered {balance_gained}. "
|
||||
f"Local wallet DB ('.wallet/') state is corrupted — "
|
||||
f"the counter for keyset is stuck at a bad index range."
|
||||
)
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as recovery_err:
|
||||
logger.error(
|
||||
"swap_to_primary_mint: recovery failed",
|
||||
extra={"error": str(recovery_err)},
|
||||
)
|
||||
raise ValueError(
|
||||
f"Mint on primary failed and recovery unsuccessful: {e}"
|
||||
) from e
|
||||
else:
|
||||
logger.error(
|
||||
"swap_to_primary_mint: mint on primary failed after successful melt",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
"minted_amount": minted_amount,
|
||||
"mint_quote_id": mint_quote.quote,
|
||||
},
|
||||
)
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"swap_to_primary_mint: completed successfully",
|
||||
@@ -443,7 +536,7 @@ async def fetch_all_balances(
|
||||
"unit": unit,
|
||||
"wallet_balance": proofs_balance,
|
||||
"user_balance": user_balance,
|
||||
"owner_balance": proofs_balance - user_balance,
|
||||
"owner_balance": proofs_balance - user_balance if proofs_balance != 0 else 0,
|
||||
}
|
||||
return result
|
||||
except Exception as e:
|
||||
@@ -491,7 +584,9 @@ async def fetch_all_balances(
|
||||
total_wallet_balance_sats += proofs_balance_sats
|
||||
total_user_balance_sats += user_balance_sats
|
||||
|
||||
owner_balance = total_wallet_balance_sats - total_user_balance_sats
|
||||
owner_balance = 0
|
||||
if total_wallet_balance_sats != 0:
|
||||
owner_balance = total_wallet_balance_sats - total_user_balance_sats
|
||||
|
||||
return (
|
||||
balance_details,
|
||||
|
||||
177
tests/unit/test_count_tokens_local.py
Normal file
177
tests/unit/test_count_tokens_local.py
Normal file
@@ -0,0 +1,177 @@
|
||||
"""Unit tests for the local count_tokens shim.
|
||||
|
||||
The shim runs whenever an upstream that does not support Anthropic's
|
||||
``/v1/messages`` endpoint is asked for a token count. It must always
|
||||
return a 200 JSON ``{"input_tokens": N}`` response and must never raise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
from routstr.payment.models import Architecture, Model, Pricing
|
||||
from routstr.upstream import count_tokens as count_tokens_module
|
||||
from routstr.upstream.count_tokens import count_tokens_locally
|
||||
|
||||
|
||||
def _make_model(model_id: str = "anthropic/claude-3-5-sonnet") -> Model:
|
||||
pricing = Pricing(prompt=0.000003, completion=0.000015)
|
||||
architecture = Architecture(
|
||||
modality="text",
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"],
|
||||
tokenizer="cl100k_base",
|
||||
instruct_type=None,
|
||||
)
|
||||
return Model(
|
||||
id=model_id,
|
||||
name=model_id,
|
||||
created=0,
|
||||
description="",
|
||||
context_length=200_000,
|
||||
architecture=architecture,
|
||||
pricing=pricing,
|
||||
)
|
||||
|
||||
|
||||
def _body(payload: dict[str, Any]) -> bytes:
|
||||
return json.dumps(payload).encode()
|
||||
|
||||
|
||||
def _read_payload(response: Any) -> dict[str, Any]:
|
||||
body = response.body if isinstance(response.body, bytes) else bytes(response.body)
|
||||
return json.loads(body.decode())
|
||||
|
||||
|
||||
def test_returns_input_tokens_for_simple_messages() -> None:
|
||||
model = _make_model()
|
||||
request_body = _body(
|
||||
{
|
||||
"model": model.id,
|
||||
"messages": [{"role": "user", "content": "hello world"}],
|
||||
}
|
||||
)
|
||||
|
||||
response = count_tokens_locally(request_body, model)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.media_type == "application/json"
|
||||
payload = _read_payload(response)
|
||||
assert "input_tokens" in payload
|
||||
assert isinstance(payload["input_tokens"], int)
|
||||
assert payload["input_tokens"] >= 0
|
||||
|
||||
|
||||
def test_falls_back_to_estimator_when_litellm_raises() -> None:
|
||||
model = _make_model()
|
||||
request_body = _body(
|
||||
{
|
||||
"model": model.id,
|
||||
"messages": [{"role": "user", "content": "this is a longer message"}],
|
||||
}
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
count_tokens_module,
|
||||
"_count_with_litellm",
|
||||
side_effect=RuntimeError("boom"),
|
||||
):
|
||||
response = count_tokens_locally(request_body, model)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = _read_payload(response)
|
||||
assert payload["input_tokens"] >= 1
|
||||
|
||||
|
||||
def test_handles_missing_model_object() -> None:
|
||||
request_body = _body(
|
||||
{
|
||||
"model": "anthropic/claude-3-5-sonnet",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
)
|
||||
|
||||
response = count_tokens_locally(request_body, None)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = _read_payload(response)
|
||||
assert payload["input_tokens"] >= 0
|
||||
|
||||
|
||||
def test_handles_empty_request_body() -> None:
|
||||
response = count_tokens_locally(b"", _make_model())
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = _read_payload(response)
|
||||
assert payload["input_tokens"] >= 0
|
||||
|
||||
|
||||
def test_handles_malformed_json() -> None:
|
||||
response = count_tokens_locally(b"not-json", _make_model())
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = _read_payload(response)
|
||||
assert payload["input_tokens"] >= 0
|
||||
|
||||
|
||||
def test_includes_system_prompt_in_count() -> None:
|
||||
model = _make_model()
|
||||
short = _body(
|
||||
{
|
||||
"model": model.id,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
)
|
||||
with_system = _body(
|
||||
{
|
||||
"model": model.id,
|
||||
"system": "You are a helpful assistant with a long preamble " * 10,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
)
|
||||
|
||||
short_count = _read_payload(count_tokens_locally(short, model))["input_tokens"]
|
||||
long_count = _read_payload(count_tokens_locally(with_system, model))["input_tokens"]
|
||||
|
||||
assert long_count > short_count
|
||||
|
||||
|
||||
def test_supports_anthropic_system_block_list() -> None:
|
||||
model = _make_model()
|
||||
request_body = _body(
|
||||
{
|
||||
"model": model.id,
|
||||
"system": [{"type": "text", "text": "be terse" * 50}],
|
||||
"messages": [{"role": "user", "content": "ok"}],
|
||||
}
|
||||
)
|
||||
|
||||
response = count_tokens_locally(request_body, model)
|
||||
|
||||
payload = _read_payload(response)
|
||||
assert payload["input_tokens"] > 0
|
||||
|
||||
|
||||
def test_uses_forwarded_model_id_when_present() -> None:
|
||||
model = _make_model("anthropic/claude-3-5-sonnet")
|
||||
model.forwarded_model_id = "claude-3-5-sonnet-20241022"
|
||||
request_body = _body(
|
||||
{
|
||||
"model": "ignored",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
)
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture(model_name: str, body: dict[str, Any]) -> int:
|
||||
captured["model"] = model_name
|
||||
return 7
|
||||
|
||||
with patch.object(count_tokens_module, "_count_with_litellm", side_effect=_capture):
|
||||
response = count_tokens_locally(request_body, model)
|
||||
|
||||
assert captured["model"] == "claude-3-5-sonnet-20241022"
|
||||
assert _read_payload(response)["input_tokens"] == 7
|
||||
@@ -908,7 +908,7 @@ async def test_forward_request_skips_litellm_when_provider_supports_messages() -
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forward_request_skips_litellm_for_count_tokens() -> None:
|
||||
async def test_forward_request_handles_count_tokens_locally() -> None:
|
||||
provider = _make_provider()
|
||||
key = _make_key()
|
||||
model = _make_model()
|
||||
@@ -923,19 +923,25 @@ async def test_forward_request_skips_litellm_for_count_tokens() -> None:
|
||||
with patch.object(
|
||||
provider,
|
||||
"prepare_request_body",
|
||||
side_effect=RuntimeError("stop here"),
|
||||
side_effect=AssertionError("upstream should not be called"),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="stop here"):
|
||||
await provider.forward_request(
|
||||
request=request,
|
||||
path="messages/count_tokens",
|
||||
headers={},
|
||||
request_body=_anthropic_request_body(),
|
||||
key=key,
|
||||
max_cost_for_model=10_000,
|
||||
session=session,
|
||||
model_obj=model,
|
||||
)
|
||||
response = await provider.forward_request(
|
||||
request=request,
|
||||
path="messages/count_tokens",
|
||||
headers={},
|
||||
request_body=_anthropic_request_body(),
|
||||
key=key,
|
||||
max_cost_for_model=10_000,
|
||||
session=session,
|
||||
model_obj=model,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.body if isinstance(response.body, bytes) else bytes(response.body)
|
||||
payload = json.loads(body.decode())
|
||||
assert "input_tokens" in payload
|
||||
assert isinstance(payload["input_tokens"], int)
|
||||
assert payload["input_tokens"] >= 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1010,7 +1016,7 @@ async def test_forward_x_cashu_request_skips_litellm_when_native_messages() -> N
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forward_x_cashu_request_skips_litellm_for_count_tokens() -> None:
|
||||
async def test_forward_x_cashu_request_handles_count_tokens_locally() -> None:
|
||||
provider = _make_provider()
|
||||
model = _make_model()
|
||||
request = _make_request()
|
||||
@@ -1024,18 +1030,24 @@ async def test_forward_x_cashu_request_skips_litellm_for_count_tokens() -> None:
|
||||
with patch.object(
|
||||
provider,
|
||||
"prepare_request_body",
|
||||
side_effect=RuntimeError("stop here"),
|
||||
side_effect=AssertionError("upstream should not be called"),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="stop here"):
|
||||
await provider.forward_x_cashu_request(
|
||||
request=request,
|
||||
path="v1/messages/count_tokens",
|
||||
headers={},
|
||||
amount=5_000,
|
||||
unit="sat",
|
||||
max_cost_for_model=10_000,
|
||||
model_obj=model,
|
||||
)
|
||||
response = await provider.forward_x_cashu_request(
|
||||
request=request,
|
||||
path="v1/messages/count_tokens",
|
||||
headers={},
|
||||
amount=5_000,
|
||||
unit="sat",
|
||||
max_cost_for_model=10_000,
|
||||
model_obj=model,
|
||||
mint="https://mint",
|
||||
payment_token_hash="h",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.body if isinstance(response.body, bytes) else bytes(response.body)
|
||||
payload = json.loads(body.decode())
|
||||
assert "input_tokens" in payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -123,6 +123,7 @@ async def test_swap_to_primary_mint_insufficient_for_fees() -> None:
|
||||
mock_token_wallet = Mock()
|
||||
mock_token_wallet.load_mint = AsyncMock()
|
||||
mock_token_wallet.load_proofs = AsyncMock()
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=0)
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.load_mint = AsyncMock()
|
||||
@@ -166,6 +167,7 @@ async def test_swap_to_primary_mint_melt_error_wrapped() -> None:
|
||||
mock_token_wallet = Mock()
|
||||
mock_token_wallet.load_mint = AsyncMock()
|
||||
mock_token_wallet.load_proofs = AsyncMock()
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=0)
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.load_mint = AsyncMock()
|
||||
@@ -261,6 +263,7 @@ async def test_swap_to_primary_mint_success() -> None:
|
||||
mock_token_wallet = Mock()
|
||||
mock_token_wallet.load_mint = AsyncMock()
|
||||
mock_token_wallet.load_proofs = AsyncMock()
|
||||
mock_token_wallet.get_fees_for_proofs = Mock(return_value=0)
|
||||
|
||||
mock_primary_wallet = Mock()
|
||||
mock_primary_wallet.load_mint = AsyncMock()
|
||||
|
||||
Reference in New Issue
Block a user