Bind local chat to engine context limits

capsules/assistant/browser/agent-context.js

@@ -84,7 +84,7 @@ export function resolveReasoningPolicy(requested, caps = TEXT_CONTRACT_CAPABILIT
 /** Char/4 estimator — not the provider tokenizer. Pack against a margin, not the raw cap. */
 export const TOKEN_ESTIMATOR = "chars/4";
 const ESTIMATOR_HEADROOM = 0.92;
-const MIN_VIABLE_ESTIMATED_INPUT = 512;
+const MIN_VIABLE_ESTIMATED_INPUT = 32;
 
 export class ContextOverflowError extends Error {
   constructor(message) {
@@ -94,6 +94,21 @@ export class ContextOverflowError extends Error {
   }
 }
 
+export function capabilitiesForOffer(offer, capabilities = TEXT_CONTRACT_CAPABILITIES) {
+  if (!offer?.context && offer?.routeKind !== "local") return capabilities;
+  const contextWindow = offer.context?.context_window_tokens;
+  const outputLimit = offer.context?.max_output_tokens;
+  if (!Number.isInteger(contextWindow) || contextWindow <= 0 ||
+      !Number.isInteger(outputLimit) || outputLimit <= 0 || outputLimit >= contextWindow) {
+    throw new ContextOverflowError("Selected local model has incomplete context limits");
+  }
+  return { ...capabilities, context: {
+    ...capabilities.context,
+    maxInputTokens: contextWindow,
+    maxOutputTokens: outputLimit,
+  } };
+}
+
 export function approxTokens(text) {
   return Math.ceil(String(text || "").length / 4);
 }
@@ -793,6 +808,7 @@ export function compileContext({
     tokenEstimator: TOKEN_ESTIMATOR,
     estimatorHeadroom: tokenBudget.estimatorHeadroom,
     estimatedInputTokens: used,
+    outputReserve: tokenBudget.outputReserve,
     estimatedSystemTokens: systemTokens,
     estimatedRuntimeTokens: runtimeTokens,
     estimatedCurrentInputTokens: currentInputTokens,

capsules/assistant/browser/agent-live.js

@@ -18,6 +18,7 @@ import {
 } from "./model-contract.js";
 import {
   compileContext,
+  capabilitiesForOffer,
   attachProviderPayload,
   splitSessionMessages,
   makeBudget,
@@ -324,6 +325,8 @@ export function compileLiveContext({
   degradedFallback = false,
   debug = false,
 } = {}) {
+  const offer = selectedLiveOffer();
+  const effectiveCapabilities = capabilitiesForOffer(offer, capabilities);
   const raw = sessionMessages || session?.messages || [];
   let { history, currentInput: splitCurrent } = splitSessionMessages(raw);
   const current = currentInput || splitCurrent;
@@ -365,12 +368,13 @@ export function compileLiveContext({
     currentInput: current,
     constraints,
     intent: "",
-    budget: makeBudget(capabilities.context, clampLiveMaxTokens(maxTokens)),
-    capabilities,
+    budget: makeBudget(effectiveCapabilities.context, clampLiveMaxTokens(maxTokens)),
+    capabilities: effectiveCapabilities,
     transcriptChars: transcriptCharCount(raw),
     thinkingChars,
   });
   attachProviderPayload(compiled);
+  compiled.manifest.offerId = offer?.offerId || "";
   logContextManifest(compiled.manifest, compiled.invariants, { debug });
   return compiled;
 }
@@ -523,6 +527,10 @@ export async function streamChatViaContract(
     activeRun = null;
     throw contractError("no_model_offers", "no text model offer on this Home");
   }
+  if (!turn.providerRunId && manifest.offerId && offer?.offerId !== manifest.offerId) {
+    activeRun = null;
+    throw contractError("model_selection_changed", "Selected model changed while context was prepared");
+  }
   const resuming = Boolean(turn.providerRunId);
   let created;
   let runId;
@@ -535,7 +543,10 @@ export async function streamChatViaContract(
   try {
     created = resuming
       ? await modelRunCall("runs_get", { run_id: turn.providerRunId, request_id: newRequestId() })
-      : await modelRunCall("runs_create", textRunCreateBody({ offer, messages, requestId: createRequestId }));
+      : await modelRunCall("runs_create", textRunCreateBody({
+          offer, messages, requestId: createRequestId,
+          maxOutputTokens: manifest.outputReserve,
+        }));
     runId = typeof created?.run_id === "string" ? created.run_id : "";
     if (!runId || runId.trim() !== runId || runId.length > 80 || (resuming && runId !== turn.providerRunId)) {
       throw contractError("no_run_id", "contract returned no run id");

capsules/assistant/browser/model-contract.js

@@ -285,7 +285,9 @@ export function textOfferRows(offers, backendReports = {}) {
       label: offer.title,
       detail: offerRowDetail(offer, facts),
       streamOutput: offer.stream_output === true,
+      routeKind: offerRouteKind(offer),
       inputSchemas: Array.isArray(offer.input_schemas) ? [...offer.input_schemas] : [],
+      context: offer.context && typeof offer.context === "object" ? { ...offer.context } : null,
       selectionFacts: facts,
     };
   });
@@ -315,29 +317,33 @@ export function transcriptPrompt(messages) {
   return lines.join("\n\n");
 }
 
-export function textRunCreateBody({ offer, messages, requestId }) {
+export function textRunCreateBody({ offer, messages, requestId, maxOutputTokens = null }) {
   if (!offer || typeof offer.offerId !== "string" || typeof offer.operation !== "string") {
     throw contractError("no_offer", "no text model offer selected");
   }
   if (typeof requestId !== "string" || requestId.trim() === "") {
     throw contractError("no_request_id", "run request needs a request id");
   }
+  const input = offer.inputSchemas?.includes(MODEL_TEXT_INPUT_V2_SCHEMA)
+    ? {
+        schema: MODEL_TEXT_INPUT_V2_SCHEMA,
+        messages: messages.map(({ role, content }) => ({
+          role: role === "agent" ? "assistant" : role,
+          content,
+        })),
+      }
+    : {
+        schema: MODEL_TEXT_INPUT_SCHEMA,
+        prompt: transcriptPrompt(messages),
+      };
+  if (offer.context && Number.isInteger(maxOutputTokens) && maxOutputTokens > 0) {
+    input.max_output_tokens = maxOutputTokens;
+  }
   return {
     offer_id: offer.offerId,
     operation: offer.operation,
     request_id: requestId,
-    input: offer.inputSchemas?.includes(MODEL_TEXT_INPUT_V2_SCHEMA)
-      ? {
-          schema: MODEL_TEXT_INPUT_V2_SCHEMA,
-          messages: messages.map(({ role, content }) => ({
-            role: role === "agent" ? "assistant" : role,
-            content,
-          })),
-        }
-      : {
-          schema: MODEL_TEXT_INPUT_SCHEMA,
-          prompt: transcriptPrompt(messages),
-        },
+    input,
   };
 }
 

capsules/model-provider/src/adapters.rs

@@ -476,6 +476,7 @@ enum LocalTextBackend {
         engine: LocalArtifactConfig,
         model: LocalArtifactConfig,
         settings: LocalLlamaSettings,
+        requested_output_tokens: Option<u32>,
     },
 }
 
@@ -1155,6 +1156,7 @@ impl AdapterExecutor for LiveAdapterExecutor {
                     engine: engine.clone(),
                     model: model.clone(),
                     settings: settings.clone(),
+                    requested_output_tokens: requested_local_output_tokens(input, settings)?,
                 },
                 offer,
                 binding,
@@ -1298,6 +1300,14 @@ fn validate_text_messages(
     offer: &ConfiguredOffer,
     input: &Value,
 ) -> std::result::Result<Vec<Value>, AdapterFault> {
+    if input.get("max_output_tokens").is_some()
+        && !matches!(offer.adapter, AdapterConfig::LocalLlamaCppText { .. })
+    {
+        return Err(AdapterFault::context(
+            "model output request is not supported by this offer",
+            "max_output_tokens is supported only by local llama offers",
+        ));
+    }
     let invalid = || {
         AdapterFault::context(
             "model input is invalid",
@@ -1327,6 +1337,28 @@ fn validate_text_messages(
     }
 }
 
+fn requested_local_output_tokens(
+    input: &Value,
+    settings: &LocalLlamaSettings,
+) -> std::result::Result<Option<u32>, AdapterFault> {
+    let Some(value) = input.get("max_output_tokens") else {
+        return Ok(None);
+    };
+    let requested = value.as_u64().ok_or_else(|| {
+        AdapterFault::context(
+            "model output request is invalid",
+            "max_output_tokens must be a positive integer",
+        )
+    })?;
+    if requested == 0 || requested > u64::from(crate::config::local_output_token_limit(settings)) {
+        return Err(AdapterFault::context(
+            "model output request exceeds the profile",
+            "max_output_tokens exceeds local model limit",
+        ));
+    }
+    Ok(Some(requested as u32))
+}
+
 pub(crate) fn is_local_text_backend_state(value: &Value) -> bool {
     value
         .get("schema")
@@ -1684,7 +1716,7 @@ async fn run_local_text_worker_inner(
     )
     .then(HostedBackendReport::default);
     let mut local_socket = None;
-    let (api_url, api_key, body, private_endpoint) = match &task.backend {
+    let (api_url, api_key, body, private_endpoint, context_window_tokens) = match &task.backend {
         LocalTextBackend::OpenRouterDecisions {
             api_url,
             api_key,
@@ -1714,6 +1746,7 @@ async fn run_local_text_worker_inner(
             api_key.clone(),
             text_generation_request_body(&task.offer, model, &task.messages, None),
             false,
+            None,
         ),
         LocalTextBackend::OpenAiResponses {
             api_url,
@@ -1736,13 +1769,15 @@ async fn run_local_text_worker_inner(
                     })?,
             ),
             false,
+            None,
         ),
         LocalTextBackend::LocalLlama {
             engines,
             offer_id,
             engine,
             model,
             settings,
+            requested_output_tokens,
         } => {
             let endpoint = engines
                 .endpoint_with_timeout(
@@ -1754,14 +1789,22 @@ async fn run_local_text_worker_inner(
                 )
                 .await
                 .map_err(map_local_llama_fault)?;
-            let body = text_generation_request_body(
+            let mut body = text_generation_request_body(
                 &task.offer,
                 &endpoint.model,
                 &task.messages,
                 Some(endpoint.enable_thinking),
             );
+            body["max_tokens"] = json!(requested_output_tokens
+                .unwrap_or_else(|| { crate::config::local_output_token_limit(settings) }));
             local_socket = endpoint.unix_socket;
-            (endpoint.api_url, None, body, true)
+            (
+                endpoint.api_url,
+                None,
+                body,
+                true,
+                Some(settings.context_size / settings.parallel),
+            )
         }
     };
     let client = backend_client_with_socket(
@@ -1776,6 +1819,52 @@ async fn run_local_text_worker_inner(
     } else {
         task.hosted_socket.as_deref()
     };
+    if let Some(context_window_tokens) = context_window_tokens {
+        let count_url = format!("{api_url}/input_tokens");
+        let count_request = client
+            .post(count_url)
+            .header("content-type", "application/json")
+            .json(&body);
+        let counted = tokio::select! {
+            _ = task.cancel_rx.changed() => return Ok(worker_settlement_unknown_result()),
+            response = count_request.send() => response.map_err(|err| map_text_reqwest_failure(err, true))?,
+        };
+        if !counted.status().is_success() {
+            return Err(map_effect_http_status(&counted, None));
+        }
+        let count = tokio::select! {
+            _ = task.cancel_rx.changed() => return Ok(worker_settlement_unknown_result()),
+            response = read_bounded_json_response_async(counted) => response?,
+        };
+        let input_tokens = count["input_tokens"]
+            .as_u64()
+            .filter(|count| *count > 0)
+            .ok_or_else(|| {
+                AdapterFault::malformed(
+                    "model backend returned invalid data",
+                    "missing chat input token count",
+                )
+            })?;
+        let output_tokens = body["max_tokens"].as_u64().unwrap_or(0);
+        if input_tokens.saturating_add(output_tokens) >= u64::from(context_window_tokens) {
+            return Err(AdapterFault::context(
+                "model context is full",
+                format!("templated input {input_tokens} plus output {output_tokens} exceeds context {context_window_tokens}"),
+            ));
+        }
+    }
+    // The count and generation share the one run deadline.
+    let client = if context_window_tokens.is_some() {
+        backend_client_with_socket(
+            remaining_run_timeout(task.deadline_ms)?
+                .as_millis()
+                .try_into()
+                .unwrap_or(u64::MAX),
+            local_socket.as_deref(),
+        )?
+    } else {
+        client
+    };
     let request = {
         let effect = if matches!(&task.backend, LocalTextBackend::OpenAiResponses { .. }) {
             "responses"

capsules/model-provider/src/config.rs

@@ -412,6 +412,10 @@ impl LocalLlamaSettings {
             MAX_LOCAL_LLAMA_PARALLEL,
             "local llama parallel",
         )?;
+        anyhow::ensure!(
+            self.context_size / self.parallel >= 128,
+            "local llama context_size must provide at least 128 tokens per slot"
+        );
         validate_nonzero_bounded(self.threads, MAX_LOCAL_LLAMA_THREADS, "local llama threads")?;
         validate_nonzero_bounded(
             self.batch_threads,
@@ -579,6 +583,15 @@ impl ConfiguredOffer {
             output_modalities: self.output_modalities.clone(),
             stream_output: self.adapter.stream_output(),
             policy: self.policy.summary(),
+            context: match &self.adapter {
+                AdapterConfig::LocalLlamaCppText { settings, .. } => {
+                    Some(crate::contract::LocalContextLimits {
+                        context_window_tokens: settings.context_size / settings.parallel,
+                        max_output_tokens: local_output_token_limit(settings),
+                    })
+                }
+                _ => None,
+            },
             hosted,
         }
     }
@@ -588,6 +601,7 @@ impl ConfiguredOffer {
     pub fn execution_summary(&self) -> OfferSummary {
         let mut summary = self.summary();
         summary.input_schemas.clear();
+        summary.context = None;
         summary
     }
 
@@ -711,6 +725,12 @@ impl ConfiguredOffer {
     }
 }
 
+pub(crate) fn local_output_token_limit(settings: &LocalLlamaSettings) -> u32 {
+    // Keep a conservative answer slice in each server slot. Output bytes have
+    // their own independent transport bound.
+    (settings.context_size / settings.parallel / 4).clamp(1, 1_024)
+}
+
 pub fn journal_root(base_path: &str, configured: Option<&str>) -> Result<PathBuf> {
     if let Some(configured) = configured {
         validate_bounded_trimmed(configured, "journal_dir", MAX_JOURNAL_DIR_BYTES)?;
@@ -1261,6 +1281,27 @@ mod tests {
         assert_ne!(responses.execution_binding_hash().unwrap(), legacy_hash);
     }
 
+    #[test]
+    fn local_context_summary_uses_one_slot_and_rejects_unusable_profiles() {
+        let mut offer = local_llama_offer(Path::new("/tmp/engine"), Path::new("/tmp/model"));
+        let original_revision = offer.execution_binding_hash().unwrap();
+        let limits = offer.summary().context.unwrap();
+        assert_eq!(limits.context_window_tokens, 4_096);
+        assert_eq!(limits.max_output_tokens, 1_024);
+        if let AdapterConfig::LocalLlamaCppText { settings, .. } = &mut offer.adapter {
+            settings.parallel = 16;
+            assert_eq!(settings.context_size / settings.parallel, 256);
+            settings.validate().unwrap();
+            settings.parallel = 64;
+            assert!(settings.validate().is_err());
+            settings.parallel = 16;
+        }
+        let limits = offer.summary().context.unwrap();
+        assert_eq!(limits.context_window_tokens, 256);
+        assert_eq!(limits.max_output_tokens, 64);
+        assert_ne!(offer.execution_binding_hash().unwrap(), original_revision);
+    }
+
     #[test]
     fn execution_binding_hash_changes_only_for_semantic_execution_inputs() {
         let openai = base_offer();