fix(ai): keep only image attachments when preserving request parts

Chat providers reject non-image file parts such as text/plain. Routine
compression now leaves image parts in place and still replaces every other
file with the omission placeholder.

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>

infrastructure/ai/harness/contextManager.test.ts

@@ -363,3 +363,56 @@ test('prepareTurnContext calls summarize when over dynamic threshold', async ()
   assert.equal(prepared.trace?.didLlmSummarize, true);
   assert.match(JSON.stringify(prepared.messages), /summary of earlier work/);
 });
+
+test('normal turn and step preparation keep user attachment parts', async () => {
+  const messages: ModelMessage[] = [
+    {
+      role: 'user',
+      content: [
+        { type: 'text', text: 'what does this image say?' },
+        { type: 'file', data: 'QUJD'.repeat(750), mediaType: 'image/png' },
+      ],
+    },
+  ];
+
+  const turn = await prepareTurnContext({
+    messages,
+    backend: 'catty',
+    contextWindow: 1_300_000,
+    trigger: 'pre-turn',
+  });
+  const step = await prepareStepContext({
+    messages: turn.messages,
+    contextWindow: 1_300_000,
+    stepNumber: 0,
+  });
+
+  for (const prepared of [turn, step]) {
+    const content = prepared.messages.find(message => message.role === 'user')?.content;
+    assert.ok(Array.isArray(content));
+    assert.equal(content.some(part => (part as { type?: string }).type === 'file'), true);
+    assert.doesNotMatch(JSON.stringify(prepared.messages), /attachment omitted to keep the AI request small/);
+  }
+});
+
+test('413 retry still trades attachments for a text placeholder', async () => {
+  const messages: ModelMessage[] = [
+    {
+      role: 'user',
+      content: [
+        { type: 'text', text: 'what does this image say?' },
+        { type: 'file', data: 'QUJD'.repeat(750), mediaType: 'image/png' },
+      ],
+    },
+  ];
+
+  const prepared = await prepareTurnContext({
+    messages,
+    backend: 'catty',
+    contextWindow: 1_300_000,
+    trigger: '413-retry',
+    compressForRequestTooLargeRetry: true,
+  });
+
+  assert.match(JSON.stringify(prepared.messages), /attachment omitted to keep the AI request small/);
+});

infrastructure/ai/harness/contextManager.ts

@@ -7,7 +7,10 @@ import {
   keepRecentContextMessages,
   prepareContextCompaction,
 } from '../contextCompaction';
-import { compressMessagesForRequestTooLargeRetry } from '../requestPayloadCompression';
+import {
+  compressMessagesForRequestTooLargeRetry,
+  type CompressMessagesForRequestTooLargeRetryOptions,
+} from '../requestPayloadCompression';
 import {
   computeCompactionThreshold,
   computeTotalInputTokens,
@@ -96,11 +99,14 @@ function emitCompactionStart(
   });
 }
 
-function applyTypedMessageCompression(messages: ModelMessage[]): {
+function applyTypedMessageCompression(
+  messages: ModelMessage[],
+  options?: CompressMessagesForRequestTooLargeRetryOptions,
+): {
   messages: ModelMessage[];
   didAdjust: boolean;
 } {
-  const compressed = compressMessagesForRequestTooLargeRetry(messages);
+  const compressed = compressMessagesForRequestTooLargeRetry(messages, options);
   return { messages: compressed.messages, didAdjust: compressed.didAdjust };
 }
 
@@ -225,7 +231,9 @@ export async function prepareTurnContext(
     did413Fallback = typed.didAdjust;
     didAdjust = didAdjust || typed.didAdjust;
   } else {
-    const typed = applyTypedMessageCompression(working);
+    // Routine pre-turn pass: keep the user's attachment parts. Only the explicit
+    // 413 retry above may trade them for text placeholders.
+    const typed = applyTypedMessageCompression(working, { preserveAttachments: true });
     if (typed.didAdjust) {
       working = typed.messages;
       didTypedCompression = true;
@@ -412,7 +420,9 @@ export async function prepareStepContext(
   });
   let working = stale.messages;
 
-  const typed = compressMessagesForRequestTooLargeRetry(working);
+  // Step-level typed compression runs before every model call, so it must keep
+  // multimodal parts: the user's image would otherwise never reach the model.
+  const typed = compressMessagesForRequestTooLargeRetry(working, { preserveAttachments: true });
   working = typed.messages;
 
   const budgetGuard = applyStepBudgetGuard(working, {

infrastructure/ai/requestPayloadCompression.test.ts

@@ -90,3 +90,77 @@ test("compressMessagesForRequestTooLargeRetry reports no adjustment for compact
   assert.equal(result.didAdjust, false);
   assert.deepEqual(result.messages, messages);
 });
+
+test("compressMessagesForRequestTooLargeRetry keeps attachments when preserveAttachments is set", () => {
+  const imageData = "A".repeat(300_000);
+  const fileData = Buffer.from("label,hostname\nprod,prod.example.com\n").toString("base64");
+  const messages: ModelMessage[] = [
+    {
+      role: "user",
+      content: [
+        { type: "text", text: "what does this screenshot say?" },
+        { type: "file", data: imageData, mediaType: "image/png" },
+      ],
+    },
+    {
+      role: "user",
+      content: [
+        { type: "text", text: "import these hosts" },
+        { type: "file", data: fileData, mediaType: "text/csv", filename: "hosts.csv" },
+      ],
+    },
+    {
+      role: "tool",
+      content: [{
+        type: "tool-result",
+        toolCallId: "call-1",
+        toolName: "terminal_execute",
+        output: { type: "text", value: "X".repeat(200_000) },
+      }],
+    },
+  ];
+
+  const result = compressMessagesForRequestTooLargeRetry(messages, { preserveAttachments: true });
+
+  // Text and tool output are still reduced. Images stay; other files become placeholders.
+  assert.equal(result.didAdjust, true);
+
+  const imageContent = result.messages[0].content;
+  assert.ok(Array.isArray(imageContent));
+  assert.deepEqual(imageContent[1], messages[0].content[1]);
+  assert.doesNotMatch(JSON.stringify(imageContent), /attachment omitted to keep the AI request small/);
+
+  const fileContent = result.messages[1].content;
+  assert.ok(Array.isArray(fileContent));
+  assert.deepEqual(fileContent[1], {
+    type: "text",
+    text: `[file attachment omitted to keep the AI request small: filename=hosts.csv, mediaType=text/csv, ${fileData.length} chars]`,
+  });
+
+  const toolContent = result.messages[2].content;
+  assert.ok(Array.isArray(toolContent));
+  const toolPart = toolContent[0] as { output?: { value?: string } };
+  assert.ok((toolPart.output?.value?.length ?? 0) < 5_000);
+});
+
+test("compressMessagesForRequestTooLargeRetry still placeholders text/plain files when preserveAttachments is set", () => {
+  const text = "hello from a text file";
+  const messages: ModelMessage[] = [
+    {
+      role: "user",
+      content: [
+        { type: "text", text: "read this note" },
+        { type: "file", data: text, mediaType: "text/plain", filename: "note.txt" },
+      ],
+    },
+  ];
+
+  const result = compressMessagesForRequestTooLargeRetry(messages, { preserveAttachments: true });
+  const content = result.messages[0].content;
+  assert.ok(Array.isArray(content));
+  assert.deepEqual(content[0], { type: "text", text: "read this note" });
+  assert.deepEqual(content[1], {
+    type: "text",
+    text: `[file attachment omitted to keep the AI request small: filename=note.txt, mediaType=text/plain, ${text.length} chars]`,
+  });
+});

infrastructure/ai/requestPayloadCompression.ts

@@ -16,6 +16,18 @@ export interface CompressMessagesForRequestTooLargeRetryResult {
   didAdjust: boolean;
 }
 
+export interface CompressMessagesForRequestTooLargeRetryOptions {
+  /**
+   * Keep image attachments (`image` parts and `file` parts whose media type
+   * starts with `image/`) instead of replacing them with a text placeholder.
+   *
+   * Other file parts stay placeholders. Chat providers reject types such as
+   * `text/plain`, and the model reads those through attachment tools.
+   * Only the 413 retry drops images as well.
+   */
+  preserveAttachments?: boolean;
+}
+
 /**
  * Collapse noisy terminal/build output.
  * Keeps semantics while removing repeated blank lines and long duplicate runs.
@@ -108,10 +120,12 @@ export function truncateTextWithHeadAndTail(
 
 export function compressMessagesForRequestTooLargeRetry(
   messages: ModelMessage[],
+  options: CompressMessagesForRequestTooLargeRetryOptions = {},
 ): CompressMessagesForRequestTooLargeRetryResult {
+  const preserveAttachments = options.preserveAttachments === true;
   let didAdjust = false;
   const compressedMessages = messages.map((message) => {
-    const compressed = compressModelMessageForRequestRetry(message);
+    const compressed = compressModelMessageForRequestRetry(message, preserveAttachments);
     if (compressed !== message) didAdjust = true;
     return compressed;
   });
@@ -122,7 +136,10 @@ export function compressMessagesForRequestTooLargeRetry(
   };
 }
 
-function compressModelMessageForRequestRetry(message: ModelMessage): ModelMessage {
+function compressModelMessageForRequestRetry(
+  message: ModelMessage,
+  preserveAttachments: boolean,
+): ModelMessage {
   if (typeof message.content === "string") {
     // User text is an instruction or exact input, not terminal output. Lossy
     // cleanup can alter commands even when the request is far below its limit.
@@ -136,7 +153,7 @@ function compressModelMessageForRequestRetry(message: ModelMessage): ModelMessag
   let didAdjust = false;
   const content = message.content.map((part) => {
     if (message.role === "user" && part.type === "text") return part;
-    const compressed = compressContentPartForRequestRetry(part);
+    const compressed = compressContentPartForRequestRetry(part, preserveAttachments);
     if (compressed !== part) didAdjust = true;
     return compressed;
   });
@@ -144,7 +161,7 @@ function compressModelMessageForRequestRetry(message: ModelMessage): ModelMessag
   return didAdjust ? ({ ...message, content } as ModelMessage) : message;
 }
 
-function compressContentPartForRequestRetry(part: unknown): unknown {
+function compressContentPartForRequestRetry(part: unknown, preserveAttachments: boolean): unknown {
   if (!part || typeof part !== "object") return part;
   const record = part as Record<string, unknown>;
   const type = record.type;
@@ -178,12 +195,18 @@ function compressContentPartForRequestRetry(part: unknown): unknown {
     }
   }
 
-  if (type === "image" && typeof record.image === "string") {
-    return omittedAttachmentTextPart("image", record.image, record);
-  }
+  const mediaType = typeof record.mediaType === "string" ? record.mediaType : "";
+  const preserveImage = preserveAttachments && (
+    type === "image" || (type === "file" && mediaType.startsWith("image/"))
+  );
+  if (!preserveImage) {
+    if (type === "image" && typeof record.image === "string") {
+      return omittedAttachmentTextPart("image", record.image, record);
+    }
 
-  if (type === "file" && typeof record.data === "string") {
-    return omittedAttachmentTextPart("file", record.data, record);
+    if (type === "file" && typeof record.data === "string") {
+      return omittedAttachmentTextPart("file", record.data, record);
+    }
   }
 
   return part;