mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-11-16 20:11:08 +00:00
264 lines
13 KiB
JavaScript
264 lines
13 KiB
JavaScript
"use strict";
|
|
|
|
const { randomUUID } = require("node:crypto");
|
|
const { PluginRpcError, RPC_ERRORS } = require("../rpcRouter.cjs");
|
|
const { assertPluginJsonValue } = require("../jsonBoundary.cjs");
|
|
const fail = (code, message) => new PluginRpcError(RPC_ERRORS[code], message);
|
|
const LIMITS = Object.freeze({ chats: 32, perRuntime: 2, turnMs: 60_000, chatMs: 300_000,
|
|
inputBytes: 16_384, outputBytes: 32_768, outputTokens: 2_048, requestsPerMinute: 8,
|
|
reservedTokensPerMinute: 65_536, contexts: 4, chunkBytes: 4_096, events: 8_192, rateOwners: 128 });
|
|
|
|
function boundedLimits(overrides) {
|
|
return Object.freeze(Object.fromEntries(Object.entries(LIMITS).map(([key, max]) => {
|
|
const value = overrides[key] ?? max;
|
|
if (!Number.isSafeInteger(value) || value < 1 || value > max) throw new TypeError(`Invalid ${key} limit`);
|
|
return [key, value];
|
|
})));
|
|
}
|
|
|
|
function abortable(promise, signal) {
|
|
signal.throwIfAborted();
|
|
return new Promise((resolve, reject) => {
|
|
const abort = () => reject(signal.reason);
|
|
signal.addEventListener("abort", abort, { once: true });
|
|
Promise.resolve(promise).then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
});
|
|
}
|
|
|
|
// In-memory, source-isolated handles. No AgentRuntime, tools, chat persistence,
|
|
// provider configuration, URLs, credentials or arbitrary system prompts cross it.
|
|
class ModelInvocationBroker {
|
|
constructor({ resolveDataOwner, selectModel, contexts, clock = Date.now, limits = {} }) {
|
|
if (typeof resolveDataOwner !== "function" || typeof selectModel !== "function") throw new TypeError("Host adapters required");
|
|
this.resolveDataOwner = resolveDataOwner;
|
|
this.selectModel = selectModel;
|
|
this.contexts = contexts;
|
|
this.clock = clock;
|
|
this.limits = boundedLimits(limits);
|
|
this.chats = new Map();
|
|
this.turns = new Map();
|
|
this.rates = new Map();
|
|
this.closed = false;
|
|
this.contextSubscription = contexts?.onDidInvalidate((handle) => {
|
|
for (const turn of [...this.turns.values()]) {
|
|
if (turn.references.some((reference) => reference.handle === handle)) {
|
|
this.#end(turn, fail("cancelled", "Selected context was revoked"));
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
async #binding(context) {
|
|
await context.assertActive();
|
|
const owner = await this.resolveDataOwner(context);
|
|
await context.assertActive();
|
|
if (!owner || (typeof owner !== "object" && typeof owner !== "string")) throw fail("permissionDenied", "Host data owner is unavailable");
|
|
return { owner, runtimeId: context.runtimeId, pluginVersion: context.pluginVersion };
|
|
}
|
|
|
|
async #ownedChat(chat, context) {
|
|
const binding = await this.#binding(context);
|
|
if (this.closed || this.chats.get(chat.handle) !== chat
|
|
|| chat.owner !== binding.owner || chat.runtimeId !== binding.runtimeId || chat.pluginVersion !== binding.pluginVersion) {
|
|
throw fail("permissionDenied", "Plugin chat is unavailable");
|
|
}
|
|
}
|
|
|
|
async #check(chat, context) {
|
|
await this.#ownedChat(chat, context);
|
|
if (!await chat.model.isCurrent()) throw fail("permissionDenied", "Selected model changed");
|
|
const currentOwner = await this.resolveDataOwner(context);
|
|
await context.assertActive();
|
|
if (currentOwner !== chat.owner || this.closed || this.chats.get(chat.handle) !== chat || chat.expiresAt <= this.clock()) {
|
|
throw fail("permissionDenied", "Plugin chat is unavailable");
|
|
}
|
|
}
|
|
|
|
#prune() {
|
|
for (const chat of [...this.chats.values()]) if (chat.expiresAt <= this.clock()) this.#close(chat);
|
|
for (const [owner, rate] of this.rates) if (rate.startedAt + 60_000 <= this.clock()) this.rates.delete(owner);
|
|
}
|
|
|
|
async open(context) {
|
|
const binding = await this.#binding(context);
|
|
// Host chooses from configured models. Plugin parameters cannot override it.
|
|
const model = await this.selectModel(context);
|
|
await context.assertActive();
|
|
if (!model || typeof model.invoke !== "function" || typeof model.isCurrent !== "function"
|
|
|| typeof model.label !== "string" || model.label.length > 160 || !await model.isCurrent()) {
|
|
throw fail("unavailable", "Choose a configured model in the host first");
|
|
}
|
|
await context.assertActive();
|
|
this.#prune();
|
|
if (this.closed || this.chats.size >= this.limits.chats
|
|
|| [...this.chats.values()].filter((chat) => chat.runtimeId === context.runtimeId).length >= this.limits.perRuntime) {
|
|
throw fail("resourceExhausted", "Plugin chat limit reached");
|
|
}
|
|
const chat = { ...binding, handle: randomUUID(), model, expiresAt: this.clock() + this.limits.chatMs, turn: null };
|
|
this.chats.set(chat.handle, chat);
|
|
return { chatHandle: chat.handle, model: model.label, expiresAt: chat.expiresAt };
|
|
}
|
|
|
|
async start({ chatHandle, prompt, contexts = [] }, context) {
|
|
assertPluginJsonValue({ chatHandle, prompt, contexts }, { maxBytes: this.limits.inputBytes + 2048 });
|
|
if (typeof prompt !== "string" || !prompt.trim() || prompt.includes("\0") || !Array.isArray(contexts)
|
|
|| contexts.length > this.limits.contexts || Buffer.byteLength(prompt) > this.limits.inputBytes) {
|
|
throw fail("invalidArgument", "Invalid model input");
|
|
}
|
|
const references = contexts.map((reference) => {
|
|
if (!reference || Array.isArray(reference) || Object.keys(reference).some((key) => !["handle", "version"].includes(key))
|
|
|| typeof reference.handle !== "string" || !reference.handle || reference.handle.length > 128
|
|
|| typeof reference.version !== "string" || !reference.version || reference.version.length > 128) {
|
|
throw fail("invalidArgument", "Invalid context reference");
|
|
}
|
|
return Object.freeze({ handle: reference.handle, version: reference.version });
|
|
});
|
|
const chat = this.chats.get(chatHandle);
|
|
if (!chat) throw fail("permissionDenied", "Plugin chat is unavailable");
|
|
await this.#check(chat, context);
|
|
if (chat.turn) throw fail("resourceExhausted", "A turn is already active");
|
|
const controller = new AbortController();
|
|
const turn = { handle: randomUUID(), chat, controller, reading: false, bytes: 0, sequence: 0, iterator: null, references: Object.freeze(references) };
|
|
chat.turn = turn;
|
|
this.turns.set(turn.handle, turn);
|
|
const abort = () => this.#end(turn, fail("cancelled", "Model turn cancelled"));
|
|
context.signal?.addEventListener("abort", abort, { once: true });
|
|
turn.cleanup = () => context.signal?.removeEventListener("abort", abort);
|
|
turn.timer = setTimeout(() => this.#end(turn, fail("deadlineExceeded", "Model turn timed out")), this.limits.turnMs);
|
|
try {
|
|
await this.#guard(turn, context);
|
|
const selected = [];
|
|
let bytes = Buffer.byteLength(prompt);
|
|
for (const reference of references) {
|
|
const result = await abortable(this.contexts.read(reference, { ...context, signal: controller.signal }), controller.signal);
|
|
bytes += Buffer.byteLength(result.text);
|
|
if (bytes > this.limits.inputBytes) throw fail("resourceExhausted", "Model input byte budget exceeded");
|
|
selected.push(result.text);
|
|
}
|
|
await this.#guard(turn, context);
|
|
this.#prune();
|
|
const rate = this.rates.get(chat.owner) ?? { startedAt: this.clock(), count: 0, tokens: 0 };
|
|
// Reserve conservatively using UTF-8 input bytes plus the output cap. Do
|
|
// not refund unknown provider usage after cancellation or disconnect.
|
|
const reserved = bytes + 1_024 + this.limits.outputTokens;
|
|
if ((!this.rates.has(chat.owner) && this.rates.size >= this.limits.rateOwners) || rate.count >= this.limits.requestsPerMinute || rate.tokens + reserved > this.limits.reservedTokensPerMinute) {
|
|
throw fail("resourceExhausted", "Model request/token budget reached");
|
|
}
|
|
rate.count++; rate.tokens += reserved; this.rates.set(chat.owner, rate);
|
|
await this.#guard(turn, context);
|
|
const iterator = await abortable(chat.model.invoke({ prompt, contexts: selected, signal: controller.signal,
|
|
maxOutputTokens: this.limits.outputTokens, maxChunkBytes: this.limits.chunkBytes }), controller.signal);
|
|
await this.#guard(turn, context);
|
|
if (!iterator || typeof iterator.next !== "function") throw fail("internal", "Model adapter failed");
|
|
turn.iterator = iterator;
|
|
return { turnHandle: turn.handle, maxOutputTokens: this.limits.outputTokens, reservedTokens: reserved };
|
|
} catch (error) {
|
|
this.#end(turn, error);
|
|
throw this.#safe(error);
|
|
}
|
|
}
|
|
|
|
async #guard(turn, context) {
|
|
turn.controller.signal.throwIfAborted();
|
|
await abortable(this.#check(turn.chat, context), turn.controller.signal);
|
|
for (const reference of turn.references) {
|
|
await abortable(this.contexts.assertCurrent(reference, { ...context, signal: turn.controller.signal }), turn.controller.signal);
|
|
}
|
|
turn.controller.signal.throwIfAborted();
|
|
if (this.turns.get(turn.handle) !== turn || turn.chat.turn !== turn) throw fail("cancelled", "Model turn ended");
|
|
}
|
|
|
|
#safe(error) {
|
|
// SDK/provider errors may contain headers, URLs or echoed request content.
|
|
return error instanceof PluginRpcError ? error : fail("unavailable", "Model invocation failed");
|
|
}
|
|
|
|
async read(turnHandle, context) {
|
|
const turn = this.turns.get(turnHandle);
|
|
if (!turn) throw fail("permissionDenied", "Model turn is unavailable");
|
|
// Validate owner before mutating/aborting another caller's turn.
|
|
await this.#ownedChat(turn.chat, context);
|
|
if (turn.reading || !turn.iterator) throw fail("resourceExhausted", "A model read is already pending");
|
|
turn.reading = true;
|
|
const abort = () => this.#end(turn, fail("cancelled", "Model read cancelled"));
|
|
context.signal?.addEventListener("abort", abort, { once: true });
|
|
try {
|
|
if (context.signal?.aborted) abort();
|
|
await this.#guard(turn, context);
|
|
if (turn.sequence >= this.limits.events) throw fail("resourceExhausted", "Model event budget exceeded");
|
|
const next = await abortable(turn.iterator.next(), turn.controller.signal);
|
|
await this.#guard(turn, context);
|
|
if (next.done) {
|
|
this.#end(turn);
|
|
return { kind: "end", sequence: turn.sequence++ };
|
|
}
|
|
const event = next.value;
|
|
if (event?.kind === "text" && typeof event.text === "string") {
|
|
const bytes = Buffer.byteLength(event.text);
|
|
if (bytes > this.limits.chunkBytes || turn.bytes + bytes > this.limits.outputBytes) {
|
|
throw fail("resourceExhausted", "Model output byte budget exceeded");
|
|
}
|
|
turn.bytes += bytes;
|
|
return { kind: "text", sequence: turn.sequence++, text: event.text };
|
|
}
|
|
if (event?.kind === "usage") {
|
|
const usage = {};
|
|
for (const key of ["inputTokens", "outputTokens", "totalTokens"]) {
|
|
if (event[key] !== undefined) {
|
|
if (!Number.isSafeInteger(event[key]) || event[key] < 0) throw fail("internal", "Invalid model usage");
|
|
usage[key] = event[key];
|
|
}
|
|
}
|
|
if ((usage.outputTokens ?? 0) > this.limits.outputTokens) throw fail("resourceExhausted", "Model output token budget exceeded");
|
|
return { kind: "usage", sequence: turn.sequence++, ...usage };
|
|
}
|
|
// A model's tool request is data, never host authorization. This phase
|
|
// exposes no execution path, and drops raw tool payloads/reasoning/errors.
|
|
throw fail("permissionDenied", "Model tools and unsupported events are disabled");
|
|
} catch (error) {
|
|
this.#end(turn, error);
|
|
throw this.#safe(error);
|
|
} finally { turn.reading = false; context.signal?.removeEventListener("abort", abort); }
|
|
}
|
|
|
|
async stop(turnHandle, context) {
|
|
const turn = this.turns.get(turnHandle);
|
|
if (!turn) return false;
|
|
await this.#ownedChat(turn.chat, context);
|
|
this.#end(turn, fail("cancelled", "Model turn cancelled"));
|
|
return true;
|
|
}
|
|
|
|
#end(turn, reason = fail("cancelled", "Model turn ended")) {
|
|
if (this.turns.get(turn.handle) !== turn) return;
|
|
this.turns.delete(turn.handle);
|
|
if (turn.chat.turn === turn) turn.chat.turn = null;
|
|
clearTimeout(turn.timer); turn.cleanup?.(); turn.controller.abort(reason);
|
|
// Never wait for a provider which ignores cancellation. Its late next()
|
|
// result is observed by abortable() and cannot re-enter the output path.
|
|
try { Promise.resolve(turn.iterator?.return?.()).catch(() => {}); } catch {}
|
|
turn.iterator = null;
|
|
}
|
|
|
|
#close(chat) {
|
|
if (chat.turn) this.#end(chat.turn);
|
|
this.chats.delete(chat.handle);
|
|
}
|
|
async close(chatHandle, context) {
|
|
const chat = this.chats.get(chatHandle);
|
|
if (!chat) return false;
|
|
await this.#ownedChat(chat, context); this.#close(chat); return true;
|
|
}
|
|
releaseRuntime(runtimeId) {
|
|
for (const chat of [...this.chats.values()]) if (chat.runtimeId === runtimeId) this.#close(chat);
|
|
this.contexts?.releaseRuntime(runtimeId);
|
|
}
|
|
releaseOwner(owner) {
|
|
for (const chat of [...this.chats.values()]) if (chat.owner === owner) this.#close(chat);
|
|
this.contexts?.releaseOwner(owner);
|
|
}
|
|
shutdown() { this.closed = true; for (const chat of [...this.chats.values()]) this.#close(chat); this.rates.clear(); this.contextSubscription?.dispose(); }
|
|
}
|
|
|
|
module.exports = { ModelInvocationBroker, MODEL_INVOCATION_LIMITS: LIMITS };
|