feat: 增加共享 Agent Browser 调试能力
需求：在 AI 编程会话中让用户与 Agent 共享同一浏览器页面，并查看控制台与网络信息。 实现：新增沙箱浏览器内核、Host API/渲染器面板、OpenCode 工具接入及安全边界测试。
This commit is contained in:
409
src/lib/agent-browser.ts
Normal file
409
src/lib/agent-browser.ts
Normal file
@@ -0,0 +1,409 @@
|
||||
import { hostApiFetch } from '@/lib/host-api';
|
||||
import type {
|
||||
AgentBrowserBounds,
|
||||
AgentBrowserCdpEvent,
|
||||
AgentBrowserCdpEventPage,
|
||||
AgentBrowserCdpResult,
|
||||
AgentBrowserPayloadChunk,
|
||||
AgentBrowserSnapshot,
|
||||
} from '../../shared/agent-browser';
|
||||
|
||||
type BrowserEnvelope = {
|
||||
success: boolean;
|
||||
browser: AgentBrowserSnapshot;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
type EventPageEnvelope = {
|
||||
success: boolean;
|
||||
page: AgentBrowserCdpEventPage;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
type CdpResultEnvelope = {
|
||||
success: boolean;
|
||||
result: AgentBrowserCdpResult;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
type PayloadEnvelope = {
|
||||
success: boolean;
|
||||
chunk: AgentBrowserPayloadChunk;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type AgentBrowserNavigateAction = 'url' | 'back' | 'forward' | 'reload';
|
||||
|
||||
export interface AgentBrowserConsoleEntry {
|
||||
id: number;
|
||||
level: string;
|
||||
text: string;
|
||||
timestamp: number;
|
||||
source?: string;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface AgentBrowserNetworkEntry {
|
||||
requestId: string;
|
||||
sequence: number;
|
||||
method: string;
|
||||
url: string;
|
||||
resourceType?: string;
|
||||
status?: number;
|
||||
statusText?: string;
|
||||
mimeType?: string;
|
||||
durationMs?: number;
|
||||
encodedDataLength?: number;
|
||||
errorText?: string;
|
||||
}
|
||||
|
||||
export interface AgentBrowserDiagnostics {
|
||||
console: AgentBrowserConsoleEntry[];
|
||||
network: AgentBrowserNetworkEntry[];
|
||||
}
|
||||
|
||||
function jsonBody(body: Record<string, unknown>): RequestInit {
|
||||
return {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
};
|
||||
}
|
||||
|
||||
function requireSuccess<T extends { success: boolean; error?: string }>(
|
||||
envelope: T,
|
||||
): T {
|
||||
if (!envelope.success) {
|
||||
throw new Error(envelope.error || '开发浏览器请求失败');
|
||||
}
|
||||
return envelope;
|
||||
}
|
||||
|
||||
export async function getAgentBrowserState(
|
||||
projectPath: string,
|
||||
): Promise<AgentBrowserSnapshot> {
|
||||
const query = new URLSearchParams({ project_path: projectPath });
|
||||
const response = await hostApiFetch<BrowserEnvelope>(
|
||||
`/api/agent-browser/state?${query.toString()}`,
|
||||
);
|
||||
return requireSuccess(response).browser;
|
||||
}
|
||||
|
||||
export async function openAgentBrowser(input: {
|
||||
projectPath: string;
|
||||
url: string;
|
||||
bounds?: AgentBrowserBounds;
|
||||
}): Promise<AgentBrowserSnapshot> {
|
||||
const response = await hostApiFetch<BrowserEnvelope>(
|
||||
'/api/agent-browser/open',
|
||||
jsonBody({
|
||||
project_path: input.projectPath,
|
||||
url: input.url,
|
||||
visible: true,
|
||||
...(input.bounds ? { bounds: input.bounds } : {}),
|
||||
}),
|
||||
);
|
||||
return requireSuccess(response).browser;
|
||||
}
|
||||
|
||||
export async function presentAgentBrowser(input: {
|
||||
projectPath: string;
|
||||
visible: boolean;
|
||||
bounds?: AgentBrowserBounds;
|
||||
}): Promise<AgentBrowserSnapshot> {
|
||||
const response = await hostApiFetch<BrowserEnvelope>(
|
||||
'/api/agent-browser/present',
|
||||
jsonBody({
|
||||
project_path: input.projectPath,
|
||||
visible: input.visible,
|
||||
...(input.bounds ? { bounds: input.bounds } : {}),
|
||||
}),
|
||||
);
|
||||
return requireSuccess(response).browser;
|
||||
}
|
||||
|
||||
export async function navigateAgentBrowser(input: {
|
||||
projectPath: string;
|
||||
action: AgentBrowserNavigateAction;
|
||||
url?: string;
|
||||
}): Promise<AgentBrowserSnapshot> {
|
||||
const response = await hostApiFetch<BrowserEnvelope>(
|
||||
'/api/agent-browser/navigate',
|
||||
jsonBody({
|
||||
project_path: input.projectPath,
|
||||
action: input.action,
|
||||
...(input.url ? { url: input.url } : {}),
|
||||
}),
|
||||
);
|
||||
return requireSuccess(response).browser;
|
||||
}
|
||||
|
||||
export async function sendAgentBrowserCdp(input: {
|
||||
projectPath: string;
|
||||
method: string;
|
||||
params?: Record<string, unknown>;
|
||||
sessionRef?: string;
|
||||
timeoutMs?: number;
|
||||
}): Promise<AgentBrowserCdpResult> {
|
||||
const response = await hostApiFetch<CdpResultEnvelope>(
|
||||
'/api/agent-browser/cdp/send',
|
||||
jsonBody({
|
||||
project_path: input.projectPath,
|
||||
method: input.method,
|
||||
...(input.params ? { params: input.params } : {}),
|
||||
...(input.sessionRef ? { session_ref: input.sessionRef } : {}),
|
||||
...(input.timeoutMs !== undefined ? { timeout_ms: input.timeoutMs } : {}),
|
||||
}),
|
||||
);
|
||||
return requireSuccess(response).result;
|
||||
}
|
||||
|
||||
export async function readAgentBrowserEvents(input: {
|
||||
projectPath: string;
|
||||
after?: number;
|
||||
methods?: string[];
|
||||
limit?: number;
|
||||
waitMs?: number;
|
||||
}): Promise<AgentBrowserCdpEventPage> {
|
||||
const response = await hostApiFetch<EventPageEnvelope>(
|
||||
'/api/agent-browser/cdp/events',
|
||||
jsonBody({
|
||||
project_path: input.projectPath,
|
||||
...(input.after !== undefined ? { after: input.after } : {}),
|
||||
...(input.methods ? { methods: input.methods } : {}),
|
||||
...(input.limit !== undefined ? { limit: input.limit } : {}),
|
||||
...(input.waitMs !== undefined ? { wait_ms: input.waitMs } : {}),
|
||||
}),
|
||||
);
|
||||
return requireSuccess(response).page;
|
||||
}
|
||||
|
||||
export async function readAgentBrowserPayload(input: {
|
||||
projectPath: string;
|
||||
handle: string;
|
||||
offset?: number;
|
||||
maxBytes?: number;
|
||||
}): Promise<AgentBrowserPayloadChunk> {
|
||||
const response = await hostApiFetch<PayloadEnvelope>(
|
||||
'/api/agent-browser/payload/read',
|
||||
jsonBody({
|
||||
project_path: input.projectPath,
|
||||
handle: input.handle,
|
||||
...(input.offset !== undefined ? { offset: input.offset } : {}),
|
||||
...(input.maxBytes !== undefined ? { max_bytes: input.maxBytes } : {}),
|
||||
}),
|
||||
);
|
||||
return requireSuccess(response).chunk;
|
||||
}
|
||||
|
||||
export async function closeAgentBrowser(
|
||||
projectPath: string,
|
||||
): Promise<AgentBrowserSnapshot> {
|
||||
const response = await hostApiFetch<BrowserEnvelope>(
|
||||
'/api/agent-browser/close',
|
||||
jsonBody({ project_path: projectPath }),
|
||||
);
|
||||
return requireSuccess(response).browser;
|
||||
}
|
||||
|
||||
export async function resetAgentBrowserProfile(
|
||||
projectPath: string,
|
||||
): Promise<AgentBrowserSnapshot> {
|
||||
const response = await hostApiFetch<BrowserEnvelope>(
|
||||
'/api/agent-browser/reset-profile',
|
||||
jsonBody({ project_path: projectPath }),
|
||||
);
|
||||
return requireSuccess(response).browser;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: null;
|
||||
}
|
||||
|
||||
function asString(value: unknown): string | undefined {
|
||||
return typeof value === 'string' ? value : undefined;
|
||||
}
|
||||
|
||||
function asNumber(value: unknown): number | undefined {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function stringifyConsoleArgument(value: unknown): string {
|
||||
const remoteObject = asRecord(value);
|
||||
if (!remoteObject) return String(value ?? '');
|
||||
if ('value' in remoteObject) {
|
||||
const primitive = remoteObject.value;
|
||||
if (typeof primitive === 'string') return primitive;
|
||||
try {
|
||||
return JSON.stringify(primitive);
|
||||
} catch {
|
||||
return String(primitive);
|
||||
}
|
||||
}
|
||||
return asString(remoteObject.description)
|
||||
?? asString(remoteObject.className)
|
||||
?? asString(remoteObject.type)
|
||||
?? '';
|
||||
}
|
||||
|
||||
function consoleEntryFromEvent(
|
||||
event: AgentBrowserCdpEvent,
|
||||
): AgentBrowserConsoleEntry | null {
|
||||
const params = asRecord(event.params);
|
||||
if (!params) {
|
||||
if (
|
||||
event.payload
|
||||
&& (
|
||||
event.method === 'Runtime.consoleAPICalled'
|
||||
|| event.method === 'Runtime.exceptionThrown'
|
||||
|| event.method === 'Log.entryAdded'
|
||||
)
|
||||
) {
|
||||
return {
|
||||
id: event.sequence,
|
||||
level: 'info',
|
||||
text: `大型 Console 事件（${event.payload.byteLength} 字节），智能体可按需读取完整内容`,
|
||||
timestamp: event.timestamp,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
if (event.method === 'Runtime.consoleAPICalled') {
|
||||
const args = Array.isArray(params.args) ? params.args : [];
|
||||
return {
|
||||
id: event.sequence,
|
||||
level: asString(params.type) ?? 'log',
|
||||
text: args.map(stringifyConsoleArgument).join(' '),
|
||||
timestamp: event.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
if (event.method === 'Runtime.exceptionThrown') {
|
||||
const details = asRecord(params.exceptionDetails);
|
||||
const exception = asRecord(details?.exception);
|
||||
const exceptionText = exception ? stringifyConsoleArgument(exception) : '';
|
||||
return {
|
||||
id: event.sequence,
|
||||
level: 'error',
|
||||
text: asString(exception?.description)
|
||||
?? (exceptionText || undefined)
|
||||
?? asString(details?.text)
|
||||
?? '未捕获异常',
|
||||
timestamp: event.timestamp,
|
||||
url: asString(details?.url),
|
||||
};
|
||||
}
|
||||
|
||||
if (event.method === 'Log.entryAdded') {
|
||||
const entry = asRecord(params.entry);
|
||||
if (!entry) return null;
|
||||
return {
|
||||
id: event.sequence,
|
||||
level: asString(entry.level) ?? 'log',
|
||||
text: asString(entry.text) ?? '',
|
||||
timestamp: event.timestamp,
|
||||
source: asString(entry.source),
|
||||
url: asString(entry.url),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function deriveAgentBrowserDiagnostics(
|
||||
events: AgentBrowserCdpEvent[],
|
||||
): AgentBrowserDiagnostics {
|
||||
const consoleEntries: AgentBrowserConsoleEntry[] = [];
|
||||
const requests = new Map<string, AgentBrowserNetworkEntry & { startedAt: number }>();
|
||||
const requestOrder: string[] = [];
|
||||
|
||||
for (const event of events) {
|
||||
const consoleEntry = consoleEntryFromEvent(event);
|
||||
if (consoleEntry) consoleEntries.push(consoleEntry);
|
||||
|
||||
const params = asRecord(event.params);
|
||||
const requestId = asString(params?.requestId);
|
||||
if (!params || !requestId) {
|
||||
if (event.payload && event.method.startsWith('Network.')) {
|
||||
const payloadKey = `${event.sessionRef ?? 'root'}:payload:${event.sequence}`;
|
||||
requestOrder.push(payloadKey);
|
||||
requests.set(payloadKey, {
|
||||
requestId: `payload:${event.sequence}`,
|
||||
sequence: event.sequence,
|
||||
method: event.method.slice('Network.'.length),
|
||||
url: `大型 Network 事件（${event.payload.byteLength} 字节），智能体可按需读取完整内容`,
|
||||
resourceType: 'payload',
|
||||
startedAt: event.timestamp,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const requestKey = `${event.sessionRef ?? 'root'}:${requestId}`;
|
||||
|
||||
if (event.method === 'Network.requestWillBeSent') {
|
||||
const request = asRecord(params.request);
|
||||
if (!requests.has(requestKey)) requestOrder.push(requestKey);
|
||||
requests.set(requestKey, {
|
||||
requestId,
|
||||
sequence: event.sequence,
|
||||
method: asString(request?.method) ?? 'GET',
|
||||
url: asString(request?.url) ?? '',
|
||||
resourceType: asString(params.type),
|
||||
startedAt: event.timestamp,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const current = requests.get(requestKey);
|
||||
if (!current) continue;
|
||||
|
||||
if (event.method === 'Network.responseReceived') {
|
||||
const response = asRecord(params.response);
|
||||
requests.set(requestKey, {
|
||||
...current,
|
||||
url: asString(response?.url) ?? current.url,
|
||||
resourceType: asString(params.type) ?? current.resourceType,
|
||||
status: asNumber(response?.status),
|
||||
statusText: asString(response?.statusText),
|
||||
mimeType: asString(response?.mimeType),
|
||||
});
|
||||
} else if (event.method === 'Network.loadingFinished') {
|
||||
requests.set(requestKey, {
|
||||
...current,
|
||||
durationMs: Math.max(0, event.timestamp - current.startedAt),
|
||||
encodedDataLength: asNumber(params.encodedDataLength),
|
||||
});
|
||||
} else if (event.method === 'Network.loadingFailed') {
|
||||
requests.set(requestKey, {
|
||||
...current,
|
||||
durationMs: Math.max(0, event.timestamp - current.startedAt),
|
||||
errorText: asString(params.errorText) ?? '请求失败',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
console: consoleEntries,
|
||||
network: requestOrder
|
||||
.map((requestId) => requests.get(requestId))
|
||||
.filter((entry): entry is AgentBrowserNetworkEntry & { startedAt: number } => Boolean(entry))
|
||||
.map((entry) => ({
|
||||
requestId: entry.requestId,
|
||||
sequence: entry.sequence,
|
||||
method: entry.method,
|
||||
url: entry.url,
|
||||
...(entry.resourceType ? { resourceType: entry.resourceType } : {}),
|
||||
...(entry.status !== undefined ? { status: entry.status } : {}),
|
||||
...(entry.statusText ? { statusText: entry.statusText } : {}),
|
||||
...(entry.mimeType ? { mimeType: entry.mimeType } : {}),
|
||||
...(entry.durationMs !== undefined ? { durationMs: entry.durationMs } : {}),
|
||||
...(entry.encodedDataLength !== undefined
|
||||
? { encodedDataLength: entry.encodedDataLength }
|
||||
: {}),
|
||||
...(entry.errorText ? { errorText: entry.errorText } : {}),
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,8 @@ const HOST_EVENT_TO_IPC_CHANNEL: Record<string, string> = {
|
||||
'oauth:code': 'oauth:code',
|
||||
'oauth:success': 'oauth:success',
|
||||
'oauth:error': 'oauth:error',
|
||||
'agent-browser:show': 'agent-browser:show',
|
||||
'agent-browser:state': 'agent-browser:state',
|
||||
};
|
||||
|
||||
function getEventSource(): EventSource {
|
||||
|
||||
Reference in New Issue
Block a user