diff --git a/ts/packages/agents/browser/src/agent/actionHandler.mts b/ts/packages/agents/browser/src/agent/actionHandler.mts index 9b02da80..097a79cc 100644 --- a/ts/packages/agents/browser/src/agent/actionHandler.mts +++ b/ts/packages/agents/browser/src/agent/actionHandler.mts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { WebSocketMessage, createWebSocket } from "common-utils/ws"; +import { createWebSocket } from "common-utils/ws"; import { WebSocket } from "ws"; import { ActionContext, @@ -94,25 +94,37 @@ async function updateBrowserContext( }; webSocket.addEventListener("message", async (event: any) => { const text = event.data.toString(); - const data = JSON.parse(text) as WebSocketMessage; + const data = JSON.parse(text); if (isWebAgentMessage(data)) { await processWebAgentMessage(data, context); return; } - if (data.target !== "dispatcher" || data.source !== "browser") { + if (data.error) { + console.error(data.error); + // TODO: Handle the case where no clients were found. Prompt the user + // to launch inline browser or run automation in the headless browser. return; } - if (data.body) { - switch (data.messageType) { + if (data.method) { + switch (data.method) { case "enableSiteTranslator": { - if (data.body == "browser.crossword") { + const targetTranslator = data.params.translator; + if (targetTranslator == "browser.crossword") { // initialize crossword state - sendSiteTranslatorStatus(data.body, "initializing", context); + sendSiteTranslatorStatus( + targetTranslator, + "initializing", + context, + ); context.agentContext.crossWordState = await getBoardSchema(context); - sendSiteTranslatorStatus(data.body, "initialized", context); + sendSiteTranslatorStatus( + targetTranslator, + "initialized", + context, + ); if (context.agentContext.crossWordState) { context.notify( @@ -126,26 +138,26 @@ async function updateBrowserContext( ); } } - await context.toggleTransientAgent(data.body, true); + await context.toggleTransientAgent(targetTranslator, true); break; } case "disableSiteTranslator": { - await context.toggleTransientAgent(data.body, false); + const targetTranslator = data.params.translator; + await context.toggleTransientAgent(targetTranslator, false); break; } - case "browserActionResponse": { - break; - } - case "debugBrowserAction": { - await executeBrowserAction( - data.body, - context as unknown as ActionContext, + case "addTabIdToIndex": + case "deleteTabIdFromIndex": + case "getTabIdFromIndex": + case "resetTabIdToIndex": { + await handleTabIndexActions( + { + actionName: data.method, + parameters: data.params, + }, + context, + data.id, ); - - break; - } - case "tabIndexRequest": { - await handleTabIndexActions(data.body, context, data.id); break; } } @@ -178,9 +190,9 @@ async function executeBrowserAction( try { context.actionIO.setDisplay("Running remote action."); - let messageType = "browserActionRequest"; + let schemaName = "browser"; if (action.translatorName === "browser.paleoBioDb") { - messageType = "browserActionRequest.paleoBioDb"; + schemaName = "browser.paleoBioDb"; } else if (action.translatorName === "browser.crossword") { const crosswordResult = await handleCrosswordAction(action, context); return createActionResult(crosswordResult); @@ -199,7 +211,7 @@ async function executeBrowserAction( // return createActionResult(instacartResult); } - await connector?.sendActionToBrowser(action, messageType); + await connector?.sendActionToBrowser(action, schemaName); } catch (ex: any) { if (ex instanceof Error) { console.error(ex); @@ -226,9 +238,7 @@ function sendSiteTranslatorStatus( if (webSocketEndpoint) { webSocketEndpoint.send( JSON.stringify({ - source: "dispatcher", - target: "browser", - messageType: "siteTranslatorStatus", + method: "browser/siteTranslatorStatus", id: callId, body: { translator: translatorName, @@ -288,11 +298,8 @@ async function handleTabIndexActions( webSocketEndpoint.send( JSON.stringify({ - source: "dispatcher", - target: "browser", - messageType: "tabIndexResponse", id: requestId, - body: responseBody, + result: responseBody, }), ); } catch (ex: any) { diff --git a/ts/packages/agents/browser/src/agent/browserConnector.mts b/ts/packages/agents/browser/src/agent/browserConnector.mts index 3b569cda..a9090d87 100644 --- a/ts/packages/agents/browser/src/agent/browserConnector.mts +++ b/ts/packages/agents/browser/src/agent/browserConnector.mts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { WebSocketMessage } from "common-utils"; import { AppAction, SessionContext } from "@typeagent/agent-sdk"; import { BrowserActionContext } from "./actionHandler.mjs"; @@ -12,44 +11,36 @@ export class BrowserConnector { this.webSocket = context.agentContext.webSocket; } - async sendActionToBrowser(action: AppAction, messageType?: string) { + async sendActionToBrowser(action: AppAction, schemaName?: string) { return new Promise((resolve, reject) => { if (this.webSocket) { try { const callId = new Date().getTime().toString(); - if (!messageType) { - messageType = "browserActionRequest"; + if (!schemaName) { + schemaName = "browser"; } this.webSocket.send( JSON.stringify({ - source: "dispatcher", - target: "browser", - messageType: messageType, id: callId, - body: action, + method: `${schemaName}/${action.actionName}`, + params: action.parameters, }), ); const handler = (event: any) => { const text = event.data.toString(); - const data = JSON.parse(text) as WebSocketMessage; - if ( - data.target == "dispatcher" && - data.source == "browser" && - data.messageType == "browserActionResponse" && - data.id == callId && - data.body - ) { + const data = JSON.parse(text); + if (data.id == callId && data.result) { this.webSocket.removeEventListener("message", handler); - resolve(data.body); + resolve(data.result); } }; this.webSocket.addEventListener("message", handler); } catch { console.log("Unable to contact browser backend."); - reject("Unable to contact browser backend."); + reject("Unable to contact browser backend (from connector)."); } } else { throw new Error("No websocket connection."); @@ -59,10 +50,7 @@ export class BrowserConnector { private async getPageDataFromBrowser(action: any) { return new Promise(async (resolve, reject) => { - const response = await this.sendActionToBrowser( - action, - "browserActionRequest", - ); + const response = await this.sendActionToBrowser(action, "browser"); if (response.data) { resolve(response.data); } else { @@ -174,7 +162,7 @@ export class BrowserConnector { }, }; - return this.sendActionToBrowser(schemaAction, "browserActionRequest"); + return this.sendActionToBrowser(schemaAction, "browser"); } async getPageUrl() { @@ -214,10 +202,7 @@ export class BrowserConnector { actionName: "awaitPageLoad", }; - const actionPromise = this.sendActionToBrowser( - action, - "browserActionRequest", - ); + const actionPromise = this.sendActionToBrowser(action, "browser"); if (timeout) { const timeoutPromise = new Promise((f) => setTimeout(f, timeout)); return Promise.race([actionPromise, timeoutPromise]); diff --git a/ts/packages/agents/browser/src/agent/instacart/planHandler.mts b/ts/packages/agents/browser/src/agent/instacart/planHandler.mts index d208c52b..dc2b0b4b 100644 --- a/ts/packages/agents/browser/src/agent/instacart/planHandler.mts +++ b/ts/packages/agents/browser/src/agent/instacart/planHandler.mts @@ -188,7 +188,7 @@ export async function handleInstacartAction( .findPageComponent("ShoppingCartDetails") .thenRun(async (context) => { const cartDetails = context["ShoppingCartDetails"]; - console.log(cartDetails); + // console.log(cartDetails); entities.push({ name: cartDetails.storeName, diff --git a/ts/packages/agents/browser/src/extension/serviceWorker.ts b/ts/packages/agents/browser/src/extension/serviceWorker.ts index 0de11901..09cb17e2 100644 --- a/ts/packages/agents/browser/src/extension/serviceWorker.ts +++ b/ts/packages/agents/browser/src/extension/serviceWorker.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { WebSocketMessageV2 } from "common-utils/ws"; import { WebSocketMessage } from "../../../../commonUtils/dist/indexBrowser"; import { isWebAgentMessage, @@ -85,52 +86,57 @@ async function ensureWebsocketConnected() { webSocket.onmessage = async (event: any, isBinary: boolean) => { const text = await event.data.text(); - const data = JSON.parse(text) as WebSocketMessage; - if (data.target == "browser") { - if (data.messageType == "browserActionRequest") { - const response = await runBrowserAction(data.body); - webSocket.send( - JSON.stringify({ - source: data.target, - target: data.source, - messageType: "browserActionResponse", - id: data.id, - body: response, - }), - ); - } else if (data.messageType == "siteTranslatorStatus") { - if (data.body.status == "initializing") { - showBadgeBusy(); - console.log(`Initializing ${data.body.translator}`); - } else if (data.body.status == "initialized") { - showBadgeHealthy(); - console.log( - `Finished initializing ${data.body.translator}`, + const data = JSON.parse(text) as WebSocketMessageV2; + + if (data.error) { + console.error(data.error); + return; + } + + if (data.method && data.method.indexOf("/") > 0) { + const [schema, actionName] = data.method?.split("/"); + + if (schema == "browser") { + if (actionName == "siteTranslatorStatus") { + if (data.params.status == "initializing") { + showBadgeBusy(); + console.log( + `Initializing ${data.params.translator}`, + ); + } else if (data.params.status == "initialized") { + showBadgeHealthy(); + console.log( + `Finished initializing ${data.params.translator}`, + ); + } + } else { + const response = await runBrowserAction({ + actionName: actionName, + parameters: data.params, + }); + + webSocket.send( + JSON.stringify({ + id: data.id, + result: response, + }), ); } - } else if ( - data.messageType.startsWith("browserActionRequest.") - ) { - const message = await runSiteAction( - data.messageType, - data.body, - ); + } else if (schema.startsWith("browser.")) { + const message = await runSiteAction(schema, { + actionName: actionName, + parameters: data.params, + }); webSocket.send( JSON.stringify({ - source: data.target, - target: data.source, - messageType: "browserActionResponse", id: data.id, - body: message, + result: message, }), ); } - - console.log( - `Browser websocket client received message: ${text}`, - ); } + console.log(`Browser websocket client received message: ${text}`); }; webSocket.onclose = (event: any) => { @@ -151,10 +157,8 @@ export function keepWebSocketAlive(webSocket: WebSocket) { if (webSocket && webSocket.readyState === WebSocket.OPEN) { webSocket.send( JSON.stringify({ - source: "browser", - target: "none", - messageType: "keepAlive", - body: {}, + method: "keepAlive", + params: {}, }), ); } else { @@ -758,20 +762,20 @@ async function getFilteredHTMLFragments( let currentSiteTranslator = ""; let currentCrosswordUrl = ""; async function toggleSiteTranslator(targetTab: chrome.tabs.Tab) { - let messageType = "enableSiteTranslator"; - let messageBody = ""; + let method = "enableSiteTranslator"; + let translatorName = ""; await ensureWebsocketConnected(); if (targetTab.url) { const host = new URL(targetTab.url).host; if (host === "paleobiodb.org" || host === "www.paleobiodb.org") { - messageType = "enableSiteTranslator"; - messageBody = "browser.paleoBioDb"; + method = "enableSiteTranslator"; + translatorName = "browser.paleoBioDb"; currentSiteTranslator = "browser.paleoBioDb"; } else { if (currentSiteTranslator == "browser.paleoBioDb") { - messageType = "disableSiteTranslator"; - messageBody = "browser.paleoBioDb"; + method = "disableSiteTranslator"; + translatorName = "browser.paleoBioDb"; } } @@ -795,8 +799,8 @@ async function toggleSiteTranslator(targetTab: chrome.tabs.Tab) { "https://www.bestcrosswords.com/bestcrosswords/guestconstructor", ) ) { - messageType = "enableSiteTranslator"; - messageBody = "browser.crossword"; + method = "enableSiteTranslator"; + translatorName = "browser.crossword"; currentSiteTranslator = "browser.crossword"; currentCrosswordUrl = targetTab.url; } @@ -808,14 +812,14 @@ async function toggleSiteTranslator(targetTab: chrome.tabs.Tab) { ]; if (commerceHosts.includes(host)) { - messageType = "enableSiteTranslator"; - messageBody = "browser.commerce"; + method = "enableSiteTranslator"; + translatorName = "browser.commerce"; currentSiteTranslator = "browser.commerce"; } if (host === "instacart.com" || host === "www.instacart.com") { - messageType = "enableSiteTranslator"; - messageBody = "browser.instacart"; + method = "enableSiteTranslator"; + translatorName = "browser.instacart"; currentSiteTranslator = "browser.instacart"; } @@ -823,14 +827,12 @@ async function toggleSiteTranslator(targetTab: chrome.tabs.Tab) { if ( webSocket && webSocket.readyState === WebSocket.OPEN && - messageBody + translatorName ) { webSocket.send( JSON.stringify({ - source: "browser", - target: "dispatcher", - messageType: messageType, - body: messageBody, + method: method, + params: { translator: translatorName }, }), ); } @@ -842,37 +844,21 @@ async function sendActionToTabIndex(action: any) { if (webSocket) { try { const callId = new Date().getTime().toString(); - const messageType = "tabIndexRequest"; webSocket.send( JSON.stringify({ - source: "browser", - target: "dispatcher", - messageType: messageType, + method: action.actionName, id: callId, - body: action, + params: action.parameters, }), ); const handler = async (event: any) => { const text = await event.data.text(); - const data = JSON.parse(text) as WebSocketMessage; - if ( - data.target == "browser" && - data.source == "dispatcher" && - data.id == callId && - data.body - ) { - switch (data.messageType) { - case "tabIndexResponse": { - webSocket.removeEventListener( - "message", - handler, - ); - resolve(data.body); - break; - } - } + const data = JSON.parse(text) as WebSocketMessageV2; + if (data.id == callId && data.result) { + webSocket.removeEventListener("message", handler); + resolve(data.result); } }; @@ -1262,10 +1248,10 @@ async function runBrowserAction(action: any) { }; } -async function runSiteAction(messageType: string, action: any) { +async function runSiteAction(schemaName: string, action: any) { let confirmationMessage = "OK"; - switch (messageType) { - case "browserActionRequest.paleoBioDb": { + switch (schemaName) { + case "browser.paleoBioDb": { const targetTab = await getActiveTab(); const actionName = action.actionName ?? action.fullActionName.split(".").at(-1); @@ -1290,7 +1276,7 @@ async function runSiteAction(messageType: string, action: any) { // to do: update confirmation to include current page screenshot. break; } - case "browserActionRequest.crossword": { + case "browser.crossword": { const targetTab = await getActiveTab(); const result = await chrome.tabs.sendMessage(targetTab.id!, { @@ -1301,7 +1287,7 @@ async function runSiteAction(messageType: string, action: any) { // to do: update confirmation to include current page screenshot. break; } - case "browserActionRequest.commerce": { + case "browser.commerce": { const targetTab = await getActiveTab(); const result = await chrome.tabs.sendMessage(targetTab.id!, { @@ -1521,10 +1507,8 @@ chrome.contextMenus?.onClicked.addListener( if (webSocket && webSocket.readyState === WebSocket.OPEN) { webSocket.send( JSON.stringify({ - source: "browser", - target: "dispatcher", - messageType: "enableSiteTranslator", - body: "browser.crossword", + method: "enableSiteTranslator", + params: { translator: "browser.crossword" }, }), ); } diff --git a/ts/packages/commonUtils/src/webSockets.ts b/ts/packages/commonUtils/src/webSockets.ts index 089523f4..b13412e4 100644 --- a/ts/packages/commonUtils/src/webSockets.ts +++ b/ts/packages/commonUtils/src/webSockets.ts @@ -17,9 +17,20 @@ export type WebSocketMessage = { body: any; }; +export type WebSocketMessageV2 = { + id?: string; + method: string; + params?: any; + result?: any; + error?: { + code?: number | undefined; + message: string; + }; +}; + export async function createWebSocket( channel: string, - role: string, + role: "dispatcher" | "client", clientId?: string, ) { return new Promise((resolve, reject) => { diff --git a/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts b/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts index d62bf881..cc712e37 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts @@ -81,6 +81,17 @@ try { "Closing duplicate socket instance for id " + clientId, ); socket.close(1013, "duplicate"); + wss.clients.delete(socket); + const tempClient = { + id: clientId, + role: role, + socket: socket, + channelName: channelName, + }; + + if (channel.clients.has(tempClient)) { + channel.clients.delete(tempClient); + } } } @@ -92,28 +103,34 @@ try { ws.on("message", (message: string) => { try { - const data = JSON.parse(message) as WebSocketMessage; - if (data.messageType != "keepAlive") { - let foundAtLeastOneTarget = false; - - // Broadcast message to all clients in the same channel that have a different role - channel.clients.forEach((client) => { - if ( - client.role !== role && - client.socket.readyState === WebSocket.OPEN - ) { - client.socket.send(message); - foundAtLeastOneTarget = true; - } - }); - - if (!foundAtLeastOneTarget) { - const errorMessage = - data.source === channelName - ? `The ${channelName} agent is not connected. The message cannot be processed.` - : `No ${channelName} clients are listening for messaages on this channel`; - ws.send(JSON.stringify({ error: errorMessage })); + const data = JSON.parse(message); + if ( + data.messageType === "keepAlive" || + data.method === "keepAlive" + ) { + return; + } + let foundAtLeastOneTarget = false; + const messageTargetRole = + role !== "client" ? "client" : "dispatcher"; + + // Broadcast message to all clients in the same channel that have a different role + channel.clients.forEach((currClient) => { + if ( + currClient.role === messageTargetRole && + currClient.socket.readyState === WebSocket.OPEN + ) { + currClient.socket.send(message); + foundAtLeastOneTarget = true; } + }); + + if (!foundAtLeastOneTarget) { + const errorMessage = + client.role === "client" + ? `The ${channelName} agent is not connected. The message cannot be processed.` + : `No ${channelName} clients are listening for messaages on this channel`; + ws.send(JSON.stringify({ error: errorMessage })); } } catch { debug("WebSocket message not parsed."); diff --git a/ts/packages/shell/src/main/browserIpc.ts b/ts/packages/shell/src/main/browserIpc.ts index 0d9310db..5627dbba 100644 --- a/ts/packages/shell/src/main/browserIpc.ts +++ b/ts/packages/shell/src/main/browserIpc.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { - WebSocketMessage, + WebSocketMessageV2, createWebSocket, keepWebSocketAlive, } from "common-utils"; @@ -11,7 +11,7 @@ import WebSocket from "ws"; export class BrowserAgentIpc { private static instance: BrowserAgentIpc; - public onMessageReceived: ((message: WebSocketMessage) => void) | null; + public onMessageReceived: ((message: WebSocketMessageV2) => void) | null; private webSocket: any; private constructor() { @@ -55,9 +55,18 @@ export class BrowserAgentIpc { this.webSocket.onmessage = async (event: any) => { const text = event.data.toString(); - const data = JSON.parse(text) as WebSocketMessage; - if (data.target == "browser" && this.onMessageReceived) { - this.onMessageReceived(data); + const data = JSON.parse(text) as WebSocketMessageV2; + if (data.method) { + let schema = data.method?.split("/")[0]; + schema = schema || "browser"; + + if ( + (schema == "browser" || + schema.startsWith("browser.")) && + this.onMessageReceived + ) { + this.onMessageReceived(data); + } } }; @@ -86,7 +95,7 @@ export class BrowserAgentIpc { }, 5 * 1000); } - public async send(message: WebSocketMessage) { + public async send(message: WebSocketMessageV2) { await this.ensureWebsocketConnected(); this.webSocket.send(JSON.stringify(message)); } diff --git a/ts/packages/shell/src/main/index.ts b/ts/packages/shell/src/main/index.ts index eefd9d86..63677a17 100644 --- a/ts/packages/shell/src/main/index.ts +++ b/ts/packages/shell/src/main/index.ts @@ -24,7 +24,7 @@ import { unlinkSync } from "fs"; import { existsSync } from "node:fs"; import { shellAgentProvider } from "./agent.js"; import { BrowserAgentIpc } from "./browserIpc.js"; -import { WebSocketMessage } from "common-utils"; +import { WebSocketMessageV2 } from "common-utils"; import { AzureSpeech } from "./azureSpeech.js"; import { auth } from "aiclient"; import { @@ -301,7 +301,7 @@ function createWindow(): void { await BrowserAgentIpc.getinstance().ensureWebsocketConnected(); BrowserAgentIpc.getinstance().onMessageReceived = ( - message: WebSocketMessage, + message: WebSocketMessageV2, ) => { inlineBrowserView?.webContents.send( "received-from-browser-ipc", @@ -586,7 +586,7 @@ async function initialize() { ipcMain.on( "send-to-browser-ipc", - async (_event, data: WebSocketMessage) => { + async (_event, data: WebSocketMessageV2) => { await BrowserAgentIpc.getinstance().send(data); }, ); diff --git a/ts/packages/shell/src/preload/webView.ts b/ts/packages/shell/src/preload/webView.ts index 1e39c4c9..e3a41438 100644 --- a/ts/packages/shell/src/preload/webView.ts +++ b/ts/packages/shell/src/preload/webView.ts @@ -8,31 +8,43 @@ import DOMPurify from "dompurify"; import { ipcRenderer } from "electron"; ipcRenderer.on("received-from-browser-ipc", async (_, data) => { - if (data.target == "browser") { - if (data.messageType == "browserActionRequest") { - const response = await runBrowserAction(data.body); - sendToBrowserAgent({ - source: data.target, - target: data.source, - messageType: "browserActionResponse", - id: data.id, - body: response, - }); - } else if (data.messageType == "siteTranslatorStatus") { - if (data.body.status == "initializing") { - console.log(`Initializing ${data.body.translator}`); - } else if (data.body.status == "initialized") { - console.log(`Finished initializing ${data.body.translator}`); + if (data.error) { + console.error(data.error); + return; + } + + if (data.method && data.method.indexOf("/") > 0) { + const [schema, actionName] = data.method?.split("/"); + + if (schema == "browser") { + if (actionName == "siteTranslatorStatus") { + if (data.body.status == "initializing") { + console.log(`Initializing ${data.body.translator}`); + } else if (data.body.status == "initialized") { + console.log( + `Finished initializing ${data.body.translator}`, + ); + } + } else { + const response = await runBrowserAction({ + actionName: actionName, + parameters: data.params, + }); + + sendToBrowserAgent({ + id: data.id, + result: response, + }); } - } else if (data.messageType.startsWith("browserActionRequest.")) { - const message = await runSiteAction(data.messageType, data.body); + } else if (schema.startsWith("browser.")) { + const message = await runSiteAction(schema, { + actionName: actionName, + parameters: data.params, + }); sendToBrowserAgent({ - source: data.target, - target: data.source, - messageType: "browserActionResponse", id: data.id, - body: message, + result: message, }); } @@ -425,20 +437,16 @@ contextBridge.exposeInMainWorld("browserConnect", { enableSiteAgent: (translatorName) => { if (translatorName) { sendToBrowserAgent({ - source: "browser", - target: "dispatcher", - messageType: "enableSiteTranslator", - body: translatorName, + method: "enableSiteTranslator", + params: { translator: translatorName }, }); } }, disableSiteAgent: (translatorName) => { if (translatorName) { sendToBrowserAgent({ - source: "browser", - target: "dispatcher", - messageType: "disableSiteTranslator", - body: translatorName, + method: "disableSiteTranslator", + params: { translator: translatorName }, }); } },