Fix to send user tool confirmation decision for yolo or non interacti… (#5677)
Co-authored-by: Ravikant Agarwal <ravikantag@google.com>
This commit is contained in:
parent
fde9849d48
commit
882a97aff9
|
@ -468,6 +468,10 @@ export class CoreToolScheduler {
|
||||||
const { request: reqInfo, tool: toolInstance } = toolCall;
|
const { request: reqInfo, tool: toolInstance } = toolCall;
|
||||||
try {
|
try {
|
||||||
if (this.config.getApprovalMode() === ApprovalMode.YOLO) {
|
if (this.config.getApprovalMode() === ApprovalMode.YOLO) {
|
||||||
|
this.setToolCallOutcome(
|
||||||
|
reqInfo.callId,
|
||||||
|
ToolConfirmationOutcome.ProceedAlways,
|
||||||
|
);
|
||||||
this.setStatusInternal(reqInfo.callId, 'scheduled');
|
this.setStatusInternal(reqInfo.callId, 'scheduled');
|
||||||
} else {
|
} else {
|
||||||
const confirmationDetails = await toolInstance.shouldConfirmExecute(
|
const confirmationDetails = await toolInstance.shouldConfirmExecute(
|
||||||
|
@ -521,6 +525,10 @@ export class CoreToolScheduler {
|
||||||
wrappedConfirmationDetails,
|
wrappedConfirmationDetails,
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
|
this.setToolCallOutcome(
|
||||||
|
reqInfo.callId,
|
||||||
|
ToolConfirmationOutcome.ProceedAlways,
|
||||||
|
);
|
||||||
this.setStatusInternal(reqInfo.callId, 'scheduled');
|
this.setStatusInternal(reqInfo.callId, 'scheduled');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -555,13 +563,7 @@ export class CoreToolScheduler {
|
||||||
await originalOnConfirm(outcome);
|
await originalOnConfirm(outcome);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.toolCalls = this.toolCalls.map((call) => {
|
this.setToolCallOutcome(callId, outcome);
|
||||||
if (call.request.callId !== callId) return call;
|
|
||||||
return {
|
|
||||||
...call,
|
|
||||||
outcome,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
if (outcome === ToolConfirmationOutcome.Cancel || signal.aborted) {
|
if (outcome === ToolConfirmationOutcome.Cancel || signal.aborted) {
|
||||||
this.setStatusInternal(
|
this.setStatusInternal(
|
||||||
|
@ -774,4 +776,14 @@ export class CoreToolScheduler {
|
||||||
this.onToolCallsUpdate([...this.toolCalls]);
|
this.onToolCallsUpdate([...this.toolCalls]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private setToolCallOutcome(callId: string, outcome: ToolConfirmationOutcome) {
|
||||||
|
this.toolCalls = this.toolCalls.map((call) => {
|
||||||
|
if (call.request.callId !== callId) return call;
|
||||||
|
return {
|
||||||
|
...call,
|
||||||
|
outcome,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,7 @@ import {
|
||||||
} from '../index.js';
|
} from '../index.js';
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import { convertToFunctionResponse } from './coreToolScheduler.js';
|
import { convertToFunctionResponse } from './coreToolScheduler.js';
|
||||||
|
import { ToolCallDecision } from '../telemetry/types.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Executes a single tool call non-interactively.
|
* Executes a single tool call non-interactively.
|
||||||
|
@ -87,6 +88,7 @@ export async function executeToolCall(
|
||||||
error_type:
|
error_type:
|
||||||
toolResult.error === undefined ? undefined : toolResult.error.type,
|
toolResult.error === undefined ? undefined : toolResult.error.type,
|
||||||
prompt_id: toolCallRequest.prompt_id,
|
prompt_id: toolCallRequest.prompt_id,
|
||||||
|
decision: ToolCallDecision.AUTO_ACCEPT,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = convertToFunctionResponse(
|
const response = convertToFunctionResponse(
|
||||||
|
|
|
@ -100,7 +100,7 @@ export function recordToolCallMetrics(
|
||||||
functionName: string,
|
functionName: string,
|
||||||
durationMs: number,
|
durationMs: number,
|
||||||
success: boolean,
|
success: boolean,
|
||||||
decision?: 'accept' | 'reject' | 'modify',
|
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept',
|
||||||
): void {
|
): void {
|
||||||
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
|
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -14,6 +14,7 @@ export enum ToolCallDecision {
|
||||||
ACCEPT = 'accept',
|
ACCEPT = 'accept',
|
||||||
REJECT = 'reject',
|
REJECT = 'reject',
|
||||||
MODIFY = 'modify',
|
MODIFY = 'modify',
|
||||||
|
AUTO_ACCEPT = 'auto_accept',
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getDecisionFromOutcome(
|
export function getDecisionFromOutcome(
|
||||||
|
@ -21,10 +22,11 @@ export function getDecisionFromOutcome(
|
||||||
): ToolCallDecision {
|
): ToolCallDecision {
|
||||||
switch (outcome) {
|
switch (outcome) {
|
||||||
case ToolConfirmationOutcome.ProceedOnce:
|
case ToolConfirmationOutcome.ProceedOnce:
|
||||||
|
return ToolCallDecision.ACCEPT;
|
||||||
case ToolConfirmationOutcome.ProceedAlways:
|
case ToolConfirmationOutcome.ProceedAlways:
|
||||||
case ToolConfirmationOutcome.ProceedAlwaysServer:
|
case ToolConfirmationOutcome.ProceedAlwaysServer:
|
||||||
case ToolConfirmationOutcome.ProceedAlwaysTool:
|
case ToolConfirmationOutcome.ProceedAlwaysTool:
|
||||||
return ToolCallDecision.ACCEPT;
|
return ToolCallDecision.AUTO_ACCEPT;
|
||||||
case ToolConfirmationOutcome.ModifyWithEditor:
|
case ToolConfirmationOutcome.ModifyWithEditor:
|
||||||
return ToolCallDecision.MODIFY;
|
return ToolCallDecision.MODIFY;
|
||||||
case ToolConfirmationOutcome.Cancel:
|
case ToolConfirmationOutcome.Cancel:
|
||||||
|
|
|
@ -104,6 +104,7 @@ describe('UiTelemetryService', () => {
|
||||||
[ToolCallDecision.ACCEPT]: 0,
|
[ToolCallDecision.ACCEPT]: 0,
|
||||||
[ToolCallDecision.REJECT]: 0,
|
[ToolCallDecision.REJECT]: 0,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
},
|
},
|
||||||
byName: {},
|
byName: {},
|
||||||
},
|
},
|
||||||
|
@ -362,6 +363,7 @@ describe('UiTelemetryService', () => {
|
||||||
[ToolCallDecision.ACCEPT]: 1,
|
[ToolCallDecision.ACCEPT]: 1,
|
||||||
[ToolCallDecision.REJECT]: 0,
|
[ToolCallDecision.REJECT]: 0,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -395,6 +397,7 @@ describe('UiTelemetryService', () => {
|
||||||
[ToolCallDecision.ACCEPT]: 0,
|
[ToolCallDecision.ACCEPT]: 0,
|
||||||
[ToolCallDecision.REJECT]: 1,
|
[ToolCallDecision.REJECT]: 1,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -434,11 +437,13 @@ describe('UiTelemetryService', () => {
|
||||||
[ToolCallDecision.ACCEPT]: 0,
|
[ToolCallDecision.ACCEPT]: 0,
|
||||||
[ToolCallDecision.REJECT]: 0,
|
[ToolCallDecision.REJECT]: 0,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
});
|
});
|
||||||
expect(tools.byName['test_tool'].decisions).toEqual({
|
expect(tools.byName['test_tool'].decisions).toEqual({
|
||||||
[ToolCallDecision.ACCEPT]: 0,
|
[ToolCallDecision.ACCEPT]: 0,
|
||||||
[ToolCallDecision.REJECT]: 0,
|
[ToolCallDecision.REJECT]: 0,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -483,6 +488,7 @@ describe('UiTelemetryService', () => {
|
||||||
[ToolCallDecision.ACCEPT]: 1,
|
[ToolCallDecision.ACCEPT]: 1,
|
||||||
[ToolCallDecision.REJECT]: 1,
|
[ToolCallDecision.REJECT]: 1,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -32,6 +32,7 @@ export interface ToolCallStats {
|
||||||
[ToolCallDecision.ACCEPT]: number;
|
[ToolCallDecision.ACCEPT]: number;
|
||||||
[ToolCallDecision.REJECT]: number;
|
[ToolCallDecision.REJECT]: number;
|
||||||
[ToolCallDecision.MODIFY]: number;
|
[ToolCallDecision.MODIFY]: number;
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: number;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -62,6 +63,7 @@ export interface SessionMetrics {
|
||||||
[ToolCallDecision.ACCEPT]: number;
|
[ToolCallDecision.ACCEPT]: number;
|
||||||
[ToolCallDecision.REJECT]: number;
|
[ToolCallDecision.REJECT]: number;
|
||||||
[ToolCallDecision.MODIFY]: number;
|
[ToolCallDecision.MODIFY]: number;
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: number;
|
||||||
};
|
};
|
||||||
byName: Record<string, ToolCallStats>;
|
byName: Record<string, ToolCallStats>;
|
||||||
};
|
};
|
||||||
|
@ -94,6 +96,7 @@ const createInitialMetrics = (): SessionMetrics => ({
|
||||||
[ToolCallDecision.ACCEPT]: 0,
|
[ToolCallDecision.ACCEPT]: 0,
|
||||||
[ToolCallDecision.REJECT]: 0,
|
[ToolCallDecision.REJECT]: 0,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
},
|
},
|
||||||
byName: {},
|
byName: {},
|
||||||
},
|
},
|
||||||
|
@ -192,6 +195,7 @@ export class UiTelemetryService extends EventEmitter {
|
||||||
[ToolCallDecision.ACCEPT]: 0,
|
[ToolCallDecision.ACCEPT]: 0,
|
||||||
[ToolCallDecision.REJECT]: 0,
|
[ToolCallDecision.REJECT]: 0,
|
||||||
[ToolCallDecision.MODIFY]: 0,
|
[ToolCallDecision.MODIFY]: 0,
|
||||||
|
[ToolCallDecision.AUTO_ACCEPT]: 0,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue