Log prompt id when a loop is detected (#4765)
Co-authored-by: N. Taylor Mullen <ntaylormullen@google.com>
This commit is contained in:
parent
6380bfe35c
commit
0ef9c0b792
|
@ -293,7 +293,7 @@ export class GeminiClient {
|
|||
originalModel?: string,
|
||||
): AsyncGenerator<ServerGeminiStreamEvent, Turn> {
|
||||
if (this.lastPromptId !== prompt_id) {
|
||||
this.loopDetector.reset();
|
||||
this.loopDetector.reset(prompt_id);
|
||||
this.lastPromptId = prompt_id;
|
||||
}
|
||||
this.sessionTurnCount++;
|
||||
|
|
|
@ -168,21 +168,21 @@ describe('LoopDetectionService', () => {
|
|||
);
|
||||
}
|
||||
|
||||
service.reset();
|
||||
service.reset('');
|
||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
||||
expect(service.addAndCheck(createContentEvent('obj.method()'))).toBe(
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
service.reset();
|
||||
service.reset('');
|
||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
||||
expect(
|
||||
service.addAndCheck(createContentEvent('arr.filter().map()')),
|
||||
).toBe(false);
|
||||
}
|
||||
|
||||
service.reset();
|
||||
service.reset('');
|
||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) {
|
||||
expect(
|
||||
service.addAndCheck(
|
||||
|
@ -203,7 +203,7 @@ describe('LoopDetectionService', () => {
|
|||
service.addAndCheck(createContentEvent('This is a sentence.')),
|
||||
).toBe(true);
|
||||
|
||||
service.reset();
|
||||
service.reset('');
|
||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
|
||||
expect(
|
||||
service.addAndCheck(createContentEvent('Is this a question? ')),
|
||||
|
@ -213,7 +213,7 @@ describe('LoopDetectionService', () => {
|
|||
service.addAndCheck(createContentEvent('Is this a question? ')),
|
||||
).toBe(true);
|
||||
|
||||
service.reset();
|
||||
service.reset('');
|
||||
for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) {
|
||||
expect(
|
||||
service.addAndCheck(createContentEvent('What excitement!\n')),
|
||||
|
|
|
@ -50,6 +50,7 @@ const SENTENCE_ENDING_PUNCTUATION_REGEX = /[.!?]+(?=\s|$)/;
|
|||
*/
|
||||
export class LoopDetectionService {
|
||||
private readonly config: Config;
|
||||
private promptId = '';
|
||||
|
||||
// Tool call tracking
|
||||
private lastToolCallKey: string | null = null;
|
||||
|
@ -129,7 +130,10 @@ export class LoopDetectionService {
|
|||
if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) {
|
||||
logLoopDetected(
|
||||
this.config,
|
||||
new LoopDetectedEvent(LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS),
|
||||
new LoopDetectedEvent(
|
||||
LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS,
|
||||
this.promptId,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
@ -170,7 +174,10 @@ export class LoopDetectionService {
|
|||
if (this.sentenceRepetitionCount >= CONTENT_LOOP_THRESHOLD) {
|
||||
logLoopDetected(
|
||||
this.config,
|
||||
new LoopDetectedEvent(LoopType.CHANTING_IDENTICAL_SENTENCES),
|
||||
new LoopDetectedEvent(
|
||||
LoopType.CHANTING_IDENTICAL_SENTENCES,
|
||||
this.promptId,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
@ -234,7 +241,7 @@ Please analyze the conversation history to determine the possibility that the co
|
|||
}
|
||||
logLoopDetected(
|
||||
this.config,
|
||||
new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP),
|
||||
new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP, this.promptId),
|
||||
);
|
||||
return true;
|
||||
} else {
|
||||
|
@ -251,7 +258,8 @@ Please analyze the conversation history to determine the possibility that the co
|
|||
/**
|
||||
* Resets all loop detection state.
|
||||
*/
|
||||
reset(): void {
|
||||
reset(promptId: string): void {
|
||||
this.promptId = promptId;
|
||||
this.resetToolCallCount();
|
||||
this.resetSentenceCount();
|
||||
this.resetLlmCheckTracking();
|
||||
|
|
|
@ -481,8 +481,8 @@ export class ClearcutLogger {
|
|||
logLoopDetectedEvent(event: LoopDetectedEvent): void {
|
||||
const data = [
|
||||
{
|
||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID,
|
||||
value: this.config?.getSessionId() ?? '',
|
||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
|
||||
value: JSON.stringify(event.prompt_id),
|
||||
},
|
||||
{
|
||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE,
|
||||
|
|
|
@ -256,11 +256,13 @@ export class LoopDetectedEvent {
|
|||
'event.name': 'loop_detected';
|
||||
'event.timestamp': string; // ISO 8601
|
||||
loop_type: LoopType;
|
||||
prompt_id: string;
|
||||
|
||||
constructor(loop_type: LoopType) {
|
||||
constructor(loop_type: LoopType, prompt_id: string) {
|
||||
this['event.name'] = 'loop_detected';
|
||||
this['event.timestamp'] = new Date().toISOString();
|
||||
this.loop_type = loop_type;
|
||||
this.prompt_id = prompt_id;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue