feat(client/compression): Log telemetry when compressing chat context. (#6195)
This commit is contained in:
parent
1a0cc68e29
commit
71f706cf29
|
@ -271,3 +271,8 @@ Metrics are numerical measurements of behavior over time. The following metrics
|
||||||
- `ai_removed_lines` (Int, if applicable): Number of lines removed/changed by AI.
|
- `ai_removed_lines` (Int, if applicable): Number of lines removed/changed by AI.
|
||||||
- `user_added_lines` (Int, if applicable): Number of lines added/changed by user in AI proposed changes.
|
- `user_added_lines` (Int, if applicable): Number of lines added/changed by user in AI proposed changes.
|
||||||
- `user_removed_lines` (Int, if applicable): Number of lines removed/changed by user in AI proposed changes.
|
- `user_removed_lines` (Int, if applicable): Number of lines removed/changed by user in AI proposed changes.
|
||||||
|
|
||||||
|
- `gemini_cli.chat_compression` (Counter, Int): Counts chat compression operations
|
||||||
|
- **Attributes**:
|
||||||
|
- `tokens_before`: (Int): Number of tokens in context prior to compression
|
||||||
|
- `tokens_after`: (Int): Number of tokens in context after compression
|
||||||
|
|
|
@ -24,6 +24,7 @@ import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
|
||||||
import { setSimulate429 } from '../utils/testUtils.js';
|
import { setSimulate429 } from '../utils/testUtils.js';
|
||||||
import { tokenLimit } from './tokenLimits.js';
|
import { tokenLimit } from './tokenLimits.js';
|
||||||
import { ideContext } from '../ide/ideContext.js';
|
import { ideContext } from '../ide/ideContext.js';
|
||||||
|
import { ClearcutLogger } from '../telemetry/clearcut-logger/clearcut-logger.js';
|
||||||
|
|
||||||
// --- Mocks ---
|
// --- Mocks ---
|
||||||
const mockChatCreateFn = vi.fn();
|
const mockChatCreateFn = vi.fn();
|
||||||
|
@ -532,6 +533,45 @@ describe('Gemini Client (client.ts)', () => {
|
||||||
expect(newChat).toBe(initialChat);
|
expect(newChat).toBe(initialChat);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('logs a telemetry event when compressing', async () => {
|
||||||
|
vi.spyOn(ClearcutLogger.prototype, 'logChatCompressionEvent');
|
||||||
|
|
||||||
|
const MOCKED_TOKEN_LIMIT = 1000;
|
||||||
|
const MOCKED_CONTEXT_PERCENTAGE_THRESHOLD = 0.5;
|
||||||
|
vi.mocked(tokenLimit).mockReturnValue(MOCKED_TOKEN_LIMIT);
|
||||||
|
vi.spyOn(client['config'], 'getChatCompression').mockReturnValue({
|
||||||
|
contextPercentageThreshold: MOCKED_CONTEXT_PERCENTAGE_THRESHOLD,
|
||||||
|
});
|
||||||
|
mockGetHistory.mockReturnValue([
|
||||||
|
{ role: 'user', parts: [{ text: '...history...' }] },
|
||||||
|
]);
|
||||||
|
|
||||||
|
const originalTokenCount =
|
||||||
|
MOCKED_TOKEN_LIMIT * MOCKED_CONTEXT_PERCENTAGE_THRESHOLD;
|
||||||
|
const newTokenCount = 100;
|
||||||
|
|
||||||
|
mockCountTokens
|
||||||
|
.mockResolvedValueOnce({ totalTokens: originalTokenCount }) // First call for the check
|
||||||
|
.mockResolvedValueOnce({ totalTokens: newTokenCount }); // Second call for the new history
|
||||||
|
|
||||||
|
// Mock the summary response from the chat
|
||||||
|
mockSendMessage.mockResolvedValue({
|
||||||
|
role: 'model',
|
||||||
|
parts: [{ text: 'This is a summary.' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
await client.tryCompressChat('prompt-id-3');
|
||||||
|
|
||||||
|
expect(
|
||||||
|
ClearcutLogger.prototype.logChatCompressionEvent,
|
||||||
|
).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
tokens_before: originalTokenCount,
|
||||||
|
tokens_after: newTokenCount,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('should trigger summarization if token count is at threshold with contextPercentageThreshold setting', async () => {
|
it('should trigger summarization if token count is at threshold with contextPercentageThreshold setting', async () => {
|
||||||
const MOCKED_TOKEN_LIMIT = 1000;
|
const MOCKED_TOKEN_LIMIT = 1000;
|
||||||
const MOCKED_CONTEXT_PERCENTAGE_THRESHOLD = 0.5;
|
const MOCKED_CONTEXT_PERCENTAGE_THRESHOLD = 0.5;
|
||||||
|
|
|
@ -43,8 +43,12 @@ import { ProxyAgent, setGlobalDispatcher } from 'undici';
|
||||||
import { DEFAULT_GEMINI_FLASH_MODEL } from '../config/models.js';
|
import { DEFAULT_GEMINI_FLASH_MODEL } from '../config/models.js';
|
||||||
import { LoopDetectionService } from '../services/loopDetectionService.js';
|
import { LoopDetectionService } from '../services/loopDetectionService.js';
|
||||||
import { ideContext } from '../ide/ideContext.js';
|
import { ideContext } from '../ide/ideContext.js';
|
||||||
import { logNextSpeakerCheck } from '../telemetry/loggers.js';
|
|
||||||
import {
|
import {
|
||||||
|
logChatCompression,
|
||||||
|
logNextSpeakerCheck,
|
||||||
|
} from '../telemetry/loggers.js';
|
||||||
|
import {
|
||||||
|
makeChatCompressionEvent,
|
||||||
MalformedJsonResponseEvent,
|
MalformedJsonResponseEvent,
|
||||||
NextSpeakerCheckEvent,
|
NextSpeakerCheckEvent,
|
||||||
} from '../telemetry/types.js';
|
} from '../telemetry/types.js';
|
||||||
|
@ -89,6 +93,20 @@ export function findIndexAfterFraction(
|
||||||
return contentLengths.length;
|
return contentLengths.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_TURNS = 100;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Threshold for compression token count as a fraction of the model's token limit.
|
||||||
|
* If the chat history exceeds this threshold, it will be compressed.
|
||||||
|
*/
|
||||||
|
const COMPRESSION_TOKEN_THRESHOLD = 0.7;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The fraction of the latest chat history to keep. A value of 0.3
|
||||||
|
* means that only the last 30% of the chat history will be kept after compression.
|
||||||
|
*/
|
||||||
|
const COMPRESSION_PRESERVE_THRESHOLD = 0.3;
|
||||||
|
|
||||||
export class GeminiClient {
|
export class GeminiClient {
|
||||||
private chat?: GeminiChat;
|
private chat?: GeminiChat;
|
||||||
private contentGenerator?: ContentGenerator;
|
private contentGenerator?: ContentGenerator;
|
||||||
|
@ -98,17 +116,6 @@ export class GeminiClient {
|
||||||
topP: 1,
|
topP: 1,
|
||||||
};
|
};
|
||||||
private sessionTurnCount = 0;
|
private sessionTurnCount = 0;
|
||||||
private readonly MAX_TURNS = 100;
|
|
||||||
/**
|
|
||||||
* Threshold for compression token count as a fraction of the model's token limit.
|
|
||||||
* If the chat history exceeds this threshold, it will be compressed.
|
|
||||||
*/
|
|
||||||
private readonly COMPRESSION_TOKEN_THRESHOLD = 0.7;
|
|
||||||
/**
|
|
||||||
* The fraction of the latest chat history to keep. A value of 0.3
|
|
||||||
* means that only the last 30% of the chat history will be kept after compression.
|
|
||||||
*/
|
|
||||||
private readonly COMPRESSION_PRESERVE_THRESHOLD = 0.3;
|
|
||||||
|
|
||||||
private readonly loopDetector: LoopDetectionService;
|
private readonly loopDetector: LoopDetectionService;
|
||||||
private lastPromptId: string;
|
private lastPromptId: string;
|
||||||
|
@ -438,7 +445,7 @@ export class GeminiClient {
|
||||||
request: PartListUnion,
|
request: PartListUnion,
|
||||||
signal: AbortSignal,
|
signal: AbortSignal,
|
||||||
prompt_id: string,
|
prompt_id: string,
|
||||||
turns: number = this.MAX_TURNS,
|
turns: number = MAX_TURNS,
|
||||||
originalModel?: string,
|
originalModel?: string,
|
||||||
): AsyncGenerator<ServerGeminiStreamEvent, Turn> {
|
): AsyncGenerator<ServerGeminiStreamEvent, Turn> {
|
||||||
if (this.lastPromptId !== prompt_id) {
|
if (this.lastPromptId !== prompt_id) {
|
||||||
|
@ -454,7 +461,7 @@ export class GeminiClient {
|
||||||
return new Turn(this.getChat(), prompt_id);
|
return new Turn(this.getChat(), prompt_id);
|
||||||
}
|
}
|
||||||
// Ensure turns never exceeds MAX_TURNS to prevent infinite loops
|
// Ensure turns never exceeds MAX_TURNS to prevent infinite loops
|
||||||
const boundedTurns = Math.min(turns, this.MAX_TURNS);
|
const boundedTurns = Math.min(turns, MAX_TURNS);
|
||||||
if (!boundedTurns) {
|
if (!boundedTurns) {
|
||||||
return new Turn(this.getChat(), prompt_id);
|
return new Turn(this.getChat(), prompt_id);
|
||||||
}
|
}
|
||||||
|
@ -673,7 +680,7 @@ export class GeminiClient {
|
||||||
const userMemory = this.config.getUserMemory();
|
const userMemory = this.config.getUserMemory();
|
||||||
const systemInstruction = getCoreSystemPrompt(userMemory);
|
const systemInstruction = getCoreSystemPrompt(userMemory);
|
||||||
|
|
||||||
const requestConfig = {
|
const requestConfig: GenerateContentConfig = {
|
||||||
abortSignal,
|
abortSignal,
|
||||||
...configToUse,
|
...configToUse,
|
||||||
systemInstruction,
|
systemInstruction,
|
||||||
|
@ -779,7 +786,7 @@ export class GeminiClient {
|
||||||
// Don't compress if not forced and we are under the limit.
|
// Don't compress if not forced and we are under the limit.
|
||||||
if (!force) {
|
if (!force) {
|
||||||
const threshold =
|
const threshold =
|
||||||
contextPercentageThreshold ?? this.COMPRESSION_TOKEN_THRESHOLD;
|
contextPercentageThreshold ?? COMPRESSION_TOKEN_THRESHOLD;
|
||||||
if (originalTokenCount < threshold * tokenLimit(model)) {
|
if (originalTokenCount < threshold * tokenLimit(model)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -787,7 +794,7 @@ export class GeminiClient {
|
||||||
|
|
||||||
let compressBeforeIndex = findIndexAfterFraction(
|
let compressBeforeIndex = findIndexAfterFraction(
|
||||||
curatedHistory,
|
curatedHistory,
|
||||||
1 - this.COMPRESSION_PRESERVE_THRESHOLD,
|
1 - COMPRESSION_PRESERVE_THRESHOLD,
|
||||||
);
|
);
|
||||||
// Find the first user message after the index. This is the start of the next turn.
|
// Find the first user message after the index. This is the start of the next turn.
|
||||||
while (
|
while (
|
||||||
|
@ -838,6 +845,14 @@ export class GeminiClient {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logChatCompression(
|
||||||
|
this.config,
|
||||||
|
makeChatCompressionEvent({
|
||||||
|
tokens_before: originalTokenCount,
|
||||||
|
tokens_after: newTokenCount,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
originalTokenCount,
|
originalTokenCount,
|
||||||
newTokenCount,
|
newTokenCount,
|
||||||
|
@ -891,3 +906,8 @@ export class GeminiClient {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const TEST_ONLY = {
|
||||||
|
COMPRESSION_PRESERVE_THRESHOLD,
|
||||||
|
COMPRESSION_TOKEN_THRESHOLD,
|
||||||
|
};
|
||||||
|
|
|
@ -4,6 +4,7 @@
|
||||||
* SPDX-License-Identifier: Apache-2.0
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import 'vitest';
|
||||||
import {
|
import {
|
||||||
vi,
|
vi,
|
||||||
describe,
|
describe,
|
||||||
|
@ -13,8 +14,13 @@ import {
|
||||||
beforeAll,
|
beforeAll,
|
||||||
afterAll,
|
afterAll,
|
||||||
} from 'vitest';
|
} from 'vitest';
|
||||||
|
import {
|
||||||
import { ClearcutLogger, LogEventEntry, TEST_ONLY } from './clearcut-logger.js';
|
ClearcutLogger,
|
||||||
|
LogEvent,
|
||||||
|
LogEventEntry,
|
||||||
|
EventNames,
|
||||||
|
TEST_ONLY,
|
||||||
|
} from './clearcut-logger.js';
|
||||||
import { ConfigParameters } from '../../config/config.js';
|
import { ConfigParameters } from '../../config/config.js';
|
||||||
import * as userAccount from '../../utils/user_account.js';
|
import * as userAccount from '../../utils/user_account.js';
|
||||||
import * as userId from '../../utils/user_id.js';
|
import * as userId from '../../utils/user_id.js';
|
||||||
|
@ -22,6 +28,48 @@ import { EventMetadataKey } from './event-metadata-key.js';
|
||||||
import { makeFakeConfig } from '../../test-utils/config.js';
|
import { makeFakeConfig } from '../../test-utils/config.js';
|
||||||
import { http, HttpResponse } from 'msw';
|
import { http, HttpResponse } from 'msw';
|
||||||
import { server } from '../../mocks/msw.js';
|
import { server } from '../../mocks/msw.js';
|
||||||
|
import { makeChatCompressionEvent } from '../types.js';
|
||||||
|
|
||||||
|
interface CustomMatchers<R = unknown> {
|
||||||
|
toHaveMetadataValue: ([key, value]: [EventMetadataKey, string]) => R;
|
||||||
|
toHaveEventName: (name: EventNames) => R;
|
||||||
|
}
|
||||||
|
|
||||||
|
declare module 'vitest' {
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-empty-object-type
|
||||||
|
interface Matchers<T = any> extends CustomMatchers<T> {}
|
||||||
|
}
|
||||||
|
|
||||||
|
expect.extend({
|
||||||
|
toHaveEventName(received: LogEventEntry[], name: EventNames) {
|
||||||
|
const { isNot } = this;
|
||||||
|
const event = JSON.parse(received[0].source_extension_json) as LogEvent;
|
||||||
|
const pass = event.event_name === (name as unknown as string);
|
||||||
|
return {
|
||||||
|
pass,
|
||||||
|
message: () =>
|
||||||
|
`event name ${event.event_name} does${isNot ? ' not ' : ''} match ${name}}`,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
|
||||||
|
toHaveMetadataValue(
|
||||||
|
received: LogEventEntry[],
|
||||||
|
[key, value]: [EventMetadataKey, string],
|
||||||
|
) {
|
||||||
|
const { isNot } = this;
|
||||||
|
const event = JSON.parse(received[0].source_extension_json) as LogEvent;
|
||||||
|
const metadata = event['event_metadata'][0];
|
||||||
|
const data = metadata.find((m) => m.gemini_cli_key === key)?.value;
|
||||||
|
|
||||||
|
const pass = data !== undefined && data === value;
|
||||||
|
|
||||||
|
return {
|
||||||
|
pass,
|
||||||
|
message: () =>
|
||||||
|
`event ${received} does${isNot ? ' not' : ''} have ${value}}`,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock('../../utils/user_account');
|
vi.mock('../../utils/user_account');
|
||||||
vi.mock('../../utils/user_id');
|
vi.mock('../../utils/user_id');
|
||||||
|
@ -47,6 +95,7 @@ describe('ClearcutLogger', () => {
|
||||||
const CLEARCUT_URL = 'https://play.googleapis.com/log';
|
const CLEARCUT_URL = 'https://play.googleapis.com/log';
|
||||||
const MOCK_DATE = new Date('2025-01-02T00:00:00.000Z');
|
const MOCK_DATE = new Date('2025-01-02T00:00:00.000Z');
|
||||||
const EXAMPLE_RESPONSE = `["${NEXT_WAIT_MS}",null,[[["ANDROID_BACKUP",0],["BATTERY_STATS",0],["SMART_SETUP",0],["TRON",0]],-3334737594024971225],[]]`;
|
const EXAMPLE_RESPONSE = `["${NEXT_WAIT_MS}",null,[[["ANDROID_BACKUP",0],["BATTERY_STATS",0],["SMART_SETUP",0],["TRON",0]],-3334737594024971225],[]]`;
|
||||||
|
|
||||||
// A helper to get the internal events array for testing
|
// A helper to get the internal events array for testing
|
||||||
const getEvents = (l: ClearcutLogger): LogEventEntry[][] =>
|
const getEvents = (l: ClearcutLogger): LogEventEntry[][] =>
|
||||||
l['events'].toArray() as LogEventEntry[][];
|
l['events'].toArray() as LogEventEntry[][];
|
||||||
|
@ -130,7 +179,7 @@ describe('ClearcutLogger', () => {
|
||||||
lifetimeGoogleAccounts: 9001,
|
lifetimeGoogleAccounts: 9001,
|
||||||
});
|
});
|
||||||
|
|
||||||
const event = logger?.createLogEvent('abc', []);
|
const event = logger?.createLogEvent(EventNames.API_ERROR, []);
|
||||||
|
|
||||||
expect(event?.event_metadata[0][0]).toEqual({
|
expect(event?.event_metadata[0][0]).toEqual({
|
||||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT,
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT,
|
||||||
|
@ -138,26 +187,13 @@ describe('ClearcutLogger', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('logs the current surface from a github action', () => {
|
it('logs the current surface', () => {
|
||||||
const { logger } = setup({});
|
|
||||||
|
|
||||||
vi.stubEnv('GITHUB_SHA', '8675309');
|
|
||||||
|
|
||||||
const event = logger?.createLogEvent('abc', []);
|
|
||||||
|
|
||||||
expect(event?.event_metadata[0][1]).toEqual({
|
|
||||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
|
|
||||||
value: 'GitHub',
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('honors the value from env.SURFACE over all others', () => {
|
|
||||||
const { logger } = setup({});
|
const { logger } = setup({});
|
||||||
|
|
||||||
vi.stubEnv('TERM_PROGRAM', 'vscode');
|
vi.stubEnv('TERM_PROGRAM', 'vscode');
|
||||||
vi.stubEnv('SURFACE', 'ide-1234');
|
vi.stubEnv('SURFACE', 'ide-1234');
|
||||||
|
|
||||||
const event = logger?.createLogEvent('abc', []);
|
const event = logger?.createLogEvent(EventNames.API_ERROR, []);
|
||||||
|
|
||||||
expect(event?.event_metadata[0][1]).toEqual({
|
expect(event?.event_metadata[0][1]).toEqual({
|
||||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
|
||||||
|
@ -209,7 +245,7 @@ describe('ClearcutLogger', () => {
|
||||||
vi.stubEnv(key, value);
|
vi.stubEnv(key, value);
|
||||||
}
|
}
|
||||||
vi.stubEnv('TERM_PROGRAM', 'vscode');
|
vi.stubEnv('TERM_PROGRAM', 'vscode');
|
||||||
const event = logger?.createLogEvent('abc', []);
|
const event = logger?.createLogEvent(EventNames.API_ERROR, []);
|
||||||
expect(event?.event_metadata[0][1]).toEqual({
|
expect(event?.event_metadata[0][1]).toEqual({
|
||||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE,
|
||||||
value: expectedValue,
|
value: expectedValue,
|
||||||
|
@ -218,10 +254,34 @@ describe('ClearcutLogger', () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('logChatCompressionEvent', () => {
|
||||||
|
it('logs an event with proper fields', () => {
|
||||||
|
const { logger } = setup();
|
||||||
|
logger?.logChatCompressionEvent(
|
||||||
|
makeChatCompressionEvent({
|
||||||
|
tokens_before: 9001,
|
||||||
|
tokens_after: 8000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const events = getEvents(logger!);
|
||||||
|
expect(events.length).toBe(1);
|
||||||
|
expect(events[0]).toHaveEventName(EventNames.CHAT_COMPRESSION);
|
||||||
|
expect(events[0]).toHaveMetadataValue([
|
||||||
|
EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_BEFORE,
|
||||||
|
'9001',
|
||||||
|
]);
|
||||||
|
expect(events[0]).toHaveMetadataValue([
|
||||||
|
EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_AFTER,
|
||||||
|
'8000',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('enqueueLogEvent', () => {
|
describe('enqueueLogEvent', () => {
|
||||||
it('should add events to the queue', () => {
|
it('should add events to the queue', () => {
|
||||||
const { logger } = setup();
|
const { logger } = setup();
|
||||||
logger!.enqueueLogEvent({ test: 'event1' });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
|
||||||
expect(getEventsSize(logger!)).toBe(1);
|
expect(getEventsSize(logger!)).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -229,27 +289,43 @@ describe('ClearcutLogger', () => {
|
||||||
const { logger } = setup();
|
const { logger } = setup();
|
||||||
|
|
||||||
for (let i = 0; i < TEST_ONLY.MAX_EVENTS; i++) {
|
for (let i = 0; i < TEST_ONLY.MAX_EVENTS; i++) {
|
||||||
logger!.enqueueLogEvent({ event_id: i });
|
logger!.enqueueLogEvent(
|
||||||
|
logger!.createLogEvent(EventNames.API_ERROR, [
|
||||||
|
{
|
||||||
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
|
||||||
|
value: `${i}`,
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_EVENTS);
|
let events = getEvents(logger!);
|
||||||
const firstEvent = JSON.parse(
|
expect(events.length).toBe(TEST_ONLY.MAX_EVENTS);
|
||||||
getEvents(logger!)[0][0].source_extension_json,
|
expect(events[0]).toHaveMetadataValue([
|
||||||
);
|
EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
|
||||||
expect(firstEvent.event_id).toBe(0);
|
'0',
|
||||||
|
]);
|
||||||
|
|
||||||
// This should push out the first event
|
// This should push out the first event
|
||||||
logger!.enqueueLogEvent({ event_id: TEST_ONLY.MAX_EVENTS });
|
logger!.enqueueLogEvent(
|
||||||
|
logger!.createLogEvent(EventNames.API_ERROR, [
|
||||||
|
{
|
||||||
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
|
||||||
|
value: `${TEST_ONLY.MAX_EVENTS}`,
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
events = getEvents(logger!);
|
||||||
|
expect(events.length).toBe(TEST_ONLY.MAX_EVENTS);
|
||||||
|
expect(events[0]).toHaveMetadataValue([
|
||||||
|
EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
|
||||||
|
'1',
|
||||||
|
]);
|
||||||
|
|
||||||
expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_EVENTS);
|
expect(events.at(TEST_ONLY.MAX_EVENTS - 1)).toHaveMetadataValue([
|
||||||
const newFirstEvent = JSON.parse(
|
EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
|
||||||
getEvents(logger!)[0][0].source_extension_json,
|
`${TEST_ONLY.MAX_EVENTS}`,
|
||||||
);
|
]);
|
||||||
expect(newFirstEvent.event_id).toBe(1);
|
|
||||||
const lastEvent = JSON.parse(
|
|
||||||
getEvents(logger!)[TEST_ONLY.MAX_EVENTS - 1][0].source_extension_json,
|
|
||||||
);
|
|
||||||
expect(lastEvent.event_id).toBe(TEST_ONLY.MAX_EVENTS);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -261,7 +337,7 @@ describe('ClearcutLogger', () => {
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
logger!.enqueueLogEvent({ event_id: 1 });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
|
||||||
|
|
||||||
const response = await logger!.flushToClearcut();
|
const response = await logger!.flushToClearcut();
|
||||||
|
|
||||||
|
@ -271,7 +347,7 @@ describe('ClearcutLogger', () => {
|
||||||
it('should clear events on successful flush', async () => {
|
it('should clear events on successful flush', async () => {
|
||||||
const { logger } = setup();
|
const { logger } = setup();
|
||||||
|
|
||||||
logger!.enqueueLogEvent({ event_id: 1 });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
|
||||||
const response = await logger!.flushToClearcut();
|
const response = await logger!.flushToClearcut();
|
||||||
|
|
||||||
expect(getEvents(logger!)).toEqual([]);
|
expect(getEvents(logger!)).toEqual([]);
|
||||||
|
@ -282,8 +358,8 @@ describe('ClearcutLogger', () => {
|
||||||
const { logger } = setup();
|
const { logger } = setup();
|
||||||
|
|
||||||
server.resetHandlers(http.post(CLEARCUT_URL, () => HttpResponse.error()));
|
server.resetHandlers(http.post(CLEARCUT_URL, () => HttpResponse.error()));
|
||||||
logger!.enqueueLogEvent({ event_id: 1 });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_REQUEST));
|
||||||
logger!.enqueueLogEvent({ event_id: 2 });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
|
||||||
expect(getEventsSize(logger!)).toBe(2);
|
expect(getEventsSize(logger!)).toBe(2);
|
||||||
|
|
||||||
const x = logger!.flushToClearcut();
|
const x = logger!.flushToClearcut();
|
||||||
|
@ -291,7 +367,9 @@ describe('ClearcutLogger', () => {
|
||||||
|
|
||||||
expect(getEventsSize(logger!)).toBe(2);
|
expect(getEventsSize(logger!)).toBe(2);
|
||||||
const events = getEvents(logger!);
|
const events = getEvents(logger!);
|
||||||
expect(JSON.parse(events[0][0].source_extension_json).event_id).toBe(1);
|
|
||||||
|
expect(events.length).toBe(2);
|
||||||
|
expect(events[0]).toHaveEventName(EventNames.API_REQUEST);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should handle an HTTP error and requeue events', async () => {
|
it('should handle an HTTP error and requeue events', async () => {
|
||||||
|
@ -310,23 +388,22 @@ describe('ClearcutLogger', () => {
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
logger!.enqueueLogEvent({ event_id: 1 });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_REQUEST));
|
||||||
logger!.enqueueLogEvent({ event_id: 2 });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
|
||||||
|
|
||||||
expect(getEvents(logger!).length).toBe(2);
|
expect(getEvents(logger!).length).toBe(2);
|
||||||
await logger!.flushToClearcut();
|
await logger!.flushToClearcut();
|
||||||
|
|
||||||
expect(getEvents(logger!).length).toBe(2);
|
|
||||||
const events = getEvents(logger!);
|
const events = getEvents(logger!);
|
||||||
expect(JSON.parse(events[0][0].source_extension_json).event_id).toBe(1);
|
|
||||||
|
expect(events[0]).toHaveEventName(EventNames.API_REQUEST);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('requeueFailedEvents logic', () => {
|
describe('requeueFailedEvents logic', () => {
|
||||||
it('should limit the number of requeued events to max_retry_events', () => {
|
it('should limit the number of requeued events to max_retry_events', () => {
|
||||||
const { logger } = setup();
|
const { logger } = setup();
|
||||||
const maxRetryEvents = TEST_ONLY.MAX_RETRY_EVENTS;
|
const eventsToLogCount = TEST_ONLY.MAX_RETRY_EVENTS + 5;
|
||||||
const eventsToLogCount = maxRetryEvents + 5;
|
|
||||||
const eventsToSend: LogEventEntry[][] = [];
|
const eventsToSend: LogEventEntry[][] = [];
|
||||||
for (let i = 0; i < eventsToLogCount; i++) {
|
for (let i = 0; i < eventsToLogCount; i++) {
|
||||||
eventsToSend.push([
|
eventsToSend.push([
|
||||||
|
@ -339,13 +416,13 @@ describe('ClearcutLogger', () => {
|
||||||
|
|
||||||
requeueFailedEvents(logger!, eventsToSend);
|
requeueFailedEvents(logger!, eventsToSend);
|
||||||
|
|
||||||
expect(getEventsSize(logger!)).toBe(maxRetryEvents);
|
expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_RETRY_EVENTS);
|
||||||
const firstRequeuedEvent = JSON.parse(
|
const firstRequeuedEvent = JSON.parse(
|
||||||
getEvents(logger!)[0][0].source_extension_json,
|
getEvents(logger!)[0][0].source_extension_json,
|
||||||
);
|
) as { event_id: string };
|
||||||
// The last `maxRetryEvents` are kept. The oldest of those is at index `eventsToLogCount - maxRetryEvents`.
|
// The last `maxRetryEvents` are kept. The oldest of those is at index `eventsToLogCount - maxRetryEvents`.
|
||||||
expect(firstRequeuedEvent.event_id).toBe(
|
expect(firstRequeuedEvent.event_id).toBe(
|
||||||
eventsToLogCount - maxRetryEvents,
|
eventsToLogCount - TEST_ONLY.MAX_RETRY_EVENTS,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -355,7 +432,7 @@ describe('ClearcutLogger', () => {
|
||||||
const spaceToLeave = 5;
|
const spaceToLeave = 5;
|
||||||
const initialEventCount = maxEvents - spaceToLeave;
|
const initialEventCount = maxEvents - spaceToLeave;
|
||||||
for (let i = 0; i < initialEventCount; i++) {
|
for (let i = 0; i < initialEventCount; i++) {
|
||||||
logger!.enqueueLogEvent({ event_id: `initial_${i}` });
|
logger!.enqueueLogEvent(logger!.createLogEvent(EventNames.API_ERROR));
|
||||||
}
|
}
|
||||||
expect(getEventsSize(logger!)).toBe(initialEventCount);
|
expect(getEventsSize(logger!)).toBe(initialEventCount);
|
||||||
|
|
||||||
|
@ -382,7 +459,7 @@ describe('ClearcutLogger', () => {
|
||||||
// The first element in the deque is the one with id 'failed_5'.
|
// The first element in the deque is the one with id 'failed_5'.
|
||||||
const firstRequeuedEvent = JSON.parse(
|
const firstRequeuedEvent = JSON.parse(
|
||||||
getEvents(logger!)[0][0].source_extension_json,
|
getEvents(logger!)[0][0].source_extension_json,
|
||||||
);
|
) as { event_id: string };
|
||||||
expect(firstRequeuedEvent.event_id).toBe('failed_5');
|
expect(firstRequeuedEvent.event_id).toBe('failed_5');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -20,6 +20,7 @@ import {
|
||||||
MalformedJsonResponseEvent,
|
MalformedJsonResponseEvent,
|
||||||
IdeConnectionEvent,
|
IdeConnectionEvent,
|
||||||
KittySequenceOverflowEvent,
|
KittySequenceOverflowEvent,
|
||||||
|
ChatCompressionEvent,
|
||||||
} from '../types.js';
|
} from '../types.js';
|
||||||
import { EventMetadataKey } from './event-metadata-key.js';
|
import { EventMetadataKey } from './event-metadata-key.js';
|
||||||
import { Config } from '../../config/config.js';
|
import { Config } from '../../config/config.js';
|
||||||
|
@ -33,20 +34,23 @@ import { FixedDeque } from 'mnemonist';
|
||||||
import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js';
|
import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js';
|
||||||
import { DetectedIde, detectIde } from '../../ide/detect-ide.js';
|
import { DetectedIde, detectIde } from '../../ide/detect-ide.js';
|
||||||
|
|
||||||
const start_session_event_name = 'start_session';
|
export enum EventNames {
|
||||||
const new_prompt_event_name = 'new_prompt';
|
START_SESSION = 'start_session',
|
||||||
const tool_call_event_name = 'tool_call';
|
NEW_PROMPT = 'new_prompt',
|
||||||
const api_request_event_name = 'api_request';
|
TOOL_CALL = 'tool_call',
|
||||||
const api_response_event_name = 'api_response';
|
API_REQUEST = 'api_request',
|
||||||
const api_error_event_name = 'api_error';
|
API_RESPONSE = 'api_response',
|
||||||
const end_session_event_name = 'end_session';
|
API_ERROR = 'api_error',
|
||||||
const flash_fallback_event_name = 'flash_fallback';
|
END_SESSION = 'end_session',
|
||||||
const loop_detected_event_name = 'loop_detected';
|
FLASH_FALLBACK = 'flash_fallback',
|
||||||
const next_speaker_check_event_name = 'next_speaker_check';
|
LOOP_DETECTED = 'loop_detected',
|
||||||
const slash_command_event_name = 'slash_command';
|
NEXT_SPEAKER_CHECK = 'next_speaker_check',
|
||||||
const malformed_json_response_event_name = 'malformed_json_response';
|
SLASH_COMMAND = 'slash_command',
|
||||||
const ide_connection_event_name = 'ide_connection';
|
MALFORMED_JSON_RESPONSE = 'malformed_json_response',
|
||||||
const kitty_sequence_overflow_event_name = 'kitty_sequence_overflow';
|
IDE_CONNECTION = 'ide_connection',
|
||||||
|
KITTY_SEQUENCE_OVERFLOW = 'kitty_sequence_overflow',
|
||||||
|
CHAT_COMPRESSION = 'chat_compression',
|
||||||
|
}
|
||||||
|
|
||||||
export interface LogResponse {
|
export interface LogResponse {
|
||||||
nextRequestWaitMs?: number;
|
nextRequestWaitMs?: number;
|
||||||
|
@ -58,7 +62,7 @@ export interface LogEventEntry {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface EventValue {
|
export interface EventValue {
|
||||||
gemini_cli_key: EventMetadataKey | string;
|
gemini_cli_key: EventMetadataKey;
|
||||||
value: string;
|
value: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -168,7 +172,7 @@ export class ClearcutLogger {
|
||||||
ClearcutLogger.instance = undefined;
|
ClearcutLogger.instance = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
enqueueLogEvent(event: object): void {
|
enqueueLogEvent(event: LogEvent): void {
|
||||||
try {
|
try {
|
||||||
// Manually handle overflow for FixedDeque, which throws when full.
|
// Manually handle overflow for FixedDeque, which throws when full.
|
||||||
const wasAtCapacity = this.events.size >= MAX_EVENTS;
|
const wasAtCapacity = this.events.size >= MAX_EVENTS;
|
||||||
|
@ -196,7 +200,7 @@ export class ClearcutLogger {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
createLogEvent(name: string, data: EventValue[]): LogEvent {
|
createLogEvent(eventName: EventNames, data: EventValue[] = []): LogEvent {
|
||||||
const email = getCachedGoogleAccount();
|
const email = getCachedGoogleAccount();
|
||||||
|
|
||||||
data = addDefaultFields(data);
|
data = addDefaultFields(data);
|
||||||
|
@ -204,7 +208,7 @@ export class ClearcutLogger {
|
||||||
const logEvent: LogEvent = {
|
const logEvent: LogEvent = {
|
||||||
console_type: 'GEMINI_CLI',
|
console_type: 'GEMINI_CLI',
|
||||||
application: 102, // GEMINI_CLI
|
application: 102, // GEMINI_CLI
|
||||||
event_name: name,
|
event_name: eventName as string,
|
||||||
event_metadata: [data],
|
event_metadata: [data],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -386,7 +390,7 @@ export class ClearcutLogger {
|
||||||
];
|
];
|
||||||
|
|
||||||
// Flush start event immediately
|
// Flush start event immediately
|
||||||
this.enqueueLogEvent(this.createLogEvent(start_session_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.START_SESSION, data));
|
||||||
this.flushToClearcut().catch((error) => {
|
this.flushToClearcut().catch((error) => {
|
||||||
console.debug('Error flushing to Clearcut:', error);
|
console.debug('Error flushing to Clearcut:', error);
|
||||||
});
|
});
|
||||||
|
@ -412,7 +416,7 @@ export class ClearcutLogger {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.NEW_PROMPT, data));
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -466,7 +470,7 @@ export class ClearcutLogger {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const logEvent = this.createLogEvent(tool_call_event_name, data);
|
const logEvent = this.createLogEvent(EventNames.TOOL_CALL, data);
|
||||||
this.enqueueLogEvent(logEvent);
|
this.enqueueLogEvent(logEvent);
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
@ -483,7 +487,7 @@ export class ClearcutLogger {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(api_request_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.API_REQUEST, data));
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -540,7 +544,7 @@ export class ClearcutLogger {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.API_RESPONSE, data));
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -572,10 +576,27 @@ export class ClearcutLogger {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.API_ERROR, data));
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logChatCompressionEvent(event: ChatCompressionEvent): void {
|
||||||
|
const data: EventValue[] = [
|
||||||
|
{
|
||||||
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_BEFORE,
|
||||||
|
value: `${event.tokens_before}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
gemini_cli_key: EventMetadataKey.GEMINI_CLI_COMPRESSION_TOKENS_AFTER,
|
||||||
|
value: `${event.tokens_after}`,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
this.enqueueLogEvent(
|
||||||
|
this.createLogEvent(EventNames.CHAT_COMPRESSION, data),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
logFlashFallbackEvent(event: FlashFallbackEvent): void {
|
logFlashFallbackEvent(event: FlashFallbackEvent): void {
|
||||||
const data: EventValue[] = [
|
const data: EventValue[] = [
|
||||||
{
|
{
|
||||||
|
@ -588,7 +609,7 @@ export class ClearcutLogger {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(flash_fallback_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.FLASH_FALLBACK, data));
|
||||||
this.flushToClearcut().catch((error) => {
|
this.flushToClearcut().catch((error) => {
|
||||||
console.debug('Error flushing to Clearcut:', error);
|
console.debug('Error flushing to Clearcut:', error);
|
||||||
});
|
});
|
||||||
|
@ -606,7 +627,7 @@ export class ClearcutLogger {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(loop_detected_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.LOOP_DETECTED, data));
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -631,7 +652,7 @@ export class ClearcutLogger {
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(
|
this.enqueueLogEvent(
|
||||||
this.createLogEvent(next_speaker_check_event_name, data),
|
this.createLogEvent(EventNames.NEXT_SPEAKER_CHECK, data),
|
||||||
);
|
);
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
@ -658,7 +679,7 @@ export class ClearcutLogger {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(slash_command_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.SLASH_COMMAND, data));
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -672,7 +693,7 @@ export class ClearcutLogger {
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(
|
this.enqueueLogEvent(
|
||||||
this.createLogEvent(malformed_json_response_event_name, data),
|
this.createLogEvent(EventNames.MALFORMED_JSON_RESPONSE, data),
|
||||||
);
|
);
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
@ -685,7 +706,7 @@ export class ClearcutLogger {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(this.createLogEvent(ide_connection_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.IDE_CONNECTION, data));
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -702,7 +723,7 @@ export class ClearcutLogger {
|
||||||
];
|
];
|
||||||
|
|
||||||
this.enqueueLogEvent(
|
this.enqueueLogEvent(
|
||||||
this.createLogEvent(kitty_sequence_overflow_event_name, data),
|
this.createLogEvent(EventNames.KITTY_SEQUENCE_OVERFLOW, data),
|
||||||
);
|
);
|
||||||
this.flushIfNeeded();
|
this.flushIfNeeded();
|
||||||
}
|
}
|
||||||
|
@ -716,7 +737,7 @@ export class ClearcutLogger {
|
||||||
];
|
];
|
||||||
|
|
||||||
// Flush immediately on session end.
|
// Flush immediately on session end.
|
||||||
this.enqueueLogEvent(this.createLogEvent(end_session_event_name, data));
|
this.enqueueLogEvent(this.createLogEvent(EventNames.END_SESSION, data));
|
||||||
this.flushToClearcut().catch((error) => {
|
this.flushToClearcut().catch((error) => {
|
||||||
console.debug('Error flushing to Clearcut:', error);
|
console.debug('Error flushing to Clearcut:', error);
|
||||||
});
|
});
|
||||||
|
|
|
@ -228,17 +228,10 @@ export enum EventMetadataKey {
|
||||||
|
|
||||||
// Logs the length of the kitty sequence that overflowed.
|
// Logs the length of the kitty sequence that overflowed.
|
||||||
GEMINI_CLI_KITTY_SEQUENCE_LENGTH = 53,
|
GEMINI_CLI_KITTY_SEQUENCE_LENGTH = 53,
|
||||||
}
|
|
||||||
|
|
||||||
export function getEventMetadataKey(
|
// Logs the number of tokens before context window compression.
|
||||||
keyName: string,
|
GEMINI_CLI_COMPRESSION_TOKENS_BEFORE = 60,
|
||||||
): EventMetadataKey | undefined {
|
|
||||||
// Access the enum member by its string name
|
|
||||||
const key = EventMetadataKey[keyName as keyof typeof EventMetadataKey];
|
|
||||||
|
|
||||||
// Check if the result is a valid enum member (not undefined and is a number)
|
// Logs the number of tokens after context window compression.
|
||||||
if (typeof key === 'number') {
|
GEMINI_CLI_COMPRESSION_TOKENS_AFTER = 61,
|
||||||
return key;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,7 +16,7 @@ export const EVENT_FLASH_FALLBACK = 'gemini_cli.flash_fallback';
|
||||||
export const EVENT_NEXT_SPEAKER_CHECK = 'gemini_cli.next_speaker_check';
|
export const EVENT_NEXT_SPEAKER_CHECK = 'gemini_cli.next_speaker_check';
|
||||||
export const EVENT_SLASH_COMMAND = 'gemini_cli.slash_command';
|
export const EVENT_SLASH_COMMAND = 'gemini_cli.slash_command';
|
||||||
export const EVENT_IDE_CONNECTION = 'gemini_cli.ide_connection';
|
export const EVENT_IDE_CONNECTION = 'gemini_cli.ide_connection';
|
||||||
|
export const EVENT_CHAT_COMPRESSION = 'gemini_cli.chat_compression';
|
||||||
export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
|
export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
|
||||||
export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
|
export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
|
||||||
export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count';
|
export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count';
|
||||||
|
|
|
@ -28,6 +28,7 @@ export {
|
||||||
logFlashFallback,
|
logFlashFallback,
|
||||||
logSlashCommand,
|
logSlashCommand,
|
||||||
logKittySequenceOverflow,
|
logKittySequenceOverflow,
|
||||||
|
logChatCompression,
|
||||||
} from './loggers.js';
|
} from './loggers.js';
|
||||||
export {
|
export {
|
||||||
StartSessionEvent,
|
StartSessionEvent,
|
||||||
|
@ -43,6 +44,8 @@ export {
|
||||||
SlashCommandEvent,
|
SlashCommandEvent,
|
||||||
makeSlashCommandEvent,
|
makeSlashCommandEvent,
|
||||||
SlashCommandStatus,
|
SlashCommandStatus,
|
||||||
|
ChatCompressionEvent,
|
||||||
|
makeChatCompressionEvent,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
|
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
|
||||||
export { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
export { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
||||||
|
|
|
@ -56,7 +56,8 @@ describe('Circular Reference Integration Test', () => {
|
||||||
const logger = ClearcutLogger.getInstance(mockConfig);
|
const logger = ClearcutLogger.getInstance(mockConfig);
|
||||||
|
|
||||||
expect(() => {
|
expect(() => {
|
||||||
logger?.enqueueLogEvent(problematicEvent);
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
logger?.enqueueLogEvent(problematicEvent as any);
|
||||||
}).not.toThrow();
|
}).not.toThrow();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -34,6 +34,7 @@ import {
|
||||||
logUserPrompt,
|
logUserPrompt,
|
||||||
logToolCall,
|
logToolCall,
|
||||||
logFlashFallback,
|
logFlashFallback,
|
||||||
|
logChatCompression,
|
||||||
} from './loggers.js';
|
} from './loggers.js';
|
||||||
import { ToolCallDecision } from './tool-call-decision.js';
|
import { ToolCallDecision } from './tool-call-decision.js';
|
||||||
import {
|
import {
|
||||||
|
@ -43,12 +44,15 @@ import {
|
||||||
ToolCallEvent,
|
ToolCallEvent,
|
||||||
UserPromptEvent,
|
UserPromptEvent,
|
||||||
FlashFallbackEvent,
|
FlashFallbackEvent,
|
||||||
|
makeChatCompressionEvent,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
import * as metrics from './metrics.js';
|
import * as metrics from './metrics.js';
|
||||||
import * as sdk from './sdk.js';
|
import * as sdk from './sdk.js';
|
||||||
import { vi, describe, beforeEach, it, expect } from 'vitest';
|
import { vi, describe, beforeEach, it, expect } from 'vitest';
|
||||||
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
||||||
import * as uiTelemetry from './uiTelemetry.js';
|
import * as uiTelemetry from './uiTelemetry.js';
|
||||||
|
import { makeFakeConfig } from '../test-utils/config.js';
|
||||||
|
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
|
||||||
|
|
||||||
describe('loggers', () => {
|
describe('loggers', () => {
|
||||||
const mockLogger = {
|
const mockLogger = {
|
||||||
|
@ -68,6 +72,45 @@ describe('loggers', () => {
|
||||||
vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z'));
|
vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z'));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('logChatCompression', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.spyOn(metrics, 'recordChatCompressionMetrics');
|
||||||
|
vi.spyOn(ClearcutLogger.prototype, 'logChatCompressionEvent');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('logs the chat compression event to Clearcut', () => {
|
||||||
|
const mockConfig = makeFakeConfig();
|
||||||
|
|
||||||
|
const event = makeChatCompressionEvent({
|
||||||
|
tokens_before: 9001,
|
||||||
|
tokens_after: 9000,
|
||||||
|
});
|
||||||
|
|
||||||
|
logChatCompression(mockConfig, event);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
ClearcutLogger.prototype.logChatCompressionEvent,
|
||||||
|
).toHaveBeenCalledWith(event);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('records the chat compression event to OTEL', () => {
|
||||||
|
const mockConfig = makeFakeConfig();
|
||||||
|
|
||||||
|
logChatCompression(
|
||||||
|
mockConfig,
|
||||||
|
makeChatCompressionEvent({
|
||||||
|
tokens_before: 9001,
|
||||||
|
tokens_after: 9000,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(metrics.recordChatCompressionMetrics).toHaveBeenCalledWith(
|
||||||
|
mockConfig,
|
||||||
|
{ tokens_before: 9001, tokens_after: 9000 },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('logCliConfiguration', () => {
|
describe('logCliConfiguration', () => {
|
||||||
it('should log the cli configuration', () => {
|
it('should log the cli configuration', () => {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
|
|
|
@ -19,6 +19,7 @@ import {
|
||||||
EVENT_NEXT_SPEAKER_CHECK,
|
EVENT_NEXT_SPEAKER_CHECK,
|
||||||
SERVICE_NAME,
|
SERVICE_NAME,
|
||||||
EVENT_SLASH_COMMAND,
|
EVENT_SLASH_COMMAND,
|
||||||
|
EVENT_CHAT_COMPRESSION,
|
||||||
} from './constants.js';
|
} from './constants.js';
|
||||||
import {
|
import {
|
||||||
ApiErrorEvent,
|
ApiErrorEvent,
|
||||||
|
@ -33,12 +34,14 @@ import {
|
||||||
LoopDetectedEvent,
|
LoopDetectedEvent,
|
||||||
SlashCommandEvent,
|
SlashCommandEvent,
|
||||||
KittySequenceOverflowEvent,
|
KittySequenceOverflowEvent,
|
||||||
|
ChatCompressionEvent,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
import {
|
import {
|
||||||
recordApiErrorMetrics,
|
recordApiErrorMetrics,
|
||||||
recordTokenUsageMetrics,
|
recordTokenUsageMetrics,
|
||||||
recordApiResponseMetrics,
|
recordApiResponseMetrics,
|
||||||
recordToolCallMetrics,
|
recordToolCallMetrics,
|
||||||
|
recordChatCompressionMetrics,
|
||||||
} from './metrics.js';
|
} from './metrics.js';
|
||||||
import { isTelemetrySdkInitialized } from './sdk.js';
|
import { isTelemetrySdkInitialized } from './sdk.js';
|
||||||
import { uiTelemetryService, UiEvent } from './uiTelemetry.js';
|
import { uiTelemetryService, UiEvent } from './uiTelemetry.js';
|
||||||
|
@ -380,6 +383,31 @@ export function logIdeConnection(
|
||||||
logger.emit(logRecord);
|
logger.emit(logRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function logChatCompression(
|
||||||
|
config: Config,
|
||||||
|
event: ChatCompressionEvent,
|
||||||
|
): void {
|
||||||
|
ClearcutLogger.getInstance(config)?.logChatCompressionEvent(event);
|
||||||
|
|
||||||
|
const attributes: LogAttributes = {
|
||||||
|
...getCommonAttributes(config),
|
||||||
|
...event,
|
||||||
|
'event.name': EVENT_CHAT_COMPRESSION,
|
||||||
|
};
|
||||||
|
|
||||||
|
const logger = logs.getLogger(SERVICE_NAME);
|
||||||
|
const logRecord: LogRecord = {
|
||||||
|
body: `Chat compression (Saved ${event.tokens_before - event.tokens_after} tokens)`,
|
||||||
|
attributes,
|
||||||
|
};
|
||||||
|
logger.emit(logRecord);
|
||||||
|
|
||||||
|
recordChatCompressionMetrics(config, {
|
||||||
|
tokens_before: event.tokens_before,
|
||||||
|
tokens_after: event.tokens_after,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function logKittySequenceOverflow(
|
export function logKittySequenceOverflow(
|
||||||
config: Config,
|
config: Config,
|
||||||
event: KittySequenceOverflowEvent,
|
event: KittySequenceOverflowEvent,
|
||||||
|
|
|
@ -14,6 +14,7 @@ import type {
|
||||||
} from '@opentelemetry/api';
|
} from '@opentelemetry/api';
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import { FileOperation } from './metrics.js';
|
import { FileOperation } from './metrics.js';
|
||||||
|
import { makeFakeConfig } from '../test-utils/config.js';
|
||||||
|
|
||||||
const mockCounterAddFn: Mock<
|
const mockCounterAddFn: Mock<
|
||||||
(value: number, attributes?: Attributes, context?: Context) => void
|
(value: number, attributes?: Attributes, context?: Context) => void
|
||||||
|
@ -28,18 +29,18 @@ const mockCreateHistogramFn: Mock<
|
||||||
(name: string, options?: unknown) => Histogram
|
(name: string, options?: unknown) => Histogram
|
||||||
> = vi.fn();
|
> = vi.fn();
|
||||||
|
|
||||||
const mockCounterInstance = {
|
const mockCounterInstance: Counter = {
|
||||||
add: mockCounterAddFn,
|
add: mockCounterAddFn,
|
||||||
} as unknown as Counter;
|
} as Partial<Counter> as Counter;
|
||||||
|
|
||||||
const mockHistogramInstance = {
|
const mockHistogramInstance: Histogram = {
|
||||||
record: mockHistogramRecordFn,
|
record: mockHistogramRecordFn,
|
||||||
} as unknown as Histogram;
|
} as Partial<Histogram> as Histogram;
|
||||||
|
|
||||||
const mockMeterInstance = {
|
const mockMeterInstance: Meter = {
|
||||||
createCounter: mockCreateCounterFn.mockReturnValue(mockCounterInstance),
|
createCounter: mockCreateCounterFn.mockReturnValue(mockCounterInstance),
|
||||||
createHistogram: mockCreateHistogramFn.mockReturnValue(mockHistogramInstance),
|
createHistogram: mockCreateHistogramFn.mockReturnValue(mockHistogramInstance),
|
||||||
} as unknown as Meter;
|
} as Partial<Meter> as Meter;
|
||||||
|
|
||||||
function originalOtelMockFactory() {
|
function originalOtelMockFactory() {
|
||||||
return {
|
return {
|
||||||
|
@ -49,15 +50,19 @@ function originalOtelMockFactory() {
|
||||||
ValueType: {
|
ValueType: {
|
||||||
INT: 1,
|
INT: 1,
|
||||||
},
|
},
|
||||||
|
diag: {
|
||||||
|
setLogger: vi.fn(),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
vi.mock('@opentelemetry/api', originalOtelMockFactory);
|
vi.mock('@opentelemetry/api');
|
||||||
|
|
||||||
describe('Telemetry Metrics', () => {
|
describe('Telemetry Metrics', () => {
|
||||||
let initializeMetricsModule: typeof import('./metrics.js').initializeMetrics;
|
let initializeMetricsModule: typeof import('./metrics.js').initializeMetrics;
|
||||||
let recordTokenUsageMetricsModule: typeof import('./metrics.js').recordTokenUsageMetrics;
|
let recordTokenUsageMetricsModule: typeof import('./metrics.js').recordTokenUsageMetrics;
|
||||||
let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric;
|
let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric;
|
||||||
|
let recordChatCompressionMetricsModule: typeof import('./metrics.js').recordChatCompressionMetrics;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
vi.resetModules();
|
vi.resetModules();
|
||||||
|
@ -71,6 +76,8 @@ describe('Telemetry Metrics', () => {
|
||||||
initializeMetricsModule = metricsJsModule.initializeMetrics;
|
initializeMetricsModule = metricsJsModule.initializeMetrics;
|
||||||
recordTokenUsageMetricsModule = metricsJsModule.recordTokenUsageMetrics;
|
recordTokenUsageMetricsModule = metricsJsModule.recordTokenUsageMetrics;
|
||||||
recordFileOperationMetricModule = metricsJsModule.recordFileOperationMetric;
|
recordFileOperationMetricModule = metricsJsModule.recordFileOperationMetric;
|
||||||
|
recordChatCompressionMetricsModule =
|
||||||
|
metricsJsModule.recordChatCompressionMetrics;
|
||||||
|
|
||||||
const otelApiModule = await import('@opentelemetry/api');
|
const otelApiModule = await import('@opentelemetry/api');
|
||||||
|
|
||||||
|
@ -85,6 +92,35 @@ describe('Telemetry Metrics', () => {
|
||||||
mockCreateHistogramFn.mockReturnValue(mockHistogramInstance);
|
mockCreateHistogramFn.mockReturnValue(mockHistogramInstance);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('recordChatCompressionMetrics', () => {
|
||||||
|
it('does not record metrics if not initialized', () => {
|
||||||
|
const lol = makeFakeConfig({});
|
||||||
|
|
||||||
|
recordChatCompressionMetricsModule(lol, {
|
||||||
|
tokens_after: 100,
|
||||||
|
tokens_before: 200,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockCounterAddFn).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('records token compression with the correct attributes', () => {
|
||||||
|
const config = makeFakeConfig({});
|
||||||
|
initializeMetricsModule(config);
|
||||||
|
|
||||||
|
recordChatCompressionMetricsModule(config, {
|
||||||
|
tokens_after: 100,
|
||||||
|
tokens_before: 200,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
|
||||||
|
'session.id': 'test-session-id',
|
||||||
|
tokens_after: 100,
|
||||||
|
tokens_before: 200,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('recordTokenUsageMetrics', () => {
|
describe('recordTokenUsageMetrics', () => {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
|
|
|
@ -21,6 +21,7 @@ import {
|
||||||
METRIC_TOKEN_USAGE,
|
METRIC_TOKEN_USAGE,
|
||||||
METRIC_SESSION_COUNT,
|
METRIC_SESSION_COUNT,
|
||||||
METRIC_FILE_OPERATION_COUNT,
|
METRIC_FILE_OPERATION_COUNT,
|
||||||
|
EVENT_CHAT_COMPRESSION,
|
||||||
} from './constants.js';
|
} from './constants.js';
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import { DiffStat } from '../tools/tools.js';
|
import { DiffStat } from '../tools/tools.js';
|
||||||
|
@ -38,6 +39,7 @@ let apiRequestCounter: Counter | undefined;
|
||||||
let apiRequestLatencyHistogram: Histogram | undefined;
|
let apiRequestLatencyHistogram: Histogram | undefined;
|
||||||
let tokenUsageCounter: Counter | undefined;
|
let tokenUsageCounter: Counter | undefined;
|
||||||
let fileOperationCounter: Counter | undefined;
|
let fileOperationCounter: Counter | undefined;
|
||||||
|
let chatCompressionCounter: Counter | undefined;
|
||||||
let isMetricsInitialized = false;
|
let isMetricsInitialized = false;
|
||||||
|
|
||||||
function getCommonAttributes(config: Config): Attributes {
|
function getCommonAttributes(config: Config): Attributes {
|
||||||
|
@ -88,6 +90,10 @@ export function initializeMetrics(config: Config): void {
|
||||||
description: 'Counts file operations (create, read, update).',
|
description: 'Counts file operations (create, read, update).',
|
||||||
valueType: ValueType.INT,
|
valueType: ValueType.INT,
|
||||||
});
|
});
|
||||||
|
chatCompressionCounter = meter.createCounter(EVENT_CHAT_COMPRESSION, {
|
||||||
|
description: 'Counts chat compression events.',
|
||||||
|
valueType: ValueType.INT,
|
||||||
|
});
|
||||||
const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, {
|
const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, {
|
||||||
description: 'Count of CLI sessions started.',
|
description: 'Count of CLI sessions started.',
|
||||||
valueType: ValueType.INT,
|
valueType: ValueType.INT,
|
||||||
|
@ -96,6 +102,17 @@ export function initializeMetrics(config: Config): void {
|
||||||
isMetricsInitialized = true;
|
isMetricsInitialized = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function recordChatCompressionMetrics(
|
||||||
|
config: Config,
|
||||||
|
args: { tokens_before: number; tokens_after: number },
|
||||||
|
) {
|
||||||
|
if (!chatCompressionCounter || !isMetricsInitialized) return;
|
||||||
|
chatCompressionCounter.add(1, {
|
||||||
|
...getCommonAttributes(config),
|
||||||
|
...args,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function recordToolCallMetrics(
|
export function recordToolCallMetrics(
|
||||||
config: Config,
|
config: Config,
|
||||||
functionName: string,
|
functionName: string,
|
||||||
|
|
|
@ -14,7 +14,7 @@ import {
|
||||||
ToolCallDecision,
|
ToolCallDecision,
|
||||||
} from './tool-call-decision.js';
|
} from './tool-call-decision.js';
|
||||||
|
|
||||||
interface BaseTelemetryEvent {
|
export interface BaseTelemetryEvent {
|
||||||
'event.name': string;
|
'event.name': string;
|
||||||
/** Current timestamp in ISO 8601 format */
|
/** Current timestamp in ISO 8601 format */
|
||||||
'event.timestamp': string;
|
'event.timestamp': string;
|
||||||
|
@ -292,7 +292,7 @@ export class NextSpeakerCheckEvent implements BaseTelemetryEvent {
|
||||||
|
|
||||||
export interface SlashCommandEvent extends BaseTelemetryEvent {
|
export interface SlashCommandEvent extends BaseTelemetryEvent {
|
||||||
'event.name': 'slash_command';
|
'event.name': 'slash_command';
|
||||||
'event.timestamp': string; // ISO 8106
|
'event.timestamp': string;
|
||||||
command: string;
|
command: string;
|
||||||
subcommand?: string;
|
subcommand?: string;
|
||||||
status?: SlashCommandStatus;
|
status?: SlashCommandStatus;
|
||||||
|
@ -317,6 +317,25 @@ export enum SlashCommandStatus {
|
||||||
ERROR = 'error',
|
ERROR = 'error',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ChatCompressionEvent extends BaseTelemetryEvent {
|
||||||
|
'event.name': 'chat_compression';
|
||||||
|
'event.timestamp': string;
|
||||||
|
tokens_before: number;
|
||||||
|
tokens_after: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function makeChatCompressionEvent({
|
||||||
|
tokens_before,
|
||||||
|
tokens_after,
|
||||||
|
}: Omit<ChatCompressionEvent, CommonFields>): ChatCompressionEvent {
|
||||||
|
return {
|
||||||
|
'event.name': 'chat_compression',
|
||||||
|
'event.timestamp': new Date().toISOString(),
|
||||||
|
tokens_before,
|
||||||
|
tokens_after,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export class MalformedJsonResponseEvent implements BaseTelemetryEvent {
|
export class MalformedJsonResponseEvent implements BaseTelemetryEvent {
|
||||||
'event.name': 'malformed_json_response';
|
'event.name': 'malformed_json_response';
|
||||||
'event.timestamp': string;
|
'event.timestamp': string;
|
||||||
|
|
Loading…
Reference in New Issue