Clearcut logging - initial implementation (#1274)
Flag-guarded initial implementation of a clearcut logger to collect telemetry data and send it to Concord for dashboards, etc.
This commit is contained in:
parent
c9950b3cb2
commit
4cfab0a893
|
@ -11,13 +11,11 @@ import { Settings } from './settings.js';
|
||||||
import { Extension } from './extension.js';
|
import { Extension } from './extension.js';
|
||||||
import * as ServerConfig from '@gemini-cli/core';
|
import * as ServerConfig from '@gemini-cli/core';
|
||||||
|
|
||||||
const MOCK_HOME_DIR = '/mock/home/user';
|
|
||||||
|
|
||||||
vi.mock('os', async (importOriginal) => {
|
vi.mock('os', async (importOriginal) => {
|
||||||
const actualOs = await importOriginal<typeof os>();
|
const actualOs = await importOriginal<typeof os>();
|
||||||
return {
|
return {
|
||||||
...actualOs,
|
...actualOs,
|
||||||
homedir: vi.fn(() => MOCK_HOME_DIR),
|
homedir: vi.fn(() => '/mock/home/user'),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -53,7 +51,7 @@ describe('loadCliConfig', () => {
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR);
|
vi.mocked(os.homedir).mockReturnValue('/mock/home/user');
|
||||||
process.env.GEMINI_API_KEY = 'test-api-key'; // Ensure API key is set for tests
|
process.env.GEMINI_API_KEY = 'test-api-key'; // Ensure API key is set for tests
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -98,7 +96,7 @@ describe('loadCliConfig telemetry', () => {
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR);
|
vi.mocked(os.homedir).mockReturnValue('/mock/home/user');
|
||||||
process.env.GEMINI_API_KEY = 'test-api-key';
|
process.env.GEMINI_API_KEY = 'test-api-key';
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -250,7 +248,7 @@ describe('loadCliConfig telemetry', () => {
|
||||||
describe('Hierarchical Memory Loading (config.ts) - Placeholder Suite', () => {
|
describe('Hierarchical Memory Loading (config.ts) - Placeholder Suite', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
vi.mocked(os.homedir).mockReturnValue(MOCK_HOME_DIR);
|
vi.mocked(os.homedir).mockReturnValue('/mock/home/user');
|
||||||
// Other common mocks would be reset here.
|
// Other common mocks would be reset here.
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -310,7 +308,7 @@ describe('Hierarchical Memory Loading (config.ts) - Placeholder Suite', () => {
|
||||||
// Example of a previously failing test structure:
|
// Example of a previously failing test structure:
|
||||||
/*
|
/*
|
||||||
it('should correctly use mocked homedir for global path', async () => {
|
it('should correctly use mocked homedir for global path', async () => {
|
||||||
const MOCK_GEMINI_DIR_LOCAL = path.join(MOCK_HOME_DIR, '.gemini');
|
const MOCK_GEMINI_DIR_LOCAL = path.join('/mock/home/user', '.gemini');
|
||||||
const MOCK_GLOBAL_PATH_LOCAL = path.join(MOCK_GEMINI_DIR_LOCAL, 'GEMINI.md');
|
const MOCK_GLOBAL_PATH_LOCAL = path.join(MOCK_GEMINI_DIR_LOCAL, 'GEMINI.md');
|
||||||
mockFs({
|
mockFs({
|
||||||
[MOCK_GLOBAL_PATH_LOCAL]: { type: 'file', content: 'GlobalContentOnly' }
|
[MOCK_GLOBAL_PATH_LOCAL]: { type: 'file', content: 'GlobalContentOnly' }
|
||||||
|
|
|
@ -226,6 +226,7 @@ export async function loadCliConfig(
|
||||||
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ??
|
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ??
|
||||||
settings.telemetry?.otlpEndpoint,
|
settings.telemetry?.otlpEndpoint,
|
||||||
logPrompts: argv.telemetryLogPrompts ?? settings.telemetry?.logPrompts,
|
logPrompts: argv.telemetryLogPrompts ?? settings.telemetry?.logPrompts,
|
||||||
|
disableDataCollection: settings.telemetry?.disableDataCollection ?? true,
|
||||||
},
|
},
|
||||||
// Git-aware file filtering settings
|
// Git-aware file filtering settings
|
||||||
fileFiltering: {
|
fileFiltering: {
|
||||||
|
|
|
@ -6,15 +6,13 @@
|
||||||
|
|
||||||
/// <reference types="vitest/globals" />
|
/// <reference types="vitest/globals" />
|
||||||
|
|
||||||
const MOCK_HOME_DIR = '/mock/home/user'; // MUST BE FIRST
|
// Mock 'os' first.
|
||||||
|
|
||||||
// Mock 'os' first. Its factory uses MOCK_HOME_DIR.
|
|
||||||
import * as osActual from 'os'; // Import for type info for the mock factory
|
import * as osActual from 'os'; // Import for type info for the mock factory
|
||||||
vi.mock('os', async (importOriginal) => {
|
vi.mock('os', async (importOriginal) => {
|
||||||
const actualOs = await importOriginal<typeof osActual>();
|
const actualOs = await importOriginal<typeof osActual>();
|
||||||
return {
|
return {
|
||||||
...actualOs,
|
...actualOs,
|
||||||
homedir: vi.fn(() => MOCK_HOME_DIR),
|
homedir: vi.fn(() => '/mock/home/user'),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -77,7 +75,7 @@ describe('Settings Loading and Merging', () => {
|
||||||
mockFsMkdirSync = vi.mocked(fs.mkdirSync);
|
mockFsMkdirSync = vi.mocked(fs.mkdirSync);
|
||||||
mockStripJsonComments = vi.mocked(stripJsonComments);
|
mockStripJsonComments = vi.mocked(stripJsonComments);
|
||||||
|
|
||||||
vi.mocked(osActual.homedir).mockReturnValue(MOCK_HOME_DIR);
|
vi.mocked(osActual.homedir).mockReturnValue('/mock/home/user');
|
||||||
(mockStripJsonComments as unknown as Mock).mockImplementation(
|
(mockStripJsonComments as unknown as Mock).mockImplementation(
|
||||||
(jsonString: string) => jsonString,
|
(jsonString: string) => jsonString,
|
||||||
);
|
);
|
||||||
|
|
|
@ -131,6 +131,8 @@ export async function main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
logUserPrompt(config, {
|
logUserPrompt(config, {
|
||||||
|
'event.name': 'user_prompt',
|
||||||
|
'event.timestamp': new Date().toISOString(),
|
||||||
prompt: input,
|
prompt: input,
|
||||||
prompt_length: input.length,
|
prompt_length: input.length,
|
||||||
});
|
});
|
||||||
|
|
|
@ -38,12 +38,17 @@ const MockedGeminiClientClass = vi.hoisted(() =>
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const MockedUserPromptEvent = vi.hoisted(() =>
|
||||||
|
vi.fn().mockImplementation(() => {}),
|
||||||
|
);
|
||||||
|
|
||||||
vi.mock('@gemini-cli/core', async (importOriginal) => {
|
vi.mock('@gemini-cli/core', async (importOriginal) => {
|
||||||
const actualCoreModule = (await importOriginal()) as any;
|
const actualCoreModule = (await importOriginal()) as any;
|
||||||
return {
|
return {
|
||||||
...actualCoreModule,
|
...actualCoreModule,
|
||||||
GitService: vi.fn(),
|
GitService: vi.fn(),
|
||||||
GeminiClient: MockedGeminiClientClass,
|
GeminiClient: MockedGeminiClientClass,
|
||||||
|
UserPromptEvent: MockedUserPromptEvent,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -283,6 +288,7 @@ describe('useGeminiStream', () => {
|
||||||
getProjectRoot: vi.fn(() => '/test/dir'),
|
getProjectRoot: vi.fn(() => '/test/dir'),
|
||||||
getCheckpointingEnabled: vi.fn(() => false),
|
getCheckpointingEnabled: vi.fn(() => false),
|
||||||
getGeminiClient: mockGetGeminiClient,
|
getGeminiClient: mockGetGeminiClient,
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
addHistory: vi.fn(),
|
addHistory: vi.fn(),
|
||||||
} as unknown as Config;
|
} as unknown as Config;
|
||||||
mockOnDebugMessage = vi.fn();
|
mockOnDebugMessage = vi.fn();
|
||||||
|
|
|
@ -23,6 +23,7 @@ import {
|
||||||
EditorType,
|
EditorType,
|
||||||
ThoughtSummary,
|
ThoughtSummary,
|
||||||
isAuthError,
|
isAuthError,
|
||||||
|
UserPromptEvent,
|
||||||
} from '@gemini-cli/core';
|
} from '@gemini-cli/core';
|
||||||
import { type Part, type PartListUnion } from '@google/genai';
|
import { type Part, type PartListUnion } from '@google/genai';
|
||||||
import {
|
import {
|
||||||
|
@ -213,10 +214,10 @@ export const useGeminiStream = (
|
||||||
|
|
||||||
if (typeof query === 'string') {
|
if (typeof query === 'string') {
|
||||||
const trimmedQuery = query.trim();
|
const trimmedQuery = query.trim();
|
||||||
logUserPrompt(config, {
|
logUserPrompt(
|
||||||
prompt: trimmedQuery,
|
config,
|
||||||
prompt_length: trimmedQuery.length,
|
new UserPromptEvent(trimmedQuery.length, trimmedQuery),
|
||||||
});
|
);
|
||||||
onDebugMessage(`User query: '${trimmedQuery}'`);
|
onDebugMessage(`User query: '${trimmedQuery}'`);
|
||||||
await logger?.logMessage(MessageSenderType.USER, trimmedQuery);
|
await logger?.logMessage(MessageSenderType.USER, trimmedQuery);
|
||||||
|
|
||||||
|
|
|
@ -48,6 +48,7 @@ const mockToolRegistry = {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getToolRegistry: vi.fn(() => mockToolRegistry as unknown as ToolRegistry),
|
getToolRegistry: vi.fn(() => mockToolRegistry as unknown as ToolRegistry),
|
||||||
getApprovalMode: vi.fn(() => ApprovalMode.DEFAULT),
|
getApprovalMode: vi.fn(() => ApprovalMode.DEFAULT),
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
};
|
};
|
||||||
|
|
||||||
const mockTool: Tool = {
|
const mockTool: Tool = {
|
||||||
|
|
|
@ -33,8 +33,10 @@ import {
|
||||||
DEFAULT_TELEMETRY_TARGET,
|
DEFAULT_TELEMETRY_TARGET,
|
||||||
DEFAULT_OTLP_ENDPOINT,
|
DEFAULT_OTLP_ENDPOINT,
|
||||||
TelemetryTarget,
|
TelemetryTarget,
|
||||||
|
StartSessionEvent,
|
||||||
} from '../telemetry/index.js';
|
} from '../telemetry/index.js';
|
||||||
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from './models.js';
|
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from './models.js';
|
||||||
|
import { ClearcutLogger } from '../telemetry/clearcut-logger/clearcut-logger.js';
|
||||||
|
|
||||||
export enum ApprovalMode {
|
export enum ApprovalMode {
|
||||||
DEFAULT = 'default',
|
DEFAULT = 'default',
|
||||||
|
@ -55,6 +57,7 @@ export interface TelemetrySettings {
|
||||||
target?: TelemetryTarget;
|
target?: TelemetryTarget;
|
||||||
otlpEndpoint?: string;
|
otlpEndpoint?: string;
|
||||||
logPrompts?: boolean;
|
logPrompts?: boolean;
|
||||||
|
disableDataCollection?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class MCPServerConfig {
|
export class MCPServerConfig {
|
||||||
|
@ -114,6 +117,7 @@ export interface ConfigParameters {
|
||||||
fileDiscoveryService?: FileDiscoveryService;
|
fileDiscoveryService?: FileDiscoveryService;
|
||||||
bugCommand?: BugCommandSettings;
|
bugCommand?: BugCommandSettings;
|
||||||
model: string;
|
model: string;
|
||||||
|
disableDataCollection?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class Config {
|
export class Config {
|
||||||
|
@ -150,6 +154,7 @@ export class Config {
|
||||||
private readonly cwd: string;
|
private readonly cwd: string;
|
||||||
private readonly bugCommand: BugCommandSettings | undefined;
|
private readonly bugCommand: BugCommandSettings | undefined;
|
||||||
private readonly model: string;
|
private readonly model: string;
|
||||||
|
private readonly disableDataCollection: boolean;
|
||||||
|
|
||||||
constructor(params: ConfigParameters) {
|
constructor(params: ConfigParameters) {
|
||||||
this.sessionId = params.sessionId;
|
this.sessionId = params.sessionId;
|
||||||
|
@ -189,6 +194,8 @@ export class Config {
|
||||||
this.fileDiscoveryService = params.fileDiscoveryService ?? null;
|
this.fileDiscoveryService = params.fileDiscoveryService ?? null;
|
||||||
this.bugCommand = params.bugCommand;
|
this.bugCommand = params.bugCommand;
|
||||||
this.model = params.model;
|
this.model = params.model;
|
||||||
|
this.disableDataCollection =
|
||||||
|
params.telemetry?.disableDataCollection ?? true;
|
||||||
|
|
||||||
if (params.contextFileName) {
|
if (params.contextFileName) {
|
||||||
setGeminiMdFilename(params.contextFileName);
|
setGeminiMdFilename(params.contextFileName);
|
||||||
|
@ -197,6 +204,12 @@ export class Config {
|
||||||
if (this.telemetrySettings.enabled) {
|
if (this.telemetrySettings.enabled) {
|
||||||
initializeTelemetry(this);
|
initializeTelemetry(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!this.disableDataCollection) {
|
||||||
|
ClearcutLogger.getInstance(this)?.enqueueLogEvent(
|
||||||
|
new StartSessionEvent(this),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async refreshAuth(authMethod: AuthType) {
|
async refreshAuth(authMethod: AuthType) {
|
||||||
|
@ -370,6 +383,10 @@ export class Config {
|
||||||
return this.fileDiscoveryService;
|
return this.fileDiscoveryService;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getDisableDataCollection(): boolean {
|
||||||
|
return this.disableDataCollection;
|
||||||
|
}
|
||||||
|
|
||||||
async getGitService(): Promise<GitService> {
|
async getGitService(): Promise<GitService> {
|
||||||
if (!this.gitService) {
|
if (!this.gitService) {
|
||||||
this.gitService = new GitService(this.targetDir);
|
this.gitService = new GitService(this.targetDir);
|
||||||
|
|
|
@ -77,6 +77,7 @@ describe('CoreToolScheduler', () => {
|
||||||
|
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
} as Config;
|
} as Config;
|
||||||
|
|
||||||
const scheduler = new CoreToolScheduler({
|
const scheduler = new CoreToolScheduler({
|
||||||
|
|
|
@ -16,6 +16,7 @@ import {
|
||||||
EditorType,
|
EditorType,
|
||||||
Config,
|
Config,
|
||||||
logToolCall,
|
logToolCall,
|
||||||
|
ToolCallEvent,
|
||||||
} from '../index.js';
|
} from '../index.js';
|
||||||
import { Part, PartListUnion } from '@google/genai';
|
import { Part, PartListUnion } from '@google/genai';
|
||||||
import { getResponseTextFromParts } from '../utils/generateContentResponseUtilities.js';
|
import { getResponseTextFromParts } from '../utils/generateContentResponseUtilities.js';
|
||||||
|
@ -652,20 +653,7 @@ export class CoreToolScheduler {
|
||||||
this.toolCalls = [];
|
this.toolCalls = [];
|
||||||
|
|
||||||
for (const call of completedCalls) {
|
for (const call of completedCalls) {
|
||||||
logToolCall(
|
logToolCall(this.config, new ToolCallEvent(call));
|
||||||
this.config,
|
|
||||||
{
|
|
||||||
function_name: call.request.name,
|
|
||||||
function_args: call.request.args,
|
|
||||||
duration_ms: call.durationMs ?? 0,
|
|
||||||
success: call.status === 'success',
|
|
||||||
error:
|
|
||||||
call.status === 'error'
|
|
||||||
? call.response.error?.message
|
|
||||||
: undefined,
|
|
||||||
},
|
|
||||||
call.outcome,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.onAllToolCallsComplete) {
|
if (this.onAllToolCallsComplete) {
|
||||||
|
|
|
@ -27,6 +27,7 @@ const mockModelsModule = {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
getTelemetryLogPromptsEnabled: () => true,
|
getTelemetryLogPromptsEnabled: () => true,
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
} as unknown as Config;
|
} as unknown as Config;
|
||||||
|
|
||||||
describe('GeminiChat', () => {
|
describe('GeminiChat', () => {
|
||||||
|
|
|
@ -24,12 +24,16 @@ import {
|
||||||
logApiRequest,
|
logApiRequest,
|
||||||
logApiResponse,
|
logApiResponse,
|
||||||
logApiError,
|
logApiError,
|
||||||
getFinalUsageMetadata,
|
|
||||||
} from '../telemetry/loggers.js';
|
} from '../telemetry/loggers.js';
|
||||||
import {
|
import {
|
||||||
getStructuredResponse,
|
getStructuredResponse,
|
||||||
getStructuredResponseFromParts,
|
getStructuredResponseFromParts,
|
||||||
} from '../utils/generateContentResponseUtilities.js';
|
} from '../utils/generateContentResponseUtilities.js';
|
||||||
|
import {
|
||||||
|
ApiErrorEvent,
|
||||||
|
ApiRequestEvent,
|
||||||
|
ApiResponseEvent,
|
||||||
|
} from '../telemetry/types.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns true if the response is valid, false otherwise.
|
* Returns true if the response is valid, false otherwise.
|
||||||
|
@ -152,14 +156,8 @@ export class GeminiChat {
|
||||||
contents: Content[],
|
contents: Content[],
|
||||||
model: string,
|
model: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const shouldLogUserPrompts = (config: Config): boolean =>
|
|
||||||
config.getTelemetryLogPromptsEnabled() ?? false;
|
|
||||||
|
|
||||||
const requestText = this._getRequestTextFromContents(contents);
|
const requestText = this._getRequestTextFromContents(contents);
|
||||||
logApiRequest(this.config, {
|
logApiRequest(this.config, new ApiRequestEvent(model, requestText));
|
||||||
model,
|
|
||||||
request_text: shouldLogUserPrompts(this.config) ? requestText : undefined,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async _logApiResponse(
|
private async _logApiResponse(
|
||||||
|
@ -167,31 +165,20 @@ export class GeminiChat {
|
||||||
usageMetadata?: GenerateContentResponseUsageMetadata,
|
usageMetadata?: GenerateContentResponseUsageMetadata,
|
||||||
responseText?: string,
|
responseText?: string,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
logApiResponse(this.config, {
|
logApiResponse(
|
||||||
model: this.model,
|
this.config,
|
||||||
duration_ms: durationMs,
|
new ApiResponseEvent(this.model, durationMs, usageMetadata, responseText),
|
||||||
status_code: 200, // Assuming 200 for success
|
);
|
||||||
input_token_count: usageMetadata?.promptTokenCount ?? 0,
|
|
||||||
output_token_count: usageMetadata?.candidatesTokenCount ?? 0,
|
|
||||||
cached_content_token_count: usageMetadata?.cachedContentTokenCount ?? 0,
|
|
||||||
thoughts_token_count: usageMetadata?.thoughtsTokenCount ?? 0,
|
|
||||||
tool_token_count: usageMetadata?.toolUsePromptTokenCount ?? 0,
|
|
||||||
response_text: responseText,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private _logApiError(durationMs: number, error: unknown): void {
|
private _logApiError(durationMs: number, error: unknown): void {
|
||||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||||
const errorType = error instanceof Error ? error.name : 'unknown';
|
const errorType = error instanceof Error ? error.name : 'unknown';
|
||||||
const statusCode = 'unknown';
|
|
||||||
|
|
||||||
logApiError(this.config, {
|
logApiError(
|
||||||
model: this.model,
|
this.config,
|
||||||
error: errorMessage,
|
new ApiErrorEvent(this.model, errorMessage, durationMs, errorType),
|
||||||
status_code: statusCode,
|
);
|
||||||
error_type: errorType,
|
|
||||||
duration_ms: durationMs,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -402,6 +389,17 @@ export class GeminiChat {
|
||||||
this.history = history;
|
this.history = history;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getFinalUsageMetadata(
|
||||||
|
chunks: GenerateContentResponse[],
|
||||||
|
): GenerateContentResponseUsageMetadata | undefined {
|
||||||
|
const lastChunkWithMetadata = chunks
|
||||||
|
.slice()
|
||||||
|
.reverse()
|
||||||
|
.find((chunk) => chunk.usageMetadata);
|
||||||
|
|
||||||
|
return lastChunkWithMetadata?.usageMetadata;
|
||||||
|
}
|
||||||
|
|
||||||
private async *processStreamResponse(
|
private async *processStreamResponse(
|
||||||
streamResponse: AsyncGenerator<GenerateContentResponse>,
|
streamResponse: AsyncGenerator<GenerateContentResponse>,
|
||||||
inputContent: Content,
|
inputContent: Content,
|
||||||
|
@ -444,7 +442,7 @@ export class GeminiChat {
|
||||||
const fullText = getStructuredResponseFromParts(allParts);
|
const fullText = getStructuredResponseFromParts(allParts);
|
||||||
await this._logApiResponse(
|
await this._logApiResponse(
|
||||||
durationMs,
|
durationMs,
|
||||||
getFinalUsageMetadata(chunks),
|
this.getFinalUsageMetadata(chunks),
|
||||||
fullText,
|
fullText,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,7 +16,10 @@ import {
|
||||||
} from '../index.js';
|
} from '../index.js';
|
||||||
import { Part, Type } from '@google/genai';
|
import { Part, Type } from '@google/genai';
|
||||||
|
|
||||||
const mockConfig = {} as unknown as Config;
|
const mockConfig = {
|
||||||
|
getSessionId: () => 'test-session-id',
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
|
} as unknown as Config;
|
||||||
|
|
||||||
describe('executeToolCall', () => {
|
describe('executeToolCall', () => {
|
||||||
let mockToolRegistry: ToolRegistry;
|
let mockToolRegistry: ToolRegistry;
|
||||||
|
|
|
@ -33,6 +33,8 @@ export async function executeToolCall(
|
||||||
);
|
);
|
||||||
const durationMs = Date.now() - startTime;
|
const durationMs = Date.now() - startTime;
|
||||||
logToolCall(config, {
|
logToolCall(config, {
|
||||||
|
'event.name': 'tool_call',
|
||||||
|
'event.timestamp': new Date().toISOString(),
|
||||||
function_name: toolCallRequest.name,
|
function_name: toolCallRequest.name,
|
||||||
function_args: toolCallRequest.args,
|
function_args: toolCallRequest.args,
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
|
@ -67,6 +69,8 @@ export async function executeToolCall(
|
||||||
|
|
||||||
const durationMs = Date.now() - startTime;
|
const durationMs = Date.now() - startTime;
|
||||||
logToolCall(config, {
|
logToolCall(config, {
|
||||||
|
'event.name': 'tool_call',
|
||||||
|
'event.timestamp': new Date().toISOString(),
|
||||||
function_name: toolCallRequest.name,
|
function_name: toolCallRequest.name,
|
||||||
function_args: toolCallRequest.args,
|
function_args: toolCallRequest.args,
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
|
@ -89,6 +93,8 @@ export async function executeToolCall(
|
||||||
const error = e instanceof Error ? e : new Error(String(e));
|
const error = e instanceof Error ? e : new Error(String(e));
|
||||||
const durationMs = Date.now() - startTime;
|
const durationMs = Date.now() - startTime;
|
||||||
logToolCall(config, {
|
logToolCall(config, {
|
||||||
|
'event.name': 'tool_call',
|
||||||
|
'event.timestamp': new Date().toISOString(),
|
||||||
function_name: toolCallRequest.name,
|
function_name: toolCallRequest.name,
|
||||||
function_args: toolCallRequest.args,
|
function_args: toolCallRequest.args,
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
|
|
|
@ -0,0 +1,338 @@
|
||||||
|
/**
|
||||||
|
* @license
|
||||||
|
* Copyright 2025 Google LLC
|
||||||
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Buffer } from 'buffer';
|
||||||
|
import * as https from 'https';
|
||||||
|
import {
|
||||||
|
StartSessionEvent,
|
||||||
|
EndSessionEvent,
|
||||||
|
UserPromptEvent,
|
||||||
|
ToolCallEvent,
|
||||||
|
ApiRequestEvent,
|
||||||
|
ApiResponseEvent,
|
||||||
|
ApiErrorEvent,
|
||||||
|
} from '../types.js';
|
||||||
|
import { EventMetadataKey } from './event-metadata-key.js';
|
||||||
|
import { Config } from '../../config/config.js';
|
||||||
|
import { getPersistentUserId } from '../../utils/user_id.js';
|
||||||
|
|
||||||
|
const start_session_event_name = 'start_session';
|
||||||
|
const new_prompt_event_name = 'new_prompt';
|
||||||
|
const tool_call_event_name = 'tool_call';
|
||||||
|
const api_request_event_name = 'api_request';
|
||||||
|
const api_response_event_name = 'api_response';
|
||||||
|
const api_error_event_name = 'api_error';
|
||||||
|
const end_session_event_name = 'end_session';
|
||||||
|
|
||||||
|
export interface LogResponse {
|
||||||
|
nextRequestWaitMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Singleton class for batch posting log events to Clearcut. When a new event comes in, the elapsed time
|
||||||
|
// is checked and events are flushed to Clearcut if at least a minute has passed since the last flush.
|
||||||
|
export class ClearcutLogger {
|
||||||
|
private static instance: ClearcutLogger;
|
||||||
|
private config?: Config;
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format.
|
||||||
|
private readonly events: any = [];
|
||||||
|
private last_flush_time: number = Date.now();
|
||||||
|
private flush_interval_ms: number = 1000 * 60; // Wait at least a minute before flushing events.
|
||||||
|
|
||||||
|
private constructor(config?: Config) {
|
||||||
|
this.config = config;
|
||||||
|
}
|
||||||
|
|
||||||
|
static getInstance(config?: Config): ClearcutLogger | undefined {
|
||||||
|
if (config === undefined || config?.getDisableDataCollection())
|
||||||
|
return undefined;
|
||||||
|
if (!ClearcutLogger.instance) {
|
||||||
|
ClearcutLogger.instance = new ClearcutLogger(config);
|
||||||
|
}
|
||||||
|
return ClearcutLogger.instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Clearcut expects this format.
|
||||||
|
enqueueLogEvent(event: any): void {
|
||||||
|
this.events.push([
|
||||||
|
{
|
||||||
|
event_time_ms: Date.now(),
|
||||||
|
source_extension_json: JSON.stringify(event),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
createLogEvent(name: string, data: Map<EventMetadataKey, string>): object {
|
||||||
|
return {
|
||||||
|
Application: 'GEMINI_CLI',
|
||||||
|
event_name: name,
|
||||||
|
client_install_id: getPersistentUserId(),
|
||||||
|
event_metadata: [data] as object[],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
flushIfNeeded(): void {
|
||||||
|
if (Date.now() - this.last_flush_time < this.flush_interval_ms) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.flushToClearcut();
|
||||||
|
this.last_flush_time = Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
flushToClearcut(): Promise<LogResponse> {
|
||||||
|
return new Promise<Buffer>((resolve, reject) => {
|
||||||
|
const request = [
|
||||||
|
{
|
||||||
|
log_source_name: 'CONCORD',
|
||||||
|
request_time_ms: Date.now(),
|
||||||
|
log_event: this.events,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const body = JSON.stringify(request);
|
||||||
|
const options = {
|
||||||
|
hostname: 'play.googleapis.com',
|
||||||
|
path: '/log',
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Length': Buffer.byteLength(body) },
|
||||||
|
};
|
||||||
|
const bufs: Buffer[] = [];
|
||||||
|
const req = https.request(options, (res) => {
|
||||||
|
res.on('data', (buf) => bufs.push(buf));
|
||||||
|
res.on('end', () => {
|
||||||
|
resolve(Buffer.concat(bufs));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
req.on('error', (e) => {
|
||||||
|
reject(e);
|
||||||
|
});
|
||||||
|
req.end(body);
|
||||||
|
}).then((buf: Buffer) => {
|
||||||
|
try {
|
||||||
|
this.events.length = 0;
|
||||||
|
return this.decodeLogResponse(buf) || {};
|
||||||
|
} catch (error: unknown) {
|
||||||
|
console.error('Error flushing log events:', error);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Visible for testing. Decodes protobuf-encoded response from Clearcut server.
|
||||||
|
decodeLogResponse(buf: Buffer): LogResponse | undefined {
|
||||||
|
// TODO(obrienowen): return specific errors to facilitate debugging.
|
||||||
|
if (buf.length < 1) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first byte of the buffer is `field<<3 | type`. We're looking for field
|
||||||
|
// 1, with type varint, represented by type=0. If the first byte isn't 8, that
|
||||||
|
// means field 1 is missing or the message is corrupted. Either way, we return
|
||||||
|
// undefined.
|
||||||
|
if (buf.readUInt8(0) !== 8) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
let ms = BigInt(0);
|
||||||
|
let cont = true;
|
||||||
|
|
||||||
|
// In each byte, the most significant bit is the continuation bit. If it's
|
||||||
|
// set, we keep going. The lowest 7 bits, are data bits. They are concatenated
|
||||||
|
// in reverse order to form the final number.
|
||||||
|
for (let i = 1; cont && i < buf.length; i++) {
|
||||||
|
const byte = buf.readUInt8(i);
|
||||||
|
ms |= BigInt(byte & 0x7f) << BigInt(7 * (i - 1));
|
||||||
|
cont = (byte & 0x80) !== 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cont) {
|
||||||
|
// We have fallen off the buffer without seeing a terminating byte. The
|
||||||
|
// message is corrupted.
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
nextRequestWaitMs: Number(ms),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
logStartSessionEvent(event: StartSessionEvent): void {
|
||||||
|
const data: Map<EventMetadataKey, string> = new Map();
|
||||||
|
|
||||||
|
data.set(EventMetadataKey.GEMINI_CLI_START_SESSION_MODEL, event.model);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_EMBEDDING_MODEL,
|
||||||
|
event.embedding_model,
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_SANDBOX,
|
||||||
|
event.sandbox_enabled.toString(),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_CORE_TOOLS,
|
||||||
|
event.core_tools_enabled,
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_APPROVAL_MODE,
|
||||||
|
event.approval_mode,
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_API_KEY_ENABLED,
|
||||||
|
event.api_key_enabled.toString(),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED,
|
||||||
|
event.vertex_ai_enabled.toString(),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED,
|
||||||
|
event.debug_enabled.toString(),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_MCP_SERVERS,
|
||||||
|
event.mcp_servers,
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED,
|
||||||
|
event.telemetry_enabled.toString(),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED,
|
||||||
|
event.telemetry_log_user_prompts_enabled.toString(),
|
||||||
|
);
|
||||||
|
|
||||||
|
this.enqueueLogEvent(this.createLogEvent(start_session_event_name, data));
|
||||||
|
this.flushIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
logNewPromptEvent(event: UserPromptEvent): void {
|
||||||
|
const data: Map<EventMetadataKey, string> = new Map();
|
||||||
|
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_USER_PROMPT_LENGTH,
|
||||||
|
JSON.stringify(event.prompt_length),
|
||||||
|
);
|
||||||
|
|
||||||
|
this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data));
|
||||||
|
this.flushIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
logToolCallEvent(event: ToolCallEvent): void {
|
||||||
|
const data: Map<EventMetadataKey, string> = new Map();
|
||||||
|
|
||||||
|
data.set(EventMetadataKey.GEMINI_CLI_TOOL_CALL_NAME, event.function_name);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_TOOL_CALL_DECISION,
|
||||||
|
JSON.stringify(event.decision),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_TOOL_CALL_SUCCESS,
|
||||||
|
JSON.stringify(event.success),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_TOOL_CALL_DURATION_MS,
|
||||||
|
JSON.stringify(event.duration_ms),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_TOOL_ERROR_MESSAGE,
|
||||||
|
JSON.stringify(event.error),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_TOOL_CALL_ERROR_TYPE,
|
||||||
|
JSON.stringify(event.error_type),
|
||||||
|
);
|
||||||
|
|
||||||
|
this.enqueueLogEvent(this.createLogEvent(tool_call_event_name, data));
|
||||||
|
this.flushIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
logApiRequestEvent(event: ApiRequestEvent): void {
|
||||||
|
const data: Map<EventMetadataKey, string> = new Map();
|
||||||
|
|
||||||
|
data.set(EventMetadataKey.GEMINI_CLI_API_REQUEST_MODEL, event.model);
|
||||||
|
|
||||||
|
this.enqueueLogEvent(this.createLogEvent(api_request_event_name, data));
|
||||||
|
this.flushIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
logApiResponseEvent(event: ApiResponseEvent): void {
|
||||||
|
const data: Map<EventMetadataKey, string> = new Map();
|
||||||
|
|
||||||
|
data.set(EventMetadataKey.GEMINI_CLI_API_RESPONSE_MODEL, event.model);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_RESPONSE_STATUS_CODE,
|
||||||
|
JSON.stringify(event.status_code),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_RESPONSE_DURATION_MS,
|
||||||
|
JSON.stringify(event.duration_ms),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_ERROR_MESSAGE,
|
||||||
|
JSON.stringify(event.error),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT,
|
||||||
|
JSON.stringify(event.input_token_count),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT,
|
||||||
|
JSON.stringify(event.output_token_count),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT,
|
||||||
|
JSON.stringify(event.cached_content_token_count),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT,
|
||||||
|
JSON.stringify(event.thoughts_token_count),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT,
|
||||||
|
JSON.stringify(event.tool_token_count),
|
||||||
|
);
|
||||||
|
|
||||||
|
this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data));
|
||||||
|
this.flushIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
logApiErrorEvent(event: ApiErrorEvent): void {
|
||||||
|
const data: Map<EventMetadataKey, string> = new Map();
|
||||||
|
|
||||||
|
data.set(EventMetadataKey.GEMINI_CLI_API_ERROR_MODEL, event.model);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_ERROR_TYPE,
|
||||||
|
JSON.stringify(event.error_type),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_ERROR_STATUS_CODE,
|
||||||
|
JSON.stringify(event.status_code),
|
||||||
|
);
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_API_ERROR_DURATION_MS,
|
||||||
|
JSON.stringify(event.duration_ms),
|
||||||
|
);
|
||||||
|
|
||||||
|
this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data));
|
||||||
|
this.flushIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
logEndSessionEvent(event: EndSessionEvent): void {
|
||||||
|
const data: Map<EventMetadataKey, string> = new Map();
|
||||||
|
|
||||||
|
data.set(
|
||||||
|
EventMetadataKey.GEMINI_CLI_END_SESSION_ID,
|
||||||
|
event?.session_id?.toString() ?? '',
|
||||||
|
);
|
||||||
|
|
||||||
|
this.enqueueLogEvent(this.createLogEvent(end_session_event_name, data));
|
||||||
|
// Flush immediately on session end.
|
||||||
|
this.flushToClearcut();
|
||||||
|
}
|
||||||
|
|
||||||
|
shutdown() {
|
||||||
|
const event = new EndSessionEvent(this.config);
|
||||||
|
this.logEndSessionEvent(event);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,153 @@
|
||||||
|
/**
|
||||||
|
* @license
|
||||||
|
* Copyright 2025 Google LLC
|
||||||
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Defines valid event metadata keys for Clearcut logging.
|
||||||
|
export enum EventMetadataKey {
|
||||||
|
GEMINI_CLI_KEY_UNKNOWN = 0,
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// Start Session Event Keys
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
// Logs the model id used in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_MODEL = 1,
|
||||||
|
|
||||||
|
// Logs the embedding model id used in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_EMBEDDING_MODEL = 2,
|
||||||
|
|
||||||
|
// Logs the sandbox that was used in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_SANDBOX = 3,
|
||||||
|
|
||||||
|
// Logs the core tools that were enabled in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_CORE_TOOLS = 4,
|
||||||
|
|
||||||
|
// Logs the approval mode that was used in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_APPROVAL_MODE = 5,
|
||||||
|
|
||||||
|
// Logs whether an API key was used in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_API_KEY_ENABLED = 6,
|
||||||
|
|
||||||
|
// Logs whether the Vertex API was used in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_VERTEX_API_ENABLED = 7,
|
||||||
|
|
||||||
|
// Logs whether debug mode was enabled in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_DEBUG_MODE_ENABLED = 8,
|
||||||
|
|
||||||
|
// Logs the MCP servers that were enabled in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_MCP_SERVERS = 9,
|
||||||
|
|
||||||
|
// Logs whether user-collected telemetry was enabled in the session.
|
||||||
|
GEMINI_CLI_START_SESSION_TELEMETRY_ENABLED = 10,
|
||||||
|
|
||||||
|
// Logs whether prompt collection was enabled for user-collected telemetry.
|
||||||
|
GEMINI_CLI_START_SESSION_TELEMETRY_LOG_USER_PROMPTS_ENABLED = 11,
|
||||||
|
|
||||||
|
// Logs whether the session was configured to respect gitignore files.
|
||||||
|
GEMINI_CLI_START_SESSION_RESPECT_GITIGNORE = 12,
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// User Prompt Event Keys
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
// Logs the length of the prompt.
|
||||||
|
GEMINI_CLI_USER_PROMPT_LENGTH = 13,
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// Tool Call Event Keys
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
// Logs the function name.
|
||||||
|
GEMINI_CLI_TOOL_CALL_NAME = 14,
|
||||||
|
|
||||||
|
// Logs the user's decision about how to handle the tool call.
|
||||||
|
GEMINI_CLI_TOOL_CALL_DECISION = 15,
|
||||||
|
|
||||||
|
// Logs whether the tool call succeeded.
|
||||||
|
GEMINI_CLI_TOOL_CALL_SUCCESS = 16,
|
||||||
|
|
||||||
|
// Logs the tool call duration in milliseconds.
|
||||||
|
GEMINI_CLI_TOOL_CALL_DURATION_MS = 17,
|
||||||
|
|
||||||
|
// Logs the tool call error message, if any.
|
||||||
|
GEMINI_CLI_TOOL_ERROR_MESSAGE = 18,
|
||||||
|
|
||||||
|
// Logs the tool call error type, if any.
|
||||||
|
GEMINI_CLI_TOOL_CALL_ERROR_TYPE = 19,
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// GenAI API Request Event Keys
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
// Logs the model id of the request.
|
||||||
|
GEMINI_CLI_API_REQUEST_MODEL = 20,
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// GenAI API Response Event Keys
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
// Logs the model id of the API call.
|
||||||
|
GEMINI_CLI_API_RESPONSE_MODEL = 21,
|
||||||
|
|
||||||
|
// Logs the status code of the response.
|
||||||
|
GEMINI_CLI_API_RESPONSE_STATUS_CODE = 22,
|
||||||
|
|
||||||
|
// Logs the duration of the API call in milliseconds.
|
||||||
|
GEMINI_CLI_API_RESPONSE_DURATION_MS = 23,
|
||||||
|
|
||||||
|
// Logs the error message of the API call, if any.
|
||||||
|
GEMINI_CLI_API_ERROR_MESSAGE = 24,
|
||||||
|
|
||||||
|
// Logs the input token count of the API call.
|
||||||
|
GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT = 25,
|
||||||
|
|
||||||
|
// Logs the output token count of the API call.
|
||||||
|
GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT = 26,
|
||||||
|
|
||||||
|
// Logs the cached token count of the API call.
|
||||||
|
GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT = 27,
|
||||||
|
|
||||||
|
// Logs the thinking token count of the API call.
|
||||||
|
GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT = 28,
|
||||||
|
|
||||||
|
// Logs the tool use token count of the API call.
|
||||||
|
GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT = 29,
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// GenAI API Error Event Keys
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
// Logs the model id of the API call.
|
||||||
|
GEMINI_CLI_API_ERROR_MODEL = 30,
|
||||||
|
|
||||||
|
// Logs the error type.
|
||||||
|
GEMINI_CLI_API_ERROR_TYPE = 31,
|
||||||
|
|
||||||
|
// Logs the status code of the error response.
|
||||||
|
GEMINI_CLI_API_ERROR_STATUS_CODE = 32,
|
||||||
|
|
||||||
|
// Logs the duration of the API call in milliseconds.
|
||||||
|
GEMINI_CLI_API_ERROR_DURATION_MS = 33,
|
||||||
|
|
||||||
|
// ==========================================================================
|
||||||
|
// End Session Event Keys
|
||||||
|
// ===========================================================================
|
||||||
|
|
||||||
|
// Logs the end of a session.
|
||||||
|
GEMINI_CLI_END_SESSION_ID = 34,
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getEventMetadataKey(
|
||||||
|
keyName: string,
|
||||||
|
): EventMetadataKey | undefined {
|
||||||
|
// Access the enum member by its string name
|
||||||
|
const key = EventMetadataKey[keyName as keyof typeof EventMetadataKey];
|
||||||
|
|
||||||
|
// Check if the result is a valid enum member (not undefined and is a number)
|
||||||
|
if (typeof key === 'number') {
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
|
@ -25,15 +25,15 @@ export {
|
||||||
logApiRequest,
|
logApiRequest,
|
||||||
logApiError,
|
logApiError,
|
||||||
logApiResponse,
|
logApiResponse,
|
||||||
getFinalUsageMetadata,
|
|
||||||
} from './loggers.js';
|
} from './loggers.js';
|
||||||
export {
|
export {
|
||||||
|
StartSessionEvent,
|
||||||
|
EndSessionEvent,
|
||||||
UserPromptEvent,
|
UserPromptEvent,
|
||||||
ToolCallEvent,
|
ToolCallEvent,
|
||||||
ApiRequestEvent,
|
ApiRequestEvent,
|
||||||
ApiErrorEvent,
|
ApiErrorEvent,
|
||||||
ApiResponseEvent,
|
ApiResponseEvent,
|
||||||
CliConfigEvent,
|
|
||||||
TelemetryEvent,
|
TelemetryEvent,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
|
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
|
||||||
|
|
|
@ -4,14 +4,24 @@
|
||||||
* SPDX-License-Identifier: Apache-2.0
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ToolConfirmationOutcome } from '../tools/tools.js';
|
import {
|
||||||
import { AuthType } from '../core/contentGenerator.js';
|
AuthType,
|
||||||
|
CompletedToolCall,
|
||||||
|
ContentGeneratorConfig,
|
||||||
|
EditTool,
|
||||||
|
ErroredToolCall,
|
||||||
|
GeminiClient,
|
||||||
|
ToolConfirmationOutcome,
|
||||||
|
ToolRegistry,
|
||||||
|
} from '../index.js';
|
||||||
import { logs } from '@opentelemetry/api-logs';
|
import { logs } from '@opentelemetry/api-logs';
|
||||||
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import {
|
import {
|
||||||
EVENT_API_REQUEST,
|
EVENT_API_REQUEST,
|
||||||
EVENT_API_RESPONSE,
|
EVENT_API_RESPONSE,
|
||||||
|
EVENT_CLI_CONFIG,
|
||||||
|
EVENT_TOOL_CALL,
|
||||||
EVENT_USER_PROMPT,
|
EVENT_USER_PROMPT,
|
||||||
} from './constants.js';
|
} from './constants.js';
|
||||||
import {
|
import {
|
||||||
|
@ -20,13 +30,19 @@ import {
|
||||||
logCliConfiguration,
|
logCliConfiguration,
|
||||||
logUserPrompt,
|
logUserPrompt,
|
||||||
logToolCall,
|
logToolCall,
|
||||||
ToolCallDecision,
|
|
||||||
getFinalUsageMetadata,
|
|
||||||
} from './loggers.js';
|
} from './loggers.js';
|
||||||
|
import {
|
||||||
|
ApiRequestEvent,
|
||||||
|
ApiResponseEvent,
|
||||||
|
StartSessionEvent,
|
||||||
|
ToolCallDecision,
|
||||||
|
ToolCallEvent,
|
||||||
|
UserPromptEvent,
|
||||||
|
} from './types.js';
|
||||||
import * as metrics from './metrics.js';
|
import * as metrics from './metrics.js';
|
||||||
import * as sdk from './sdk.js';
|
import * as sdk from './sdk.js';
|
||||||
import { vi, describe, beforeEach, it, expect } from 'vitest';
|
import { vi, describe, beforeEach, it, expect } from 'vitest';
|
||||||
import { GenerateContentResponse } from '@google/genai';
|
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
||||||
|
|
||||||
describe('loggers', () => {
|
describe('loggers', () => {
|
||||||
const mockLogger = {
|
const mockLogger = {
|
||||||
|
@ -54,8 +70,11 @@ describe('loggers', () => {
|
||||||
apiKey: 'test-api-key',
|
apiKey: 'test-api-key',
|
||||||
authType: AuthType.USE_VERTEX_AI,
|
authType: AuthType.USE_VERTEX_AI,
|
||||||
}),
|
}),
|
||||||
|
getTelemetryEnabled: () => true,
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
getTelemetryLogPromptsEnabled: () => true,
|
getTelemetryLogPromptsEnabled: () => true,
|
||||||
getFileFilteringRespectGitIgnore: () => true,
|
getFileFilteringRespectGitIgnore: () => true,
|
||||||
|
getFileFilteringAllowBuildArtifacts: () => false,
|
||||||
getDebugMode: () => true,
|
getDebugMode: () => true,
|
||||||
getMcpServers: () => ({
|
getMcpServers: () => ({
|
||||||
'test-server': {
|
'test-server': {
|
||||||
|
@ -63,15 +82,18 @@ describe('loggers', () => {
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
getQuestion: () => 'test-question',
|
getQuestion: () => 'test-question',
|
||||||
|
getTargetDir: () => 'target-dir',
|
||||||
|
getProxy: () => 'http://test.proxy.com:8080',
|
||||||
} as unknown as Config;
|
} as unknown as Config;
|
||||||
|
|
||||||
logCliConfiguration(mockConfig);
|
const startSessionEvent = new StartSessionEvent(mockConfig);
|
||||||
|
logCliConfiguration(mockConfig, startSessionEvent);
|
||||||
|
|
||||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
body: 'CLI configuration loaded.',
|
body: 'CLI configuration loaded.',
|
||||||
attributes: {
|
attributes: {
|
||||||
'session.id': 'test-session-id',
|
'session.id': 'test-session-id',
|
||||||
'event.name': 'gemini_cli.config',
|
'event.name': EVENT_CLI_CONFIG,
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
model: 'test-model',
|
model: 'test-model',
|
||||||
embedding_model: 'test-embedding-model',
|
embedding_model: 'test-embedding-model',
|
||||||
|
@ -92,14 +114,13 @@ describe('loggers', () => {
|
||||||
describe('logUserPrompt', () => {
|
describe('logUserPrompt', () => {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
|
getTelemetryEnabled: () => true,
|
||||||
getTelemetryLogPromptsEnabled: () => true,
|
getTelemetryLogPromptsEnabled: () => true,
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
} as unknown as Config;
|
} as unknown as Config;
|
||||||
|
|
||||||
it('should log a user prompt', () => {
|
it('should log a user prompt', () => {
|
||||||
const event = {
|
const event = new UserPromptEvent(11, 'test-prompt');
|
||||||
prompt: 'test-prompt',
|
|
||||||
prompt_length: 11,
|
|
||||||
};
|
|
||||||
|
|
||||||
logUserPrompt(mockConfig, event);
|
logUserPrompt(mockConfig, event);
|
||||||
|
|
||||||
|
@ -118,12 +139,12 @@ describe('loggers', () => {
|
||||||
it('should not log prompt if disabled', () => {
|
it('should not log prompt if disabled', () => {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
|
getTelemetryEnabled: () => true,
|
||||||
getTelemetryLogPromptsEnabled: () => false,
|
getTelemetryLogPromptsEnabled: () => false,
|
||||||
|
getTargetDir: () => 'target-dir',
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
} as unknown as Config;
|
} as unknown as Config;
|
||||||
const event = {
|
const event = new UserPromptEvent(11, 'test-prompt');
|
||||||
prompt: 'test-prompt',
|
|
||||||
prompt_length: 11,
|
|
||||||
};
|
|
||||||
|
|
||||||
logUserPrompt(mockConfig, event);
|
logUserPrompt(mockConfig, event);
|
||||||
|
|
||||||
|
@ -142,6 +163,10 @@ describe('loggers', () => {
|
||||||
describe('logApiResponse', () => {
|
describe('logApiResponse', () => {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
|
getTargetDir: () => 'target-dir',
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
|
getTelemetryEnabled: () => true,
|
||||||
|
getTelemetryLogPromptsEnabled: () => true,
|
||||||
} as Config;
|
} as Config;
|
||||||
|
|
||||||
const mockMetrics = {
|
const mockMetrics = {
|
||||||
|
@ -159,17 +184,19 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log an API response with all fields', () => {
|
it('should log an API response with all fields', () => {
|
||||||
const event = {
|
const usageData: GenerateContentResponseUsageMetadata = {
|
||||||
model: 'test-model',
|
promptTokenCount: 17,
|
||||||
status_code: 200,
|
candidatesTokenCount: 50,
|
||||||
duration_ms: 100,
|
cachedContentTokenCount: 10,
|
||||||
input_token_count: 17,
|
thoughtsTokenCount: 5,
|
||||||
output_token_count: 50,
|
toolUsePromptTokenCount: 2,
|
||||||
cached_content_token_count: 10,
|
|
||||||
thoughts_token_count: 5,
|
|
||||||
tool_token_count: 2,
|
|
||||||
response_text: 'test-response',
|
|
||||||
};
|
};
|
||||||
|
const event = new ApiResponseEvent(
|
||||||
|
'test-model',
|
||||||
|
100,
|
||||||
|
usageData,
|
||||||
|
'test-response',
|
||||||
|
);
|
||||||
|
|
||||||
logApiResponse(mockConfig, event);
|
logApiResponse(mockConfig, event);
|
||||||
|
|
||||||
|
@ -209,22 +236,25 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log an API response with an error', () => {
|
it('should log an API response with an error', () => {
|
||||||
const event = {
|
const usageData: GenerateContentResponseUsageMetadata = {
|
||||||
model: 'test-model',
|
promptTokenCount: 17,
|
||||||
duration_ms: 100,
|
candidatesTokenCount: 50,
|
||||||
error: 'test-error',
|
cachedContentTokenCount: 10,
|
||||||
input_token_count: 17,
|
thoughtsTokenCount: 5,
|
||||||
output_token_count: 50,
|
toolUsePromptTokenCount: 2,
|
||||||
cached_content_token_count: 10,
|
|
||||||
thoughts_token_count: 5,
|
|
||||||
tool_token_count: 2,
|
|
||||||
response_text: 'test-response',
|
|
||||||
};
|
};
|
||||||
|
const event = new ApiResponseEvent(
|
||||||
|
'test-model',
|
||||||
|
100,
|
||||||
|
usageData,
|
||||||
|
'test-response',
|
||||||
|
'test-error',
|
||||||
|
);
|
||||||
|
|
||||||
logApiResponse(mockConfig, event);
|
logApiResponse(mockConfig, event);
|
||||||
|
|
||||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
body: 'API response from test-model. Status: N/A. Duration: 100ms.',
|
body: 'API response from test-model. Status: 200. Duration: 100ms.',
|
||||||
attributes: {
|
attributes: {
|
||||||
'session.id': 'test-session-id',
|
'session.id': 'test-session-id',
|
||||||
...event,
|
...event,
|
||||||
|
@ -239,13 +269,14 @@ describe('loggers', () => {
|
||||||
describe('logApiRequest', () => {
|
describe('logApiRequest', () => {
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
|
getTargetDir: () => 'target-dir',
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
|
getTelemetryEnabled: () => true,
|
||||||
|
getTelemetryLogPromptsEnabled: () => true,
|
||||||
} as Config;
|
} as Config;
|
||||||
|
|
||||||
it('should log an API request with request_text', () => {
|
it('should log an API request with request_text', () => {
|
||||||
const event = {
|
const event = new ApiRequestEvent('test-model', 'This is a test request');
|
||||||
model: 'test-model',
|
|
||||||
request_text: 'This is a test request',
|
|
||||||
};
|
|
||||||
|
|
||||||
logApiRequest(mockConfig, event);
|
logApiRequest(mockConfig, event);
|
||||||
|
|
||||||
|
@ -262,9 +293,7 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log an API request without request_text', () => {
|
it('should log an API request without request_text', () => {
|
||||||
const event = {
|
const event = new ApiRequestEvent('test-model');
|
||||||
model: 'test-model',
|
|
||||||
};
|
|
||||||
|
|
||||||
logApiRequest(mockConfig, event);
|
logApiRequest(mockConfig, event);
|
||||||
|
|
||||||
|
@ -281,8 +310,46 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('logToolCall', () => {
|
describe('logToolCall', () => {
|
||||||
|
const cfg1 = {
|
||||||
|
getSessionId: () => 'test-session-id',
|
||||||
|
getTargetDir: () => 'target-dir',
|
||||||
|
getGeminiClient: () => mockGeminiClient,
|
||||||
|
} as Config;
|
||||||
|
const cfg2 = {
|
||||||
|
getSessionId: () => 'test-session-id',
|
||||||
|
getTargetDir: () => 'target-dir',
|
||||||
|
getProxy: () => 'http://test.proxy.com:8080',
|
||||||
|
getContentGeneratorConfig: () =>
|
||||||
|
({ model: 'test-model' }) as ContentGeneratorConfig,
|
||||||
|
getModel: () => 'test-model',
|
||||||
|
getEmbeddingModel: () => 'test-embedding-model',
|
||||||
|
getWorkingDir: () => 'test-working-dir',
|
||||||
|
getSandbox: () => true,
|
||||||
|
getCoreTools: () => ['ls', 'read-file'],
|
||||||
|
getApprovalMode: () => 'default',
|
||||||
|
getTelemetryLogPromptsEnabled: () => true,
|
||||||
|
getFileFilteringRespectGitIgnore: () => true,
|
||||||
|
getFileFilteringAllowBuildArtifacts: () => false,
|
||||||
|
getDebugMode: () => true,
|
||||||
|
getMcpServers: () => ({
|
||||||
|
'test-server': {
|
||||||
|
command: 'test-command',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
getQuestion: () => 'test-question',
|
||||||
|
getToolRegistry: () => new ToolRegistry(cfg1),
|
||||||
|
getFullContext: () => false,
|
||||||
|
getUserMemory: () => 'user-memory',
|
||||||
|
} as unknown as Config;
|
||||||
|
|
||||||
|
const mockGeminiClient = new GeminiClient(cfg2);
|
||||||
const mockConfig = {
|
const mockConfig = {
|
||||||
getSessionId: () => 'test-session-id',
|
getSessionId: () => 'test-session-id',
|
||||||
|
getTargetDir: () => 'target-dir',
|
||||||
|
getGeminiClient: () => mockGeminiClient,
|
||||||
|
getDisableDataCollection: () => false,
|
||||||
|
getTelemetryEnabled: () => true,
|
||||||
|
getTelemetryLogPromptsEnabled: () => true,
|
||||||
} as Config;
|
} as Config;
|
||||||
|
|
||||||
const mockMetrics = {
|
const mockMetrics = {
|
||||||
|
@ -297,23 +364,36 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log a tool call with all fields', () => {
|
it('should log a tool call with all fields', () => {
|
||||||
const event = {
|
const call: CompletedToolCall = {
|
||||||
function_name: 'test-function',
|
status: 'success',
|
||||||
function_args: {
|
request: {
|
||||||
|
name: 'test-function',
|
||||||
|
args: {
|
||||||
arg1: 'value1',
|
arg1: 'value1',
|
||||||
arg2: 2,
|
arg2: 2,
|
||||||
},
|
},
|
||||||
duration_ms: 100,
|
callId: 'test-call-id',
|
||||||
success: true,
|
isClientInitiated: true,
|
||||||
|
},
|
||||||
|
response: {
|
||||||
|
callId: 'test-call-id',
|
||||||
|
responseParts: 'test-response',
|
||||||
|
resultDisplay: undefined,
|
||||||
|
error: undefined,
|
||||||
|
},
|
||||||
|
tool: new EditTool(mockConfig),
|
||||||
|
durationMs: 100,
|
||||||
|
outcome: ToolConfirmationOutcome.ProceedOnce,
|
||||||
};
|
};
|
||||||
|
const event = new ToolCallEvent(call);
|
||||||
|
|
||||||
logToolCall(mockConfig, event, ToolConfirmationOutcome.ProceedOnce);
|
logToolCall(mockConfig, event);
|
||||||
|
|
||||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
body: 'Tool call: test-function. Decision: accept. Success: true. Duration: 100ms.',
|
body: 'Tool call: test-function. Decision: accept. Success: true. Duration: 100ms.',
|
||||||
attributes: {
|
attributes: {
|
||||||
'session.id': 'test-session-id',
|
'session.id': 'test-session-id',
|
||||||
'event.name': 'gemini_cli.tool_call',
|
'event.name': EVENT_TOOL_CALL,
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
function_name: 'test-function',
|
function_name: 'test-function',
|
||||||
function_args: JSON.stringify(
|
function_args: JSON.stringify(
|
||||||
|
@ -339,23 +419,35 @@ describe('loggers', () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
it('should log a tool call with a reject decision', () => {
|
it('should log a tool call with a reject decision', () => {
|
||||||
const event = {
|
const call: ErroredToolCall = {
|
||||||
function_name: 'test-function',
|
status: 'error',
|
||||||
function_args: {
|
request: {
|
||||||
|
name: 'test-function',
|
||||||
|
args: {
|
||||||
arg1: 'value1',
|
arg1: 'value1',
|
||||||
arg2: 2,
|
arg2: 2,
|
||||||
},
|
},
|
||||||
duration_ms: 100,
|
callId: 'test-call-id',
|
||||||
success: false,
|
isClientInitiated: true,
|
||||||
|
},
|
||||||
|
response: {
|
||||||
|
callId: 'test-call-id',
|
||||||
|
responseParts: 'test-response',
|
||||||
|
resultDisplay: undefined,
|
||||||
|
error: undefined,
|
||||||
|
},
|
||||||
|
durationMs: 100,
|
||||||
|
outcome: ToolConfirmationOutcome.Cancel,
|
||||||
};
|
};
|
||||||
|
const event = new ToolCallEvent(call);
|
||||||
|
|
||||||
logToolCall(mockConfig, event, ToolConfirmationOutcome.Cancel);
|
logToolCall(mockConfig, event);
|
||||||
|
|
||||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
body: 'Tool call: test-function. Decision: reject. Success: false. Duration: 100ms.',
|
body: 'Tool call: test-function. Decision: reject. Success: false. Duration: 100ms.',
|
||||||
attributes: {
|
attributes: {
|
||||||
'session.id': 'test-session-id',
|
'session.id': 'test-session-id',
|
||||||
'event.name': 'gemini_cli.tool_call',
|
'event.name': EVENT_TOOL_CALL,
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
function_name: 'test-function',
|
function_name: 'test-function',
|
||||||
function_args: JSON.stringify(
|
function_args: JSON.stringify(
|
||||||
|
@ -382,23 +474,36 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log a tool call with a modify decision', () => {
|
it('should log a tool call with a modify decision', () => {
|
||||||
const event = {
|
const call: CompletedToolCall = {
|
||||||
function_name: 'test-function',
|
status: 'success',
|
||||||
function_args: {
|
request: {
|
||||||
|
name: 'test-function',
|
||||||
|
args: {
|
||||||
arg1: 'value1',
|
arg1: 'value1',
|
||||||
arg2: 2,
|
arg2: 2,
|
||||||
},
|
},
|
||||||
duration_ms: 100,
|
callId: 'test-call-id',
|
||||||
success: true,
|
isClientInitiated: true,
|
||||||
|
},
|
||||||
|
response: {
|
||||||
|
callId: 'test-call-id',
|
||||||
|
responseParts: 'test-response',
|
||||||
|
resultDisplay: undefined,
|
||||||
|
error: undefined,
|
||||||
|
},
|
||||||
|
outcome: ToolConfirmationOutcome.ModifyWithEditor,
|
||||||
|
tool: new EditTool(mockConfig),
|
||||||
|
durationMs: 100,
|
||||||
};
|
};
|
||||||
|
const event = new ToolCallEvent(call);
|
||||||
|
|
||||||
logToolCall(mockConfig, event, ToolConfirmationOutcome.ModifyWithEditor);
|
logToolCall(mockConfig, event);
|
||||||
|
|
||||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
body: 'Tool call: test-function. Decision: modify. Success: true. Duration: 100ms.',
|
body: 'Tool call: test-function. Decision: modify. Success: true. Duration: 100ms.',
|
||||||
attributes: {
|
attributes: {
|
||||||
'session.id': 'test-session-id',
|
'session.id': 'test-session-id',
|
||||||
'event.name': 'gemini_cli.tool_call',
|
'event.name': EVENT_TOOL_CALL,
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
function_name: 'test-function',
|
function_name: 'test-function',
|
||||||
function_args: JSON.stringify(
|
function_args: JSON.stringify(
|
||||||
|
@ -425,15 +530,27 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log a tool call without a decision', () => {
|
it('should log a tool call without a decision', () => {
|
||||||
const event = {
|
const call: CompletedToolCall = {
|
||||||
function_name: 'test-function',
|
status: 'success',
|
||||||
function_args: {
|
request: {
|
||||||
|
name: 'test-function',
|
||||||
|
args: {
|
||||||
arg1: 'value1',
|
arg1: 'value1',
|
||||||
arg2: 2,
|
arg2: 2,
|
||||||
},
|
},
|
||||||
duration_ms: 100,
|
callId: 'test-call-id',
|
||||||
success: true,
|
isClientInitiated: true,
|
||||||
|
},
|
||||||
|
response: {
|
||||||
|
callId: 'test-call-id',
|
||||||
|
responseParts: 'test-response',
|
||||||
|
resultDisplay: undefined,
|
||||||
|
error: undefined,
|
||||||
|
},
|
||||||
|
tool: new EditTool(mockConfig),
|
||||||
|
durationMs: 100,
|
||||||
};
|
};
|
||||||
|
const event = new ToolCallEvent(call);
|
||||||
|
|
||||||
logToolCall(mockConfig, event);
|
logToolCall(mockConfig, event);
|
||||||
|
|
||||||
|
@ -441,7 +558,7 @@ describe('loggers', () => {
|
||||||
body: 'Tool call: test-function. Success: true. Duration: 100ms.',
|
body: 'Tool call: test-function. Success: true. Duration: 100ms.',
|
||||||
attributes: {
|
attributes: {
|
||||||
'session.id': 'test-session-id',
|
'session.id': 'test-session-id',
|
||||||
'event.name': 'gemini_cli.tool_call',
|
'event.name': EVENT_TOOL_CALL,
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
function_name: 'test-function',
|
function_name: 'test-function',
|
||||||
function_args: JSON.stringify(
|
function_args: JSON.stringify(
|
||||||
|
@ -467,17 +584,29 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log a failed tool call with an error', () => {
|
it('should log a failed tool call with an error', () => {
|
||||||
const event = {
|
const call: ErroredToolCall = {
|
||||||
function_name: 'test-function',
|
status: 'error',
|
||||||
function_args: {
|
request: {
|
||||||
|
name: 'test-function',
|
||||||
|
args: {
|
||||||
arg1: 'value1',
|
arg1: 'value1',
|
||||||
arg2: 2,
|
arg2: 2,
|
||||||
},
|
},
|
||||||
duration_ms: 100,
|
callId: 'test-call-id',
|
||||||
success: false,
|
isClientInitiated: true,
|
||||||
error: 'test-error',
|
},
|
||||||
error_type: 'test-error-type',
|
response: {
|
||||||
|
callId: 'test-call-id',
|
||||||
|
responseParts: 'test-response',
|
||||||
|
resultDisplay: undefined,
|
||||||
|
error: {
|
||||||
|
name: 'test-error-type',
|
||||||
|
message: 'test-error',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
durationMs: 100,
|
||||||
};
|
};
|
||||||
|
const event = new ToolCallEvent(call);
|
||||||
|
|
||||||
logToolCall(mockConfig, event);
|
logToolCall(mockConfig, event);
|
||||||
|
|
||||||
|
@ -485,7 +614,7 @@ describe('loggers', () => {
|
||||||
body: 'Tool call: test-function. Success: false. Duration: 100ms.',
|
body: 'Tool call: test-function. Success: false. Duration: 100ms.',
|
||||||
attributes: {
|
attributes: {
|
||||||
'session.id': 'test-session-id',
|
'session.id': 'test-session-id',
|
||||||
'event.name': 'gemini_cli.tool_call',
|
'event.name': EVENT_TOOL_CALL,
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
function_name: 'test-function',
|
function_name: 'test-function',
|
||||||
function_args: JSON.stringify(
|
function_args: JSON.stringify(
|
||||||
|
@ -515,75 +644,3 @@ describe('loggers', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('getFinalUsageMetadata', () => {
|
|
||||||
const createMockResponse = (
|
|
||||||
usageMetadata?: GenerateContentResponse['usageMetadata'],
|
|
||||||
): GenerateContentResponse =>
|
|
||||||
({
|
|
||||||
text: () => '',
|
|
||||||
data: () => ({}) as Record<string, unknown>,
|
|
||||||
functionCalls: () => [],
|
|
||||||
executableCode: () => [],
|
|
||||||
codeExecutionResult: () => [],
|
|
||||||
usageMetadata,
|
|
||||||
}) as unknown as GenerateContentResponse;
|
|
||||||
|
|
||||||
it('should return the usageMetadata from the last chunk that has it', () => {
|
|
||||||
const chunks: GenerateContentResponse[] = [
|
|
||||||
createMockResponse({
|
|
||||||
promptTokenCount: 10,
|
|
||||||
candidatesTokenCount: 20,
|
|
||||||
totalTokenCount: 30,
|
|
||||||
}),
|
|
||||||
createMockResponse(),
|
|
||||||
createMockResponse({
|
|
||||||
promptTokenCount: 15,
|
|
||||||
candidatesTokenCount: 25,
|
|
||||||
totalTokenCount: 40,
|
|
||||||
}),
|
|
||||||
createMockResponse(),
|
|
||||||
];
|
|
||||||
|
|
||||||
const result = getFinalUsageMetadata(chunks);
|
|
||||||
expect(result).toEqual({
|
|
||||||
promptTokenCount: 15,
|
|
||||||
candidatesTokenCount: 25,
|
|
||||||
totalTokenCount: 40,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return undefined if no chunks have usageMetadata', () => {
|
|
||||||
const chunks: GenerateContentResponse[] = [
|
|
||||||
createMockResponse(),
|
|
||||||
createMockResponse(),
|
|
||||||
createMockResponse(),
|
|
||||||
];
|
|
||||||
|
|
||||||
const result = getFinalUsageMetadata(chunks);
|
|
||||||
expect(result).toBeUndefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return the metadata from the only chunk if it has it', () => {
|
|
||||||
const chunks: GenerateContentResponse[] = [
|
|
||||||
createMockResponse({
|
|
||||||
promptTokenCount: 1,
|
|
||||||
candidatesTokenCount: 2,
|
|
||||||
totalTokenCount: 3,
|
|
||||||
}),
|
|
||||||
];
|
|
||||||
|
|
||||||
const result = getFinalUsageMetadata(chunks);
|
|
||||||
expect(result).toEqual({
|
|
||||||
promptTokenCount: 1,
|
|
||||||
candidatesTokenCount: 2,
|
|
||||||
totalTokenCount: 3,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return undefined for an empty array of chunks', () => {
|
|
||||||
const chunks: GenerateContentResponse[] = [];
|
|
||||||
const result = getFinalUsageMetadata(chunks);
|
|
||||||
expect(result).toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
|
@ -20,6 +20,7 @@ import {
|
||||||
ApiErrorEvent,
|
ApiErrorEvent,
|
||||||
ApiRequestEvent,
|
ApiRequestEvent,
|
||||||
ApiResponseEvent,
|
ApiResponseEvent,
|
||||||
|
StartSessionEvent,
|
||||||
ToolCallEvent,
|
ToolCallEvent,
|
||||||
UserPromptEvent,
|
UserPromptEvent,
|
||||||
} from './types.js';
|
} from './types.js';
|
||||||
|
@ -30,15 +31,10 @@ import {
|
||||||
recordToolCallMetrics,
|
recordToolCallMetrics,
|
||||||
} from './metrics.js';
|
} from './metrics.js';
|
||||||
import { isTelemetrySdkInitialized } from './sdk.js';
|
import { isTelemetrySdkInitialized } from './sdk.js';
|
||||||
import { ToolConfirmationOutcome } from '../tools/tools.js';
|
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
|
||||||
import {
|
|
||||||
GenerateContentResponse,
|
|
||||||
GenerateContentResponseUsageMetadata,
|
|
||||||
} from '@google/genai';
|
|
||||||
import { AuthType } from '../core/contentGenerator.js';
|
|
||||||
|
|
||||||
const shouldLogUserPrompts = (config: Config): boolean =>
|
const shouldLogUserPrompts = (config: Config): boolean =>
|
||||||
config.getTelemetryLogPromptsEnabled() ?? false;
|
config.getTelemetryLogPromptsEnabled();
|
||||||
|
|
||||||
function getCommonAttributes(config: Config): LogAttributes {
|
function getCommonAttributes(config: Config): LogAttributes {
|
||||||
return {
|
return {
|
||||||
|
@ -46,59 +42,30 @@ function getCommonAttributes(config: Config): LogAttributes {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum ToolCallDecision {
|
export function logCliConfiguration(
|
||||||
ACCEPT = 'accept',
|
config: Config,
|
||||||
REJECT = 'reject',
|
event: StartSessionEvent,
|
||||||
MODIFY = 'modify',
|
): void {
|
||||||
}
|
ClearcutLogger.getInstance(config)?.logStartSessionEvent(event);
|
||||||
|
|
||||||
export function getDecisionFromOutcome(
|
|
||||||
outcome: ToolConfirmationOutcome,
|
|
||||||
): ToolCallDecision {
|
|
||||||
switch (outcome) {
|
|
||||||
case ToolConfirmationOutcome.ProceedOnce:
|
|
||||||
case ToolConfirmationOutcome.ProceedAlways:
|
|
||||||
case ToolConfirmationOutcome.ProceedAlwaysServer:
|
|
||||||
case ToolConfirmationOutcome.ProceedAlwaysTool:
|
|
||||||
return ToolCallDecision.ACCEPT;
|
|
||||||
case ToolConfirmationOutcome.ModifyWithEditor:
|
|
||||||
return ToolCallDecision.MODIFY;
|
|
||||||
case ToolConfirmationOutcome.Cancel:
|
|
||||||
default:
|
|
||||||
return ToolCallDecision.REJECT;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function logCliConfiguration(config: Config): void {
|
|
||||||
if (!isTelemetrySdkInitialized()) return;
|
if (!isTelemetrySdkInitialized()) return;
|
||||||
|
|
||||||
const generatorConfig = config.getContentGeneratorConfig();
|
|
||||||
let useGemini = false;
|
|
||||||
let useVertex = false;
|
|
||||||
|
|
||||||
if (generatorConfig && generatorConfig.authType) {
|
|
||||||
useGemini = generatorConfig.authType === AuthType.USE_GEMINI;
|
|
||||||
useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI;
|
|
||||||
}
|
|
||||||
|
|
||||||
const mcpServers = config.getMcpServers();
|
|
||||||
const attributes: LogAttributes = {
|
const attributes: LogAttributes = {
|
||||||
...getCommonAttributes(config),
|
...getCommonAttributes(config),
|
||||||
'event.name': EVENT_CLI_CONFIG,
|
'event.name': EVENT_CLI_CONFIG,
|
||||||
'event.timestamp': new Date().toISOString(),
|
'event.timestamp': new Date().toISOString(),
|
||||||
model: config.getModel(),
|
model: event.model,
|
||||||
embedding_model: config.getEmbeddingModel(),
|
embedding_model: event.embedding_model,
|
||||||
sandbox_enabled: !!config.getSandbox(),
|
sandbox_enabled: event.sandbox_enabled,
|
||||||
core_tools_enabled: (config.getCoreTools() ?? []).join(','),
|
core_tools_enabled: event.core_tools_enabled,
|
||||||
approval_mode: config.getApprovalMode(),
|
approval_mode: event.approval_mode,
|
||||||
api_key_enabled: useGemini || useVertex,
|
api_key_enabled: event.api_key_enabled,
|
||||||
vertex_ai_enabled: useVertex,
|
vertex_ai_enabled: event.vertex_ai_enabled,
|
||||||
log_user_prompts_enabled: config.getTelemetryLogPromptsEnabled(),
|
log_user_prompts_enabled: event.telemetry_log_user_prompts_enabled,
|
||||||
file_filtering_respect_git_ignore:
|
file_filtering_respect_git_ignore: event.file_filtering_respect_git_ignore,
|
||||||
config.getFileFilteringRespectGitIgnore(),
|
debug_mode: event.debug_enabled,
|
||||||
debug_mode: config.getDebugMode(),
|
mcp_servers: event.mcp_servers,
|
||||||
mcp_servers: mcpServers ? Object.keys(mcpServers).join(',') : '',
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const logger = logs.getLogger(SERVICE_NAME);
|
const logger = logs.getLogger(SERVICE_NAME);
|
||||||
const logRecord: LogRecord = {
|
const logRecord: LogRecord = {
|
||||||
body: 'CLI configuration loaded.',
|
body: 'CLI configuration loaded.',
|
||||||
|
@ -107,12 +74,8 @@ export function logCliConfiguration(config: Config): void {
|
||||||
logger.emit(logRecord);
|
logger.emit(logRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function logUserPrompt(
|
export function logUserPrompt(config: Config, event: UserPromptEvent): void {
|
||||||
config: Config,
|
ClearcutLogger.getInstance(config)?.logNewPromptEvent(event);
|
||||||
event: Omit<UserPromptEvent, 'event.name' | 'event.timestamp' | 'prompt'> & {
|
|
||||||
prompt: string;
|
|
||||||
},
|
|
||||||
): void {
|
|
||||||
if (!isTelemetrySdkInitialized()) return;
|
if (!isTelemetrySdkInitialized()) return;
|
||||||
|
|
||||||
const attributes: LogAttributes = {
|
const attributes: LogAttributes = {
|
||||||
|
@ -134,22 +97,16 @@ export function logUserPrompt(
|
||||||
logger.emit(logRecord);
|
logger.emit(logRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function logToolCall(
|
export function logToolCall(config: Config, event: ToolCallEvent): void {
|
||||||
config: Config,
|
ClearcutLogger.getInstance(config)?.logToolCallEvent(event);
|
||||||
event: Omit<ToolCallEvent, 'event.name' | 'event.timestamp' | 'decision'>,
|
|
||||||
outcome?: ToolConfirmationOutcome,
|
|
||||||
): void {
|
|
||||||
if (!isTelemetrySdkInitialized()) return;
|
if (!isTelemetrySdkInitialized()) return;
|
||||||
|
|
||||||
const decision = outcome ? getDecisionFromOutcome(outcome) : undefined;
|
|
||||||
|
|
||||||
const attributes: LogAttributes = {
|
const attributes: LogAttributes = {
|
||||||
...getCommonAttributes(config),
|
...getCommonAttributes(config),
|
||||||
...event,
|
...event,
|
||||||
'event.name': EVENT_TOOL_CALL,
|
'event.name': EVENT_TOOL_CALL,
|
||||||
'event.timestamp': new Date().toISOString(),
|
'event.timestamp': new Date().toISOString(),
|
||||||
function_args: JSON.stringify(event.function_args, null, 2),
|
function_args: JSON.stringify(event.function_args, null, 2),
|
||||||
decision,
|
|
||||||
};
|
};
|
||||||
if (event.error) {
|
if (event.error) {
|
||||||
attributes['error.message'] = event.error;
|
attributes['error.message'] = event.error;
|
||||||
|
@ -157,9 +114,10 @@ export function logToolCall(
|
||||||
attributes['error.type'] = event.error_type;
|
attributes['error.type'] = event.error_type;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const logger = logs.getLogger(SERVICE_NAME);
|
const logger = logs.getLogger(SERVICE_NAME);
|
||||||
const logRecord: LogRecord = {
|
const logRecord: LogRecord = {
|
||||||
body: `Tool call: ${event.function_name}${decision ? `. Decision: ${decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
|
body: `Tool call: ${event.function_name}${event.decision ? `. Decision: ${event.decision}` : ''}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`,
|
||||||
attributes,
|
attributes,
|
||||||
};
|
};
|
||||||
logger.emit(logRecord);
|
logger.emit(logRecord);
|
||||||
|
@ -168,21 +126,21 @@ export function logToolCall(
|
||||||
event.function_name,
|
event.function_name,
|
||||||
event.duration_ms,
|
event.duration_ms,
|
||||||
event.success,
|
event.success,
|
||||||
decision,
|
event.decision,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function logApiRequest(
|
export function logApiRequest(config: Config, event: ApiRequestEvent): void {
|
||||||
config: Config,
|
ClearcutLogger.getInstance(config)?.logApiRequestEvent(event);
|
||||||
event: Omit<ApiRequestEvent, 'event.name' | 'event.timestamp'>,
|
|
||||||
): void {
|
|
||||||
if (!isTelemetrySdkInitialized()) return;
|
if (!isTelemetrySdkInitialized()) return;
|
||||||
|
|
||||||
const attributes: LogAttributes = {
|
const attributes: LogAttributes = {
|
||||||
...getCommonAttributes(config),
|
...getCommonAttributes(config),
|
||||||
...event,
|
...event,
|
||||||
'event.name': EVENT_API_REQUEST,
|
'event.name': EVENT_API_REQUEST,
|
||||||
'event.timestamp': new Date().toISOString(),
|
'event.timestamp': new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
|
||||||
const logger = logs.getLogger(SERVICE_NAME);
|
const logger = logs.getLogger(SERVICE_NAME);
|
||||||
const logRecord: LogRecord = {
|
const logRecord: LogRecord = {
|
||||||
body: `API request to ${event.model}.`,
|
body: `API request to ${event.model}.`,
|
||||||
|
@ -191,17 +149,18 @@ export function logApiRequest(
|
||||||
logger.emit(logRecord);
|
logger.emit(logRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function logApiError(
|
export function logApiError(config: Config, event: ApiErrorEvent): void {
|
||||||
config: Config,
|
ClearcutLogger.getInstance(config)?.logApiErrorEvent(event);
|
||||||
event: Omit<ApiErrorEvent, 'event.name' | 'event.timestamp'>,
|
|
||||||
): void {
|
|
||||||
if (!isTelemetrySdkInitialized()) return;
|
if (!isTelemetrySdkInitialized()) return;
|
||||||
|
|
||||||
const attributes: LogAttributes = {
|
const attributes: LogAttributes = {
|
||||||
...getCommonAttributes(config),
|
...getCommonAttributes(config),
|
||||||
...event,
|
...event,
|
||||||
'event.name': EVENT_API_ERROR,
|
'event.name': EVENT_API_ERROR,
|
||||||
'event.timestamp': new Date().toISOString(),
|
'event.timestamp': new Date().toISOString(),
|
||||||
['error.message']: event.error,
|
['error.message']: event.error,
|
||||||
|
model_name: event.model,
|
||||||
|
duration: event.duration_ms,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (event.error_type) {
|
if (event.error_type) {
|
||||||
|
@ -226,10 +185,8 @@ export function logApiError(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function logApiResponse(
|
export function logApiResponse(config: Config, event: ApiResponseEvent): void {
|
||||||
config: Config,
|
ClearcutLogger.getInstance(config)?.logApiResponseEvent(event);
|
||||||
event: Omit<ApiResponseEvent, 'event.name' | 'event.timestamp'>,
|
|
||||||
): void {
|
|
||||||
if (!isTelemetrySdkInitialized()) return;
|
if (!isTelemetrySdkInitialized()) return;
|
||||||
const attributes: LogAttributes = {
|
const attributes: LogAttributes = {
|
||||||
...getCommonAttributes(config),
|
...getCommonAttributes(config),
|
||||||
|
@ -287,15 +244,3 @@ export function logApiResponse(
|
||||||
);
|
);
|
||||||
recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool');
|
recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool');
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getFinalUsageMetadata(
|
|
||||||
chunks: GenerateContentResponse[],
|
|
||||||
): GenerateContentResponseUsageMetadata | undefined {
|
|
||||||
// Only the last streamed item has the final token count.
|
|
||||||
const lastChunkWithMetadata = chunks
|
|
||||||
.slice()
|
|
||||||
.reverse()
|
|
||||||
.find((chunk) => chunk.usageMetadata);
|
|
||||||
|
|
||||||
return lastChunkWithMetadata?.usageMetadata;
|
|
||||||
}
|
|
||||||
|
|
|
@ -29,6 +29,8 @@ import { Config } from '../config/config.js';
|
||||||
import { SERVICE_NAME } from './constants.js';
|
import { SERVICE_NAME } from './constants.js';
|
||||||
import { initializeMetrics } from './metrics.js';
|
import { initializeMetrics } from './metrics.js';
|
||||||
import { logCliConfiguration } from './loggers.js';
|
import { logCliConfiguration } from './loggers.js';
|
||||||
|
import { StartSessionEvent } from './types.js';
|
||||||
|
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
|
||||||
|
|
||||||
// For troubleshooting, set the log level to DiagLogLevel.DEBUG
|
// For troubleshooting, set the log level to DiagLogLevel.DEBUG
|
||||||
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
|
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
|
||||||
|
@ -113,7 +115,7 @@ export function initializeTelemetry(config: Config): void {
|
||||||
console.log('OpenTelemetry SDK started successfully.');
|
console.log('OpenTelemetry SDK started successfully.');
|
||||||
telemetryInitialized = true;
|
telemetryInitialized = true;
|
||||||
initializeMetrics(config);
|
initializeMetrics(config);
|
||||||
logCliConfiguration(config);
|
logCliConfiguration(config, new StartSessionEvent(config));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error starting OpenTelemetry SDK:', error);
|
console.error('Error starting OpenTelemetry SDK:', error);
|
||||||
}
|
}
|
||||||
|
@ -127,6 +129,7 @@ export async function shutdownTelemetry(): Promise<void> {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
ClearcutLogger.getInstance()?.shutdown();
|
||||||
await sdk.shutdown();
|
await sdk.shutdown();
|
||||||
console.log('OpenTelemetry SDK shut down successfully.');
|
console.log('OpenTelemetry SDK shut down successfully.');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
|
@ -13,6 +13,7 @@ import {
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||||
import * as loggers from './loggers.js';
|
import * as loggers from './loggers.js';
|
||||||
|
import { StartSessionEvent } from './types.js';
|
||||||
|
|
||||||
vi.mock('@opentelemetry/sdk-node');
|
vi.mock('@opentelemetry/sdk-node');
|
||||||
vi.mock('../config/config.js');
|
vi.mock('../config/config.js');
|
||||||
|
@ -55,10 +56,11 @@ describe('telemetry', () => {
|
||||||
|
|
||||||
it('should initialize the telemetry service', () => {
|
it('should initialize the telemetry service', () => {
|
||||||
initializeTelemetry(mockConfig);
|
initializeTelemetry(mockConfig);
|
||||||
|
const event = new StartSessionEvent(mockConfig);
|
||||||
|
|
||||||
expect(NodeSDK).toHaveBeenCalled();
|
expect(NodeSDK).toHaveBeenCalled();
|
||||||
expect(mockNodeSdk.start).toHaveBeenCalled();
|
expect(mockNodeSdk.start).toHaveBeenCalled();
|
||||||
expect(loggers.logCliConfiguration).toHaveBeenCalledWith(mockConfig);
|
expect(loggers.logCliConfiguration).toHaveBeenCalledWith(mockConfig, event);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should shutdown the telemetry service', async () => {
|
it('should shutdown the telemetry service', async () => {
|
||||||
|
|
|
@ -4,16 +4,108 @@
|
||||||
* SPDX-License-Identifier: Apache-2.0
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ToolCallDecision } from './loggers.js';
|
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
||||||
|
import { Config } from '../config/config.js';
|
||||||
|
import { CompletedToolCall } from '../core/coreToolScheduler.js';
|
||||||
|
import { ToolConfirmationOutcome } from '../tools/tools.js';
|
||||||
|
import { AuthType } from '../core/contentGenerator.js';
|
||||||
|
|
||||||
export interface UserPromptEvent {
|
export enum ToolCallDecision {
|
||||||
|
ACCEPT = 'accept',
|
||||||
|
REJECT = 'reject',
|
||||||
|
MODIFY = 'modify',
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getDecisionFromOutcome(
|
||||||
|
outcome: ToolConfirmationOutcome,
|
||||||
|
): ToolCallDecision {
|
||||||
|
switch (outcome) {
|
||||||
|
case ToolConfirmationOutcome.ProceedOnce:
|
||||||
|
case ToolConfirmationOutcome.ProceedAlways:
|
||||||
|
case ToolConfirmationOutcome.ProceedAlwaysServer:
|
||||||
|
case ToolConfirmationOutcome.ProceedAlwaysTool:
|
||||||
|
return ToolCallDecision.ACCEPT;
|
||||||
|
case ToolConfirmationOutcome.ModifyWithEditor:
|
||||||
|
return ToolCallDecision.MODIFY;
|
||||||
|
case ToolConfirmationOutcome.Cancel:
|
||||||
|
default:
|
||||||
|
return ToolCallDecision.REJECT;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class StartSessionEvent {
|
||||||
|
'event.name': 'cli_config';
|
||||||
|
'event.timestamp': string; // ISO 8601
|
||||||
|
model: string;
|
||||||
|
embedding_model: string;
|
||||||
|
sandbox_enabled: boolean;
|
||||||
|
core_tools_enabled: string;
|
||||||
|
approval_mode: string;
|
||||||
|
api_key_enabled: boolean;
|
||||||
|
vertex_ai_enabled: boolean;
|
||||||
|
debug_enabled: boolean;
|
||||||
|
mcp_servers: string;
|
||||||
|
telemetry_enabled: boolean;
|
||||||
|
telemetry_log_user_prompts_enabled: boolean;
|
||||||
|
file_filtering_respect_git_ignore: boolean;
|
||||||
|
|
||||||
|
constructor(config: Config) {
|
||||||
|
const generatorConfig = config.getContentGeneratorConfig();
|
||||||
|
const mcpServers = config.getMcpServers();
|
||||||
|
|
||||||
|
let useGemini = false;
|
||||||
|
let useVertex = false;
|
||||||
|
if (generatorConfig && generatorConfig.authType) {
|
||||||
|
useGemini = generatorConfig.authType === AuthType.USE_GEMINI;
|
||||||
|
useVertex = generatorConfig.authType === AuthType.USE_VERTEX_AI;
|
||||||
|
}
|
||||||
|
|
||||||
|
this['event.name'] = 'cli_config';
|
||||||
|
this.model = config.getModel();
|
||||||
|
this.embedding_model = config.getEmbeddingModel();
|
||||||
|
this.sandbox_enabled =
|
||||||
|
typeof config.getSandbox() === 'string' || !!config.getSandbox();
|
||||||
|
this.core_tools_enabled = (config.getCoreTools() ?? []).join(',');
|
||||||
|
this.approval_mode = config.getApprovalMode();
|
||||||
|
this.api_key_enabled = useGemini || useVertex;
|
||||||
|
this.vertex_ai_enabled = useVertex;
|
||||||
|
this.debug_enabled = config.getDebugMode();
|
||||||
|
this.mcp_servers = mcpServers ? Object.keys(mcpServers).join(',') : '';
|
||||||
|
this.telemetry_enabled = config.getTelemetryEnabled();
|
||||||
|
this.telemetry_log_user_prompts_enabled =
|
||||||
|
config.getTelemetryLogPromptsEnabled();
|
||||||
|
this.file_filtering_respect_git_ignore =
|
||||||
|
config.getFileFilteringRespectGitIgnore();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class EndSessionEvent {
|
||||||
|
'event.name': 'end_session';
|
||||||
|
'event.timestamp': string; // ISO 8601
|
||||||
|
session_id?: string;
|
||||||
|
|
||||||
|
constructor(config?: Config) {
|
||||||
|
this['event.name'] = 'end_session';
|
||||||
|
this['event.timestamp'] = new Date().toISOString();
|
||||||
|
this.session_id = config?.getSessionId();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class UserPromptEvent {
|
||||||
'event.name': 'user_prompt';
|
'event.name': 'user_prompt';
|
||||||
'event.timestamp': string; // ISO 8601
|
'event.timestamp': string; // ISO 8601
|
||||||
prompt_length: number;
|
prompt_length: number;
|
||||||
prompt?: string;
|
prompt?: string;
|
||||||
|
|
||||||
|
constructor(prompt_length: number, prompt?: string) {
|
||||||
|
this['event.name'] = 'user_prompt';
|
||||||
|
this['event.timestamp'] = new Date().toISOString();
|
||||||
|
this.prompt_length = prompt_length;
|
||||||
|
this.prompt = prompt;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ToolCallEvent {
|
export class ToolCallEvent {
|
||||||
'event.name': 'tool_call';
|
'event.name': 'tool_call';
|
||||||
'event.timestamp': string; // ISO 8601
|
'event.timestamp': string; // ISO 8601
|
||||||
function_name: string;
|
function_name: string;
|
||||||
|
@ -23,16 +115,37 @@ export interface ToolCallEvent {
|
||||||
decision?: ToolCallDecision;
|
decision?: ToolCallDecision;
|
||||||
error?: string;
|
error?: string;
|
||||||
error_type?: string;
|
error_type?: string;
|
||||||
|
|
||||||
|
constructor(call: CompletedToolCall) {
|
||||||
|
this['event.name'] = 'tool_call';
|
||||||
|
this['event.timestamp'] = new Date().toISOString();
|
||||||
|
this.function_name = call.request.name;
|
||||||
|
this.function_args = call.request.args;
|
||||||
|
this.duration_ms = call.durationMs ?? 0;
|
||||||
|
this.success = call.status === 'success';
|
||||||
|
this.decision = call.outcome
|
||||||
|
? getDecisionFromOutcome(call.outcome)
|
||||||
|
: undefined;
|
||||||
|
this.error = call.response.error?.message;
|
||||||
|
this.error_type = call.response.error?.name;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ApiRequestEvent {
|
export class ApiRequestEvent {
|
||||||
'event.name': 'api_request';
|
'event.name': 'api_request';
|
||||||
'event.timestamp': string; // ISO 8601
|
'event.timestamp': string; // ISO 8601
|
||||||
model: string;
|
model: string;
|
||||||
request_text?: string;
|
request_text?: string;
|
||||||
|
|
||||||
|
constructor(model: string, request_text?: string) {
|
||||||
|
this['event.name'] = 'api_request';
|
||||||
|
this['event.timestamp'] = new Date().toISOString();
|
||||||
|
this.model = model;
|
||||||
|
this.request_text = request_text;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ApiErrorEvent {
|
export class ApiErrorEvent {
|
||||||
'event.name': 'api_error';
|
'event.name': 'api_error';
|
||||||
'event.timestamp': string; // ISO 8601
|
'event.timestamp': string; // ISO 8601
|
||||||
model: string;
|
model: string;
|
||||||
|
@ -40,9 +153,25 @@ export interface ApiErrorEvent {
|
||||||
error_type?: string;
|
error_type?: string;
|
||||||
status_code?: number | string;
|
status_code?: number | string;
|
||||||
duration_ms: number;
|
duration_ms: number;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
model: string,
|
||||||
|
error: string,
|
||||||
|
duration_ms: number,
|
||||||
|
error_type?: string,
|
||||||
|
status_code?: number | string,
|
||||||
|
) {
|
||||||
|
this['event.name'] = 'api_error';
|
||||||
|
this['event.timestamp'] = new Date().toISOString();
|
||||||
|
this.model = model;
|
||||||
|
this.error = error;
|
||||||
|
this.error_type = error_type;
|
||||||
|
this.status_code = status_code;
|
||||||
|
this.duration_ms = duration_ms;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ApiResponseEvent {
|
export class ApiResponseEvent {
|
||||||
'event.name': 'api_response';
|
'event.name': 'api_response';
|
||||||
'event.timestamp': string; // ISO 8601
|
'event.timestamp': string; // ISO 8601
|
||||||
model: string;
|
model: string;
|
||||||
|
@ -55,24 +184,34 @@ export interface ApiResponseEvent {
|
||||||
thoughts_token_count: number;
|
thoughts_token_count: number;
|
||||||
tool_token_count: number;
|
tool_token_count: number;
|
||||||
response_text?: string;
|
response_text?: string;
|
||||||
}
|
|
||||||
|
|
||||||
export interface CliConfigEvent {
|
constructor(
|
||||||
'event.name': 'cli_config';
|
model: string,
|
||||||
'event.timestamp': string; // ISO 8601
|
duration_ms: number,
|
||||||
model: string;
|
usage_data?: GenerateContentResponseUsageMetadata,
|
||||||
sandbox_enabled: boolean;
|
response_text?: string,
|
||||||
core_tools_enabled: string;
|
error?: string,
|
||||||
approval_mode: string;
|
) {
|
||||||
vertex_ai_enabled: boolean;
|
this['event.name'] = 'api_response';
|
||||||
log_user_prompts_enabled: boolean;
|
this['event.timestamp'] = new Date().toISOString();
|
||||||
file_filtering_respect_git_ignore: boolean;
|
this.model = model;
|
||||||
|
this.duration_ms = duration_ms;
|
||||||
|
this.status_code = 200;
|
||||||
|
this.input_token_count = usage_data?.promptTokenCount ?? 0;
|
||||||
|
this.output_token_count = usage_data?.candidatesTokenCount ?? 0;
|
||||||
|
this.cached_content_token_count = usage_data?.cachedContentTokenCount ?? 0;
|
||||||
|
this.thoughts_token_count = usage_data?.thoughtsTokenCount ?? 0;
|
||||||
|
this.tool_token_count = usage_data?.toolUsePromptTokenCount ?? 0;
|
||||||
|
this.response_text = response_text;
|
||||||
|
this.error = error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export type TelemetryEvent =
|
export type TelemetryEvent =
|
||||||
|
| StartSessionEvent
|
||||||
|
| EndSessionEvent
|
||||||
| UserPromptEvent
|
| UserPromptEvent
|
||||||
| ToolCallEvent
|
| ToolCallEvent
|
||||||
| ApiRequestEvent
|
| ApiRequestEvent
|
||||||
| ApiErrorEvent
|
| ApiErrorEvent
|
||||||
| ApiResponseEvent
|
| ApiResponseEvent;
|
||||||
| CliConfigEvent;
|
|
||||||
|
|
|
@ -0,0 +1,58 @@
|
||||||
|
/**
|
||||||
|
* @license
|
||||||
|
* Copyright 2025 Google LLC
|
||||||
|
* SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as os from 'os';
|
||||||
|
import * as fs from 'fs';
|
||||||
|
import * as path from 'path';
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { GEMINI_DIR } from './paths.js';
|
||||||
|
|
||||||
|
const homeDir = os.homedir() ?? '';
|
||||||
|
const geminiDir = path.join(homeDir, GEMINI_DIR);
|
||||||
|
const userIdFile = path.join(geminiDir, 'user_id');
|
||||||
|
|
||||||
|
function ensureGeminiDirExists() {
|
||||||
|
if (!fs.existsSync(geminiDir)) {
|
||||||
|
fs.mkdirSync(geminiDir, { recursive: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readUserIdFromFile(): string | null {
|
||||||
|
if (fs.existsSync(userIdFile)) {
|
||||||
|
const userId = fs.readFileSync(userIdFile, 'utf-8').trim();
|
||||||
|
return userId || null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeUserIdToFile(userId: string) {
|
||||||
|
fs.writeFileSync(userIdFile, userId, 'utf-8');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Retrieves the persistent user ID from a file, creating it if it doesn't exist.
|
||||||
|
* This ID is used for unique user tracking.
|
||||||
|
* @returns A UUID string for the user.
|
||||||
|
*/
|
||||||
|
export function getPersistentUserId(): string {
|
||||||
|
try {
|
||||||
|
ensureGeminiDirExists();
|
||||||
|
let userId = readUserIdFromFile();
|
||||||
|
|
||||||
|
if (!userId) {
|
||||||
|
userId = randomUUID();
|
||||||
|
writeUserIdToFile(userId);
|
||||||
|
}
|
||||||
|
|
||||||
|
return userId;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(
|
||||||
|
'Error accessing persistent user ID file, generating ephemeral ID:',
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
return '123456789';
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue