feat(telemetry): expand cli configuration event
Adds the following attributes to the event: - embedding_model - api_key_enabled - code_assist_enabled - debug_mode - mcp_servers This additional data will provide more insight into user configurations.
This commit is contained in:
parent
5586ad5f8a
commit
c0580eaf4b
|
@ -262,13 +262,18 @@ These are timestamped records of specific events.
|
||||||
|
|
||||||
- **Attributes**:
|
- **Attributes**:
|
||||||
- `model` (string)
|
- `model` (string)
|
||||||
|
- `embedding_model` (string)
|
||||||
- `sandbox_enabled` (boolean)
|
- `sandbox_enabled` (boolean)
|
||||||
- `core_tools_enabled` (string)
|
- `core_tools_enabled` (string)
|
||||||
- `approval_mode` (string)
|
- `approval_mode` (string)
|
||||||
|
- `api_key_enabled` (boolean)
|
||||||
- `vertex_ai_enabled` (boolean)
|
- `vertex_ai_enabled` (boolean)
|
||||||
|
- `code_assist_enabled` (boolean)
|
||||||
- `log_user_prompts_enabled` (boolean)
|
- `log_user_prompts_enabled` (boolean)
|
||||||
- `file_filtering_respect_git_ignore` (boolean)
|
- `file_filtering_respect_git_ignore` (boolean)
|
||||||
- `file_filtering_allow_build_artifacts` (boolean)
|
- `file_filtering_allow_build_artifacts` (boolean)
|
||||||
|
- `debug_mode` (boolean)
|
||||||
|
- `mcp_servers` (string)
|
||||||
|
|
||||||
- `gemini_cli.user_prompt`: Fired when a user submits a prompt.
|
- `gemini_cli.user_prompt`: Fired when a user submits a prompt.
|
||||||
|
|
||||||
|
|
|
@ -8,60 +8,105 @@ import { logs } from '@opentelemetry/api-logs';
|
||||||
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
||||||
import { Config } from '../config/config.js';
|
import { Config } from '../config/config.js';
|
||||||
import { EVENT_API_RESPONSE } from './constants.js';
|
import { EVENT_API_RESPONSE } from './constants.js';
|
||||||
import { logApiResponse } from './loggers.js';
|
import { logApiResponse, logCliConfiguration } from './loggers.js';
|
||||||
import * as metrics from './metrics.js';
|
import * as metrics from './metrics.js';
|
||||||
import * as sdk from './sdk.js';
|
import * as sdk from './sdk.js';
|
||||||
import { vi, describe, beforeEach, it, expect } from 'vitest';
|
import { vi, describe, beforeEach, it, expect } from 'vitest';
|
||||||
|
|
||||||
describe('logApiResponse', () => {
|
vi.mock('@gemini-cli/cli/dist/src/utils/version', () => ({
|
||||||
const mockConfig = {
|
getCliVersion: () => 'test-version',
|
||||||
getSessionId: () => 'test-session-id',
|
}));
|
||||||
} as Config;
|
|
||||||
|
|
||||||
|
vi.mock('@gemini-cli/cli/dist/src/config/settings', () => ({
|
||||||
|
getTheme: () => 'test-theme',
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe('loggers', () => {
|
||||||
const mockLogger = {
|
const mockLogger = {
|
||||||
emit: vi.fn(),
|
emit: vi.fn(),
|
||||||
};
|
};
|
||||||
|
|
||||||
const mockMetrics = {
|
|
||||||
recordApiResponseMetrics: vi.fn(),
|
|
||||||
recordTokenUsageMetrics: vi.fn(),
|
|
||||||
};
|
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.spyOn(sdk, 'isTelemetrySdkInitialized').mockReturnValue(true);
|
vi.spyOn(sdk, 'isTelemetrySdkInitialized').mockReturnValue(true);
|
||||||
vi.spyOn(logs, 'getLogger').mockReturnValue(mockLogger);
|
vi.spyOn(logs, 'getLogger').mockReturnValue(mockLogger);
|
||||||
vi.spyOn(metrics, 'recordApiResponseMetrics').mockImplementation(
|
|
||||||
mockMetrics.recordApiResponseMetrics,
|
|
||||||
);
|
|
||||||
vi.spyOn(metrics, 'recordTokenUsageMetrics').mockImplementation(
|
|
||||||
mockMetrics.recordTokenUsageMetrics,
|
|
||||||
);
|
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z'));
|
vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z'));
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should log an API response with all fields', () => {
|
describe('logCliConfiguration', () => {
|
||||||
const event = {
|
it('should log the cli configuration', () => {
|
||||||
model: 'test-model',
|
const mockConfig = {
|
||||||
status_code: 200,
|
getSessionId: () => 'test-session-id',
|
||||||
duration_ms: 100,
|
getModel: () => 'test-model',
|
||||||
attempt: 1,
|
getEmbeddingModel: () => 'test-embedding-model',
|
||||||
output_token_count: 50,
|
getSandbox: () => true,
|
||||||
cached_content_token_count: 10,
|
getCoreTools: () => ['ls', 'read-file'],
|
||||||
thoughts_token_count: 5,
|
getApprovalMode: () => 'default',
|
||||||
tool_token_count: 2,
|
getContentGeneratorConfig: () => ({
|
||||||
response_text: 'test-response',
|
model: 'test-model',
|
||||||
|
apiKey: 'test-api-key',
|
||||||
|
vertexai: true,
|
||||||
|
codeAssist: false,
|
||||||
|
}),
|
||||||
|
getTelemetryLogUserPromptsEnabled: () => true,
|
||||||
|
getFileFilteringRespectGitIgnore: () => true,
|
||||||
|
getFileFilteringAllowBuildArtifacts: () => false,
|
||||||
|
getDebugMode: () => true,
|
||||||
|
getMcpServers: () => ({
|
||||||
|
'test-server': {
|
||||||
|
command: 'test-command',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
getQuestion: () => 'test-question',
|
||||||
|
} as unknown as Config;
|
||||||
|
|
||||||
|
logCliConfiguration(mockConfig);
|
||||||
|
|
||||||
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
|
body: 'CLI configuration loaded.',
|
||||||
|
attributes: {
|
||||||
|
'session.id': 'test-session-id',
|
||||||
|
'event.name': 'gemini_cli.config',
|
||||||
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
|
model: 'test-model',
|
||||||
|
embedding_model: 'test-embedding-model',
|
||||||
|
sandbox_enabled: true,
|
||||||
|
core_tools_enabled: 'ls,read-file',
|
||||||
|
approval_mode: 'default',
|
||||||
|
api_key_enabled: true,
|
||||||
|
vertex_ai_enabled: true,
|
||||||
|
code_assist_enabled: false,
|
||||||
|
log_user_prompts_enabled: true,
|
||||||
|
file_filtering_respect_git_ignore: true,
|
||||||
|
file_filtering_allow_build_artifacts: false,
|
||||||
|
debug_mode: true,
|
||||||
|
mcp_servers: 'test-server',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('logApiResponse', () => {
|
||||||
|
const mockConfig = {
|
||||||
|
getSessionId: () => 'test-session-id',
|
||||||
|
} as Config;
|
||||||
|
|
||||||
|
const mockMetrics = {
|
||||||
|
recordApiResponseMetrics: vi.fn(),
|
||||||
|
recordTokenUsageMetrics: vi.fn(),
|
||||||
};
|
};
|
||||||
|
|
||||||
logApiResponse(mockConfig, event);
|
beforeEach(() => {
|
||||||
|
vi.spyOn(metrics, 'recordApiResponseMetrics').mockImplementation(
|
||||||
|
mockMetrics.recordApiResponseMetrics,
|
||||||
|
);
|
||||||
|
vi.spyOn(metrics, 'recordTokenUsageMetrics').mockImplementation(
|
||||||
|
mockMetrics.recordTokenUsageMetrics,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
it('should log an API response with all fields', () => {
|
||||||
body: 'API response from test-model. Status: 200. Duration: 100ms.',
|
const event = {
|
||||||
attributes: {
|
|
||||||
'session.id': 'test-session-id',
|
|
||||||
'event.name': EVENT_API_RESPONSE,
|
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
|
||||||
[SemanticAttributes.HTTP_STATUS_CODE]: 200,
|
|
||||||
model: 'test-model',
|
model: 'test-model',
|
||||||
status_code: 200,
|
status_code: 200,
|
||||||
duration_ms: 100,
|
duration_ms: 100,
|
||||||
|
@ -71,49 +116,70 @@ describe('logApiResponse', () => {
|
||||||
thoughts_token_count: 5,
|
thoughts_token_count: 5,
|
||||||
tool_token_count: 2,
|
tool_token_count: 2,
|
||||||
response_text: 'test-response',
|
response_text: 'test-response',
|
||||||
},
|
};
|
||||||
|
|
||||||
|
logApiResponse(mockConfig, event);
|
||||||
|
|
||||||
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
|
body: 'API response from test-model. Status: 200. Duration: 100ms.',
|
||||||
|
attributes: {
|
||||||
|
'session.id': 'test-session-id',
|
||||||
|
'event.name': EVENT_API_RESPONSE,
|
||||||
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
|
[SemanticAttributes.HTTP_STATUS_CODE]: 200,
|
||||||
|
model: 'test-model',
|
||||||
|
status_code: 200,
|
||||||
|
duration_ms: 100,
|
||||||
|
attempt: 1,
|
||||||
|
output_token_count: 50,
|
||||||
|
cached_content_token_count: 10,
|
||||||
|
thoughts_token_count: 5,
|
||||||
|
tool_token_count: 2,
|
||||||
|
response_text: 'test-response',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith(
|
||||||
|
mockConfig,
|
||||||
|
'test-model',
|
||||||
|
100,
|
||||||
|
200,
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(mockMetrics.recordTokenUsageMetrics).toHaveBeenCalledWith(
|
||||||
|
mockConfig,
|
||||||
|
'test-model',
|
||||||
|
50,
|
||||||
|
'output',
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith(
|
it('should log an API response with an error', () => {
|
||||||
mockConfig,
|
const event = {
|
||||||
'test-model',
|
model: 'test-model',
|
||||||
100,
|
duration_ms: 100,
|
||||||
200,
|
attempt: 1,
|
||||||
undefined,
|
error: 'test-error',
|
||||||
);
|
output_token_count: 50,
|
||||||
|
cached_content_token_count: 10,
|
||||||
|
thoughts_token_count: 5,
|
||||||
|
tool_token_count: 2,
|
||||||
|
response_text: 'test-response',
|
||||||
|
};
|
||||||
|
|
||||||
expect(mockMetrics.recordTokenUsageMetrics).toHaveBeenCalledWith(
|
logApiResponse(mockConfig, event);
|
||||||
mockConfig,
|
|
||||||
'test-model',
|
|
||||||
50,
|
|
||||||
'output',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should log an API response with an error', () => {
|
expect(mockLogger.emit).toHaveBeenCalledWith({
|
||||||
const event = {
|
body: 'API response from test-model. Status: N/A. Duration: 100ms.',
|
||||||
model: 'test-model',
|
attributes: {
|
||||||
duration_ms: 100,
|
'session.id': 'test-session-id',
|
||||||
attempt: 1,
|
...event,
|
||||||
error: 'test-error',
|
'event.name': EVENT_API_RESPONSE,
|
||||||
output_token_count: 50,
|
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||||
cached_content_token_count: 10,
|
'error.message': 'test-error',
|
||||||
thoughts_token_count: 5,
|
},
|
||||||
tool_token_count: 2,
|
});
|
||||||
response_text: 'test-response',
|
|
||||||
};
|
|
||||||
|
|
||||||
logApiResponse(mockConfig, event);
|
|
||||||
|
|
||||||
expect(mockLogger.emit).toHaveBeenCalledWith({
|
|
||||||
body: 'API response from test-model. Status: N/A. Duration: 100ms.',
|
|
||||||
attributes: {
|
|
||||||
'session.id': 'test-session-id',
|
|
||||||
...event,
|
|
||||||
'event.name': EVENT_API_RESPONSE,
|
|
||||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
|
||||||
'error.message': 'test-error',
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -43,21 +43,28 @@ function getCommonAttributes(config: Config): LogAttributes {
|
||||||
export function logCliConfiguration(config: Config): void {
|
export function logCliConfiguration(config: Config): void {
|
||||||
if (!isTelemetrySdkInitialized()) return;
|
if (!isTelemetrySdkInitialized()) return;
|
||||||
|
|
||||||
|
const generatorConfig = config.getContentGeneratorConfig();
|
||||||
|
const mcpServers = config.getMcpServers();
|
||||||
const attributes: LogAttributes = {
|
const attributes: LogAttributes = {
|
||||||
...getCommonAttributes(config),
|
...getCommonAttributes(config),
|
||||||
'event.name': EVENT_CLI_CONFIG,
|
'event.name': EVENT_CLI_CONFIG,
|
||||||
'event.timestamp': new Date().toISOString(),
|
'event.timestamp': new Date().toISOString(),
|
||||||
model: config.getModel(),
|
model: config.getModel(),
|
||||||
|
embedding_model: config.getEmbeddingModel(),
|
||||||
sandbox_enabled:
|
sandbox_enabled:
|
||||||
typeof config.getSandbox() === 'string' ? true : config.getSandbox(),
|
typeof config.getSandbox() === 'string' ? true : config.getSandbox(),
|
||||||
core_tools_enabled: (config.getCoreTools() ?? []).join(','),
|
core_tools_enabled: (config.getCoreTools() ?? []).join(','),
|
||||||
approval_mode: config.getApprovalMode(),
|
approval_mode: config.getApprovalMode(),
|
||||||
vertex_ai_enabled: !!config.getContentGeneratorConfig().vertexai,
|
api_key_enabled: !!generatorConfig.apiKey,
|
||||||
|
vertex_ai_enabled: !!generatorConfig.vertexai,
|
||||||
|
code_assist_enabled: !!generatorConfig.codeAssist,
|
||||||
log_user_prompts_enabled: config.getTelemetryLogUserPromptsEnabled(),
|
log_user_prompts_enabled: config.getTelemetryLogUserPromptsEnabled(),
|
||||||
file_filtering_respect_git_ignore:
|
file_filtering_respect_git_ignore:
|
||||||
config.getFileFilteringRespectGitIgnore(),
|
config.getFileFilteringRespectGitIgnore(),
|
||||||
file_filtering_allow_build_artifacts:
|
file_filtering_allow_build_artifacts:
|
||||||
config.getFileFilteringAllowBuildArtifacts(),
|
config.getFileFilteringAllowBuildArtifacts(),
|
||||||
|
debug_mode: config.getDebugMode(),
|
||||||
|
mcp_servers: mcpServers ? Object.keys(mcpServers).join(',') : '',
|
||||||
};
|
};
|
||||||
const logger = logs.getLogger(SERVICE_NAME);
|
const logger = logs.getLogger(SERVICE_NAME);
|
||||||
const logRecord: LogRecord = {
|
const logRecord: LogRecord = {
|
||||||
|
|
Loading…
Reference in New Issue