Logs the auth type in the user prompts, api responses and errors (#3795)
This commit is contained in:
parent
ed00612cf7
commit
93284281de
|
@ -154,6 +154,7 @@ Logs are timestamped records of specific events. The following events are logged
|
|||
- **Attributes**:
|
||||
- `prompt_length`
|
||||
- `prompt` (this attribute is excluded if `log_prompts_enabled` is configured to be `false`)
|
||||
- `auth_type`
|
||||
|
||||
- `gemini_cli.tool_call`: This event occurs for each function call.
|
||||
- **Attributes**:
|
||||
|
@ -177,6 +178,7 @@ Logs are timestamped records of specific events. The following events are logged
|
|||
- `error_type`
|
||||
- `status_code`
|
||||
- `duration_ms`
|
||||
- `auth_type`
|
||||
|
||||
- `gemini_cli.api_response`: This event occurs upon receiving a response from Gemini API.
|
||||
- **Attributes**:
|
||||
|
@ -190,6 +192,7 @@ Logs are timestamped records of specific events. The following events are logged
|
|||
- `thoughts_token_count`
|
||||
- `tool_token_count`
|
||||
- `response_text` (if applicable)
|
||||
- `auth_type`
|
||||
|
||||
### Metrics
|
||||
|
||||
|
|
|
@ -212,6 +212,7 @@ export async function main() {
|
|||
'event.timestamp': new Date().toISOString(),
|
||||
prompt: input,
|
||||
prompt_id,
|
||||
auth_type: config.getContentGeneratorConfig().authType!,
|
||||
prompt_length: input.length,
|
||||
});
|
||||
|
||||
|
|
|
@ -273,6 +273,13 @@ describe('useGeminiStream', () => {
|
|||
return clientInstance;
|
||||
});
|
||||
|
||||
const contentGeneratorConfig = {
|
||||
model: 'test-model',
|
||||
apiKey: 'test-key',
|
||||
vertexai: false,
|
||||
authType: AuthType.USE_GEMINI,
|
||||
};
|
||||
|
||||
mockConfig = {
|
||||
apiKey: 'test-api-key',
|
||||
model: 'gemini-pro',
|
||||
|
@ -307,6 +314,9 @@ describe('useGeminiStream', () => {
|
|||
},
|
||||
setQuotaErrorOccurred: vi.fn(),
|
||||
getQuotaErrorOccurred: vi.fn(() => false),
|
||||
getContentGeneratorConfig: vi
|
||||
.fn()
|
||||
.mockReturnValue(contentGeneratorConfig),
|
||||
} as unknown as Config;
|
||||
mockOnDebugMessage = vi.fn();
|
||||
mockHandleSlashCommand = vi.fn().mockResolvedValue(false);
|
||||
|
|
|
@ -223,7 +223,12 @@ export const useGeminiStream = (
|
|||
const trimmedQuery = query.trim();
|
||||
logUserPrompt(
|
||||
config,
|
||||
new UserPromptEvent(trimmedQuery.length, prompt_id, trimmedQuery),
|
||||
new UserPromptEvent(
|
||||
trimmedQuery.length,
|
||||
prompt_id,
|
||||
config.getContentGeneratorConfig().authType!,
|
||||
trimmedQuery,
|
||||
),
|
||||
);
|
||||
onDebugMessage(`User query: '${trimmedQuery}'`);
|
||||
await logger?.logMessage(MessageSenderType.USER, trimmedQuery);
|
||||
|
|
|
@ -172,6 +172,7 @@ export class GeminiChat {
|
|||
this.config.getModel(),
|
||||
durationMs,
|
||||
prompt_id,
|
||||
this.config.getContentGeneratorConfig().authType!,
|
||||
usageMetadata,
|
||||
responseText,
|
||||
),
|
||||
|
@ -193,6 +194,7 @@ export class GeminiChat {
|
|||
errorMessage,
|
||||
durationMs,
|
||||
prompt_id,
|
||||
this.config.getContentGeneratorConfig().authType!,
|
||||
errorType,
|
||||
),
|
||||
);
|
||||
|
|
|
@ -266,6 +266,10 @@ export class ClearcutLogger {
|
|||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID,
|
||||
value: JSON.stringify(event.prompt_id),
|
||||
},
|
||||
{
|
||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
|
||||
value: JSON.stringify(event.auth_type),
|
||||
},
|
||||
];
|
||||
|
||||
this.enqueueLogEvent(this.createLogEvent(new_prompt_event_name, data));
|
||||
|
@ -377,6 +381,10 @@ export class ClearcutLogger {
|
|||
EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT,
|
||||
value: JSON.stringify(event.tool_token_count),
|
||||
},
|
||||
{
|
||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
|
||||
value: JSON.stringify(event.auth_type),
|
||||
},
|
||||
];
|
||||
|
||||
this.enqueueLogEvent(this.createLogEvent(api_response_event_name, data));
|
||||
|
@ -407,6 +415,10 @@ export class ClearcutLogger {
|
|||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_ERROR_DURATION_MS,
|
||||
value: JSON.stringify(event.duration_ms),
|
||||
},
|
||||
{
|
||||
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
|
||||
value: JSON.stringify(event.auth_type),
|
||||
},
|
||||
];
|
||||
|
||||
this.enqueueLogEvent(this.createLogEvent(api_error_event_name, data));
|
||||
|
|
|
@ -144,6 +144,9 @@ export enum EventMetadataKey {
|
|||
|
||||
// Logs the Prompt Id
|
||||
GEMINI_CLI_PROMPT_ID = 35,
|
||||
|
||||
// Logs the Auth type for the prompt, api responses and errors.
|
||||
GEMINI_CLI_AUTH_TYPE = 36,
|
||||
}
|
||||
|
||||
export function getEventMetadataKey(
|
||||
|
|
|
@ -127,7 +127,12 @@ describe('loggers', () => {
|
|||
} as unknown as Config;
|
||||
|
||||
it('should log a user prompt', () => {
|
||||
const event = new UserPromptEvent(11, 'prompt-id-8', 'test-prompt');
|
||||
const event = new UserPromptEvent(
|
||||
11,
|
||||
'prompt-id-8',
|
||||
AuthType.USE_VERTEX_AI,
|
||||
'test-prompt',
|
||||
);
|
||||
|
||||
logUserPrompt(mockConfig, event);
|
||||
|
||||
|
@ -151,7 +156,11 @@ describe('loggers', () => {
|
|||
getTargetDir: () => 'target-dir',
|
||||
getUsageStatisticsEnabled: () => true,
|
||||
} as unknown as Config;
|
||||
const event = new UserPromptEvent(11, 'test-prompt');
|
||||
const event = new UserPromptEvent(
|
||||
11,
|
||||
'test-prompt',
|
||||
AuthType.CLOUD_SHELL,
|
||||
);
|
||||
|
||||
logUserPrompt(mockConfig, event);
|
||||
|
||||
|
@ -202,6 +211,7 @@ describe('loggers', () => {
|
|||
'test-model',
|
||||
100,
|
||||
'prompt-id-1',
|
||||
AuthType.LOGIN_WITH_GOOGLE,
|
||||
usageData,
|
||||
'test-response',
|
||||
);
|
||||
|
@ -226,6 +236,7 @@ describe('loggers', () => {
|
|||
total_token_count: 0,
|
||||
response_text: 'test-response',
|
||||
prompt_id: 'prompt-id-1',
|
||||
auth_type: 'oauth-personal',
|
||||
},
|
||||
});
|
||||
|
||||
|
@ -263,6 +274,7 @@ describe('loggers', () => {
|
|||
'test-model',
|
||||
100,
|
||||
'prompt-id-1',
|
||||
AuthType.USE_GEMINI,
|
||||
usageData,
|
||||
'test-response',
|
||||
'test-error',
|
||||
|
|
|
@ -96,13 +96,20 @@ export class UserPromptEvent {
|
|||
'event.timestamp': string; // ISO 8601
|
||||
prompt_length: number;
|
||||
prompt_id: string;
|
||||
auth_type: string;
|
||||
prompt?: string;
|
||||
|
||||
constructor(prompt_length: number, prompt_Id: string, prompt?: string) {
|
||||
constructor(
|
||||
prompt_length: number,
|
||||
prompt_Id: string,
|
||||
auth_type: string,
|
||||
prompt?: string,
|
||||
) {
|
||||
this['event.name'] = 'user_prompt';
|
||||
this['event.timestamp'] = new Date().toISOString();
|
||||
this.prompt_length = prompt_length;
|
||||
this.prompt_id = prompt_Id;
|
||||
this.auth_type = auth_type;
|
||||
this.prompt = prompt;
|
||||
}
|
||||
}
|
||||
|
@ -160,12 +167,14 @@ export class ApiErrorEvent {
|
|||
status_code?: number | string;
|
||||
duration_ms: number;
|
||||
prompt_id: string;
|
||||
auth_type: string;
|
||||
|
||||
constructor(
|
||||
model: string,
|
||||
error: string,
|
||||
duration_ms: number,
|
||||
prompt_id: string,
|
||||
auth_type: string,
|
||||
error_type?: string,
|
||||
status_code?: number | string,
|
||||
) {
|
||||
|
@ -177,6 +186,7 @@ export class ApiErrorEvent {
|
|||
this.status_code = status_code;
|
||||
this.duration_ms = duration_ms;
|
||||
this.prompt_id = prompt_id;
|
||||
this.auth_type = auth_type;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -195,11 +205,13 @@ export class ApiResponseEvent {
|
|||
total_token_count: number;
|
||||
response_text?: string;
|
||||
prompt_id: string;
|
||||
auth_type: string;
|
||||
|
||||
constructor(
|
||||
model: string,
|
||||
duration_ms: number,
|
||||
prompt_id: string,
|
||||
auth_type: string,
|
||||
usage_data?: GenerateContentResponseUsageMetadata,
|
||||
response_text?: string,
|
||||
error?: string,
|
||||
|
@ -218,6 +230,7 @@ export class ApiResponseEvent {
|
|||
this.response_text = response_text;
|
||||
this.error = error;
|
||||
this.prompt_id = prompt_id;
|
||||
this.auth_type = auth_type;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue