feat: Change /stats to include more detailed breakdowns (#2615)
This commit is contained in:
parent
0fd602eb43
commit
770f862832
|
@ -823,11 +823,7 @@ const App = ({ config, settings, startupWarnings = [] }: AppProps) => {
|
|||
showMemoryUsage={
|
||||
config.getDebugMode() || config.getShowMemoryUsage()
|
||||
}
|
||||
promptTokenCount={sessionStats.currentResponse.promptTokenCount}
|
||||
candidatesTokenCount={
|
||||
sessionStats.currentResponse.candidatesTokenCount
|
||||
}
|
||||
totalTokenCount={sessionStats.currentResponse.totalTokenCount}
|
||||
promptTokenCount={sessionStats.lastPromptTokenCount}
|
||||
/>
|
||||
</Box>
|
||||
</Box>
|
||||
|
|
|
@ -23,8 +23,6 @@ interface FooterProps {
|
|||
showErrorDetails: boolean;
|
||||
showMemoryUsage?: boolean;
|
||||
promptTokenCount: number;
|
||||
candidatesTokenCount: number;
|
||||
totalTokenCount: number;
|
||||
}
|
||||
|
||||
export const Footer: React.FC<FooterProps> = ({
|
||||
|
@ -37,10 +35,10 @@ export const Footer: React.FC<FooterProps> = ({
|
|||
errorCount,
|
||||
showErrorDetails,
|
||||
showMemoryUsage,
|
||||
totalTokenCount,
|
||||
promptTokenCount,
|
||||
}) => {
|
||||
const limit = tokenLimit(model);
|
||||
const percentage = totalTokenCount / limit;
|
||||
const percentage = promptTokenCount / limit;
|
||||
|
||||
return (
|
||||
<Box marginTop={1} justifyContent="space-between" width="100%">
|
||||
|
|
|
@ -8,7 +8,7 @@ import { render } from 'ink-testing-library';
|
|||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { HistoryItemDisplay } from './HistoryItemDisplay.js';
|
||||
import { HistoryItem, MessageType } from '../types.js';
|
||||
import { CumulativeStats } from '../contexts/SessionContext.js';
|
||||
import { SessionStatsProvider } from '../contexts/SessionContext.js';
|
||||
|
||||
// Mock child components
|
||||
vi.mock('./messages/ToolGroupMessage.js', () => ({
|
||||
|
@ -36,25 +36,15 @@ describe('<HistoryItemDisplay />', () => {
|
|||
});
|
||||
|
||||
it('renders StatsDisplay for "stats" type', () => {
|
||||
const stats: CumulativeStats = {
|
||||
turnCount: 1,
|
||||
promptTokenCount: 10,
|
||||
candidatesTokenCount: 20,
|
||||
totalTokenCount: 30,
|
||||
cachedContentTokenCount: 5,
|
||||
toolUsePromptTokenCount: 2,
|
||||
thoughtsTokenCount: 3,
|
||||
apiTimeMs: 123,
|
||||
};
|
||||
const item: HistoryItem = {
|
||||
...baseItem,
|
||||
type: MessageType.STATS,
|
||||
stats,
|
||||
lastTurnStats: stats,
|
||||
duration: '1s',
|
||||
};
|
||||
const { lastFrame } = render(
|
||||
<HistoryItemDisplay {...baseItem} item={item} />,
|
||||
<SessionStatsProvider>
|
||||
<HistoryItemDisplay {...baseItem} item={item} />
|
||||
</SessionStatsProvider>,
|
||||
);
|
||||
expect(lastFrame()).toContain('Stats');
|
||||
});
|
||||
|
@ -76,25 +66,46 @@ describe('<HistoryItemDisplay />', () => {
|
|||
expect(lastFrame()).toContain('About Gemini CLI');
|
||||
});
|
||||
|
||||
it('renders SessionSummaryDisplay for "quit" type', () => {
|
||||
const stats: CumulativeStats = {
|
||||
turnCount: 1,
|
||||
promptTokenCount: 10,
|
||||
candidatesTokenCount: 20,
|
||||
totalTokenCount: 30,
|
||||
cachedContentTokenCount: 5,
|
||||
toolUsePromptTokenCount: 2,
|
||||
thoughtsTokenCount: 3,
|
||||
apiTimeMs: 123,
|
||||
it('renders ModelStatsDisplay for "model_stats" type', () => {
|
||||
const item: HistoryItem = {
|
||||
...baseItem,
|
||||
type: 'model_stats',
|
||||
};
|
||||
const { lastFrame } = render(
|
||||
<SessionStatsProvider>
|
||||
<HistoryItemDisplay {...baseItem} item={item} />
|
||||
</SessionStatsProvider>,
|
||||
);
|
||||
expect(lastFrame()).toContain(
|
||||
'No API calls have been made in this session.',
|
||||
);
|
||||
});
|
||||
|
||||
it('renders ToolStatsDisplay for "tool_stats" type', () => {
|
||||
const item: HistoryItem = {
|
||||
...baseItem,
|
||||
type: 'tool_stats',
|
||||
};
|
||||
const { lastFrame } = render(
|
||||
<SessionStatsProvider>
|
||||
<HistoryItemDisplay {...baseItem} item={item} />
|
||||
</SessionStatsProvider>,
|
||||
);
|
||||
expect(lastFrame()).toContain(
|
||||
'No tool calls have been made in this session.',
|
||||
);
|
||||
});
|
||||
|
||||
it('renders SessionSummaryDisplay for "quit" type', () => {
|
||||
const item: HistoryItem = {
|
||||
...baseItem,
|
||||
type: 'quit',
|
||||
stats,
|
||||
duration: '1s',
|
||||
};
|
||||
const { lastFrame } = render(
|
||||
<HistoryItemDisplay {...baseItem} item={item} />,
|
||||
<SessionStatsProvider>
|
||||
<HistoryItemDisplay {...baseItem} item={item} />
|
||||
</SessionStatsProvider>,
|
||||
);
|
||||
expect(lastFrame()).toContain('Agent powering down. Goodbye!');
|
||||
});
|
||||
|
|
|
@ -17,6 +17,8 @@ import { CompressionMessage } from './messages/CompressionMessage.js';
|
|||
import { Box } from 'ink';
|
||||
import { AboutBox } from './AboutBox.js';
|
||||
import { StatsDisplay } from './StatsDisplay.js';
|
||||
import { ModelStatsDisplay } from './ModelStatsDisplay.js';
|
||||
import { ToolStatsDisplay } from './ToolStatsDisplay.js';
|
||||
import { SessionSummaryDisplay } from './SessionSummaryDisplay.js';
|
||||
import { Config } from '@google/gemini-cli-core';
|
||||
|
||||
|
@ -69,16 +71,10 @@ export const HistoryItemDisplay: React.FC<HistoryItemDisplayProps> = ({
|
|||
gcpProject={item.gcpProject}
|
||||
/>
|
||||
)}
|
||||
{item.type === 'stats' && (
|
||||
<StatsDisplay
|
||||
stats={item.stats}
|
||||
lastTurnStats={item.lastTurnStats}
|
||||
duration={item.duration}
|
||||
/>
|
||||
)}
|
||||
{item.type === 'quit' && (
|
||||
<SessionSummaryDisplay stats={item.stats} duration={item.duration} />
|
||||
)}
|
||||
{item.type === 'stats' && <StatsDisplay duration={item.duration} />}
|
||||
{item.type === 'model_stats' && <ModelStatsDisplay />}
|
||||
{item.type === 'tool_stats' && <ToolStatsDisplay />}
|
||||
{item.type === 'quit' && <SessionSummaryDisplay duration={item.duration} />}
|
||||
{item.type === 'tool_group' && (
|
||||
<ToolGroupMessage
|
||||
toolCalls={item.tools}
|
||||
|
|
|
@ -0,0 +1,235 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { render } from 'ink-testing-library';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { ModelStatsDisplay } from './ModelStatsDisplay.js';
|
||||
import * as SessionContext from '../contexts/SessionContext.js';
|
||||
import { SessionMetrics } from '../contexts/SessionContext.js';
|
||||
|
||||
// Mock the context to provide controlled data for testing
|
||||
vi.mock('../contexts/SessionContext.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof SessionContext>();
|
||||
return {
|
||||
...actual,
|
||||
useSessionStats: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const useSessionStatsMock = vi.mocked(SessionContext.useSessionStats);
|
||||
|
||||
const renderWithMockedStats = (metrics: SessionMetrics) => {
|
||||
useSessionStatsMock.mockReturnValue({
|
||||
stats: {
|
||||
sessionStartTime: new Date(),
|
||||
metrics,
|
||||
lastPromptTokenCount: 0,
|
||||
},
|
||||
});
|
||||
|
||||
return render(<ModelStatsDisplay />);
|
||||
};
|
||||
|
||||
describe('<ModelStatsDisplay />', () => {
|
||||
it('should render "no API calls" message when there are no active models', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
expect(lastFrame()).toContain(
|
||||
'No API calls have been made in this session.',
|
||||
);
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should not display conditional rows if no model has data for them', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 100 },
|
||||
tokens: {
|
||||
prompt: 10,
|
||||
candidates: 20,
|
||||
total: 30,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
const output = lastFrame();
|
||||
expect(output).not.toContain('Cached');
|
||||
expect(output).not.toContain('Thoughts');
|
||||
expect(output).not.toContain('Tool');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should display conditional rows if at least one model has data', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 100 },
|
||||
tokens: {
|
||||
prompt: 10,
|
||||
candidates: 20,
|
||||
total: 30,
|
||||
cached: 5,
|
||||
thoughts: 2,
|
||||
tool: 0,
|
||||
},
|
||||
},
|
||||
'gemini-2.5-flash': {
|
||||
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 50 },
|
||||
tokens: {
|
||||
prompt: 5,
|
||||
candidates: 10,
|
||||
total: 15,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 3,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
const output = lastFrame();
|
||||
expect(output).toContain('Cached');
|
||||
expect(output).toContain('Thoughts');
|
||||
expect(output).toContain('Tool');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should display stats for multiple models correctly', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 10, totalErrors: 1, totalLatencyMs: 1000 },
|
||||
tokens: {
|
||||
prompt: 100,
|
||||
candidates: 200,
|
||||
total: 300,
|
||||
cached: 50,
|
||||
thoughts: 10,
|
||||
tool: 5,
|
||||
},
|
||||
},
|
||||
'gemini-2.5-flash': {
|
||||
api: { totalRequests: 20, totalErrors: 2, totalLatencyMs: 500 },
|
||||
tokens: {
|
||||
prompt: 200,
|
||||
candidates: 400,
|
||||
total: 600,
|
||||
cached: 100,
|
||||
thoughts: 20,
|
||||
tool: 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
const output = lastFrame();
|
||||
expect(output).toContain('gemini-2.5-pro');
|
||||
expect(output).toContain('gemini-2.5-flash');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should handle large values without wrapping or overlapping', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: {
|
||||
totalRequests: 999999999,
|
||||
totalErrors: 123456789,
|
||||
totalLatencyMs: 9876,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 987654321,
|
||||
candidates: 123456789,
|
||||
total: 999999999,
|
||||
cached: 123456789,
|
||||
thoughts: 111111111,
|
||||
tool: 222222222,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should display a single model correctly', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 100 },
|
||||
tokens: {
|
||||
prompt: 10,
|
||||
candidates: 20,
|
||||
total: 30,
|
||||
cached: 5,
|
||||
thoughts: 2,
|
||||
tool: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
const output = lastFrame();
|
||||
expect(output).toContain('gemini-2.5-pro');
|
||||
expect(output).not.toContain('gemini-2.5-flash');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
});
|
|
@ -0,0 +1,197 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { Box, Text } from 'ink';
|
||||
import { Colors } from '../colors.js';
|
||||
import { formatDuration } from '../utils/formatters.js';
|
||||
import {
|
||||
calculateAverageLatency,
|
||||
calculateCacheHitRate,
|
||||
calculateErrorRate,
|
||||
} from '../utils/computeStats.js';
|
||||
import { useSessionStats, ModelMetrics } from '../contexts/SessionContext.js';
|
||||
|
||||
const METRIC_COL_WIDTH = 28;
|
||||
const MODEL_COL_WIDTH = 22;
|
||||
|
||||
interface StatRowProps {
|
||||
title: string;
|
||||
values: Array<string | React.ReactElement>;
|
||||
isSubtle?: boolean;
|
||||
isSection?: boolean;
|
||||
}
|
||||
|
||||
const StatRow: React.FC<StatRowProps> = ({
|
||||
title,
|
||||
values,
|
||||
isSubtle = false,
|
||||
isSection = false,
|
||||
}) => (
|
||||
<Box>
|
||||
<Box width={METRIC_COL_WIDTH}>
|
||||
<Text bold={isSection} color={isSection ? undefined : Colors.LightBlue}>
|
||||
{isSubtle ? ` ↳ ${title}` : title}
|
||||
</Text>
|
||||
</Box>
|
||||
{values.map((value, index) => (
|
||||
<Box width={MODEL_COL_WIDTH} key={index}>
|
||||
<Text>{value}</Text>
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
);
|
||||
|
||||
export const ModelStatsDisplay: React.FC = () => {
|
||||
const { stats } = useSessionStats();
|
||||
const { models } = stats.metrics;
|
||||
const activeModels = Object.entries(models).filter(
|
||||
([, metrics]) => metrics.api.totalRequests > 0,
|
||||
);
|
||||
|
||||
if (activeModels.length === 0) {
|
||||
return (
|
||||
<Box
|
||||
borderStyle="round"
|
||||
borderColor={Colors.Gray}
|
||||
paddingY={1}
|
||||
paddingX={2}
|
||||
>
|
||||
<Text>No API calls have been made in this session.</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
const modelNames = activeModels.map(([name]) => name);
|
||||
|
||||
const getModelValues = (
|
||||
getter: (metrics: ModelMetrics) => string | React.ReactElement,
|
||||
) => activeModels.map(([, metrics]) => getter(metrics));
|
||||
|
||||
const hasThoughts = activeModels.some(
|
||||
([, metrics]) => metrics.tokens.thoughts > 0,
|
||||
);
|
||||
const hasTool = activeModels.some(([, metrics]) => metrics.tokens.tool > 0);
|
||||
const hasCached = activeModels.some(
|
||||
([, metrics]) => metrics.tokens.cached > 0,
|
||||
);
|
||||
|
||||
return (
|
||||
<Box
|
||||
borderStyle="round"
|
||||
borderColor={Colors.Gray}
|
||||
flexDirection="column"
|
||||
paddingY={1}
|
||||
paddingX={2}
|
||||
>
|
||||
<Text bold color={Colors.AccentPurple}>
|
||||
Model Stats For Nerds
|
||||
</Text>
|
||||
<Box height={1} />
|
||||
|
||||
{/* Header */}
|
||||
<Box>
|
||||
<Box width={METRIC_COL_WIDTH}>
|
||||
<Text bold>Metric</Text>
|
||||
</Box>
|
||||
{modelNames.map((name) => (
|
||||
<Box width={MODEL_COL_WIDTH} key={name}>
|
||||
<Text bold>{name}</Text>
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
|
||||
{/* Divider */}
|
||||
<Box
|
||||
borderStyle="single"
|
||||
borderBottom={true}
|
||||
borderTop={false}
|
||||
borderLeft={false}
|
||||
borderRight={false}
|
||||
/>
|
||||
|
||||
{/* API Section */}
|
||||
<StatRow title="API" values={[]} isSection />
|
||||
<StatRow
|
||||
title="Requests"
|
||||
values={getModelValues((m) => m.api.totalRequests.toLocaleString())}
|
||||
/>
|
||||
<StatRow
|
||||
title="Errors"
|
||||
values={getModelValues((m) => {
|
||||
const errorRate = calculateErrorRate(m);
|
||||
return (
|
||||
<Text
|
||||
color={
|
||||
m.api.totalErrors > 0 ? Colors.AccentRed : Colors.Foreground
|
||||
}
|
||||
>
|
||||
{m.api.totalErrors.toLocaleString()} ({errorRate.toFixed(1)}%)
|
||||
</Text>
|
||||
);
|
||||
})}
|
||||
/>
|
||||
<StatRow
|
||||
title="Avg Latency"
|
||||
values={getModelValues((m) => {
|
||||
const avgLatency = calculateAverageLatency(m);
|
||||
return formatDuration(avgLatency);
|
||||
})}
|
||||
/>
|
||||
|
||||
<Box height={1} />
|
||||
|
||||
{/* Tokens Section */}
|
||||
<StatRow title="Tokens" values={[]} isSection />
|
||||
<StatRow
|
||||
title="Total"
|
||||
values={getModelValues((m) => (
|
||||
<Text color={Colors.AccentYellow}>
|
||||
{m.tokens.total.toLocaleString()}
|
||||
</Text>
|
||||
))}
|
||||
/>
|
||||
<StatRow
|
||||
title="Prompt"
|
||||
isSubtle
|
||||
values={getModelValues((m) => m.tokens.prompt.toLocaleString())}
|
||||
/>
|
||||
{hasCached && (
|
||||
<StatRow
|
||||
title="Cached"
|
||||
isSubtle
|
||||
values={getModelValues((m) => {
|
||||
const cacheHitRate = calculateCacheHitRate(m);
|
||||
return (
|
||||
<Text color={Colors.AccentGreen}>
|
||||
{m.tokens.cached.toLocaleString()} ({cacheHitRate.toFixed(1)}%)
|
||||
</Text>
|
||||
);
|
||||
})}
|
||||
/>
|
||||
)}
|
||||
{hasThoughts && (
|
||||
<StatRow
|
||||
title="Thoughts"
|
||||
isSubtle
|
||||
values={getModelValues((m) => m.tokens.thoughts.toLocaleString())}
|
||||
/>
|
||||
)}
|
||||
{hasTool && (
|
||||
<StatRow
|
||||
title="Tool"
|
||||
isSubtle
|
||||
values={getModelValues((m) => m.tokens.tool.toLocaleString())}
|
||||
/>
|
||||
)}
|
||||
<StatRow
|
||||
title="Output"
|
||||
isSubtle
|
||||
values={getModelValues((m) => m.tokens.candidates.toLocaleString())}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
};
|
|
@ -5,48 +5,92 @@
|
|||
*/
|
||||
|
||||
import { render } from 'ink-testing-library';
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { SessionSummaryDisplay } from './SessionSummaryDisplay.js';
|
||||
import { type CumulativeStats } from '../contexts/SessionContext.js';
|
||||
import * as SessionContext from '../contexts/SessionContext.js';
|
||||
import { SessionMetrics } from '../contexts/SessionContext.js';
|
||||
|
||||
vi.mock('../contexts/SessionContext.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof SessionContext>();
|
||||
return {
|
||||
...actual,
|
||||
useSessionStats: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const useSessionStatsMock = vi.mocked(SessionContext.useSessionStats);
|
||||
|
||||
const renderWithMockedStats = (metrics: SessionMetrics) => {
|
||||
useSessionStatsMock.mockReturnValue({
|
||||
stats: {
|
||||
sessionStartTime: new Date(),
|
||||
metrics,
|
||||
lastPromptTokenCount: 0,
|
||||
},
|
||||
});
|
||||
|
||||
return render(<SessionSummaryDisplay duration="1h 23m 45s" />);
|
||||
};
|
||||
|
||||
describe('<SessionSummaryDisplay />', () => {
|
||||
const mockStats: CumulativeStats = {
|
||||
turnCount: 10,
|
||||
promptTokenCount: 1000,
|
||||
candidatesTokenCount: 2000,
|
||||
totalTokenCount: 3500,
|
||||
cachedContentTokenCount: 500,
|
||||
toolUsePromptTokenCount: 200,
|
||||
thoughtsTokenCount: 300,
|
||||
apiTimeMs: 50234,
|
||||
};
|
||||
it('correctly sums and displays stats from multiple models', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 10, totalErrors: 1, totalLatencyMs: 50234 },
|
||||
tokens: {
|
||||
prompt: 1000,
|
||||
candidates: 2000,
|
||||
total: 3500,
|
||||
cached: 500,
|
||||
thoughts: 300,
|
||||
tool: 200,
|
||||
},
|
||||
},
|
||||
'gemini-2.5-flash': {
|
||||
api: { totalRequests: 5, totalErrors: 0, totalLatencyMs: 12345 },
|
||||
tokens: {
|
||||
prompt: 500,
|
||||
candidates: 1000,
|
||||
total: 1500,
|
||||
cached: 100,
|
||||
thoughts: 50,
|
||||
tool: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const mockDuration = '1h 23m 45s';
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
const output = lastFrame();
|
||||
|
||||
it('renders correctly with given stats and duration', () => {
|
||||
const { lastFrame } = render(
|
||||
<SessionSummaryDisplay stats={mockStats} duration={mockDuration} />,
|
||||
);
|
||||
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
// Verify totals are summed correctly
|
||||
expect(output).toContain('Cumulative Stats (15 API calls)');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('renders zero state correctly', () => {
|
||||
const zeroStats: CumulativeStats = {
|
||||
turnCount: 0,
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
const zeroMetrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const { lastFrame } = render(
|
||||
<SessionSummaryDisplay stats={zeroStats} duration="0s" />,
|
||||
);
|
||||
|
||||
const { lastFrame } = renderWithMockedStats(zeroMetrics);
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
|
|
@ -9,31 +9,57 @@ import { Box, Text } from 'ink';
|
|||
import Gradient from 'ink-gradient';
|
||||
import { Colors } from '../colors.js';
|
||||
import { formatDuration } from '../utils/formatters.js';
|
||||
import { CumulativeStats } from '../contexts/SessionContext.js';
|
||||
import { useSessionStats } from '../contexts/SessionContext.js';
|
||||
import { computeSessionStats } from '../utils/computeStats.js';
|
||||
import { FormattedStats, StatRow, StatsColumn } from './Stats.js';
|
||||
|
||||
// --- Prop and Data Structures ---
|
||||
|
||||
interface SessionSummaryDisplayProps {
|
||||
stats: CumulativeStats;
|
||||
duration: string;
|
||||
}
|
||||
|
||||
// --- Main Component ---
|
||||
|
||||
export const SessionSummaryDisplay: React.FC<SessionSummaryDisplayProps> = ({
|
||||
stats,
|
||||
duration,
|
||||
}) => {
|
||||
const { stats } = useSessionStats();
|
||||
const { metrics } = stats;
|
||||
const computed = computeSessionStats(metrics);
|
||||
|
||||
const cumulativeFormatted: FormattedStats = {
|
||||
inputTokens: stats.promptTokenCount,
|
||||
outputTokens: stats.candidatesTokenCount,
|
||||
toolUseTokens: stats.toolUsePromptTokenCount,
|
||||
thoughtsTokens: stats.thoughtsTokenCount,
|
||||
cachedTokens: stats.cachedContentTokenCount,
|
||||
totalTokens: stats.totalTokenCount,
|
||||
inputTokens: Object.values(metrics.models).reduce(
|
||||
(acc, model) => acc + model.tokens.prompt,
|
||||
0,
|
||||
),
|
||||
outputTokens: Object.values(metrics.models).reduce(
|
||||
(acc, model) => acc + model.tokens.candidates,
|
||||
0,
|
||||
),
|
||||
toolUseTokens: Object.values(metrics.models).reduce(
|
||||
(acc, model) => acc + model.tokens.tool,
|
||||
0,
|
||||
),
|
||||
thoughtsTokens: Object.values(metrics.models).reduce(
|
||||
(acc, model) => acc + model.tokens.thoughts,
|
||||
0,
|
||||
),
|
||||
cachedTokens: Object.values(metrics.models).reduce(
|
||||
(acc, model) => acc + model.tokens.cached,
|
||||
0,
|
||||
),
|
||||
totalTokens: Object.values(metrics.models).reduce(
|
||||
(acc, model) => acc + model.tokens.total,
|
||||
0,
|
||||
),
|
||||
};
|
||||
|
||||
const totalRequests = Object.values(metrics.models).reduce(
|
||||
(acc, model) => acc + model.api.totalRequests,
|
||||
0,
|
||||
);
|
||||
|
||||
const title = 'Agent powering down. Goodbye!';
|
||||
|
||||
return (
|
||||
|
@ -57,14 +83,18 @@ export const SessionSummaryDisplay: React.FC<SessionSummaryDisplayProps> = ({
|
|||
|
||||
<Box marginTop={1}>
|
||||
<StatsColumn
|
||||
title={`Cumulative Stats (${stats.turnCount} Turns)`}
|
||||
title={`Cumulative Stats (${totalRequests} API calls)`}
|
||||
stats={cumulativeFormatted}
|
||||
isCumulative={true}
|
||||
>
|
||||
<Box marginTop={1} flexDirection="column">
|
||||
<StatRow
|
||||
label="Total duration (API)"
|
||||
value={formatDuration(stats.apiTimeMs)}
|
||||
value={formatDuration(computed.totalApiTime)}
|
||||
/>
|
||||
<StatRow
|
||||
label="Total duration (Tools)"
|
||||
value={formatDuration(computed.totalToolTime)}
|
||||
/>
|
||||
<StatRow label="Total duration (wall)" value={duration} />
|
||||
</Box>
|
||||
|
|
|
@ -5,67 +5,259 @@
|
|||
*/
|
||||
|
||||
import { render } from 'ink-testing-library';
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { StatsDisplay } from './StatsDisplay.js';
|
||||
import { type CumulativeStats } from '../contexts/SessionContext.js';
|
||||
import * as SessionContext from '../contexts/SessionContext.js';
|
||||
import { SessionMetrics } from '../contexts/SessionContext.js';
|
||||
|
||||
// Mock the context to provide controlled data for testing
|
||||
vi.mock('../contexts/SessionContext.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof SessionContext>();
|
||||
return {
|
||||
...actual,
|
||||
useSessionStats: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const useSessionStatsMock = vi.mocked(SessionContext.useSessionStats);
|
||||
|
||||
const renderWithMockedStats = (metrics: SessionMetrics) => {
|
||||
useSessionStatsMock.mockReturnValue({
|
||||
stats: {
|
||||
sessionStartTime: new Date(),
|
||||
metrics,
|
||||
lastPromptTokenCount: 0,
|
||||
},
|
||||
});
|
||||
|
||||
return render(<StatsDisplay duration="1s" />);
|
||||
};
|
||||
|
||||
describe('<StatsDisplay />', () => {
|
||||
const mockStats: CumulativeStats = {
|
||||
turnCount: 10,
|
||||
promptTokenCount: 1000,
|
||||
candidatesTokenCount: 2000,
|
||||
totalTokenCount: 3500,
|
||||
cachedContentTokenCount: 500,
|
||||
toolUsePromptTokenCount: 200,
|
||||
thoughtsTokenCount: 300,
|
||||
apiTimeMs: 50234,
|
||||
};
|
||||
|
||||
const mockLastTurnStats: CumulativeStats = {
|
||||
turnCount: 1,
|
||||
promptTokenCount: 100,
|
||||
candidatesTokenCount: 200,
|
||||
totalTokenCount: 350,
|
||||
cachedContentTokenCount: 50,
|
||||
toolUsePromptTokenCount: 20,
|
||||
thoughtsTokenCount: 30,
|
||||
apiTimeMs: 1234,
|
||||
};
|
||||
|
||||
const mockDuration = '1h 23m 45s';
|
||||
|
||||
it('renders correctly with given stats and duration', () => {
|
||||
const { lastFrame } = render(
|
||||
<StatsDisplay
|
||||
stats={mockStats}
|
||||
lastTurnStats={mockLastTurnStats}
|
||||
duration={mockDuration}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('renders zero state correctly', () => {
|
||||
const zeroStats: CumulativeStats = {
|
||||
turnCount: 0,
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
it('renders only the Performance section in its zero state', () => {
|
||||
const zeroMetrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const { lastFrame } = render(
|
||||
<StatsDisplay
|
||||
stats={zeroStats}
|
||||
lastTurnStats={zeroStats}
|
||||
duration="0s"
|
||||
/>,
|
||||
);
|
||||
const { lastFrame } = renderWithMockedStats(zeroMetrics);
|
||||
const output = lastFrame();
|
||||
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
expect(output).toContain('Performance');
|
||||
expect(output).not.toContain('Interaction Summary');
|
||||
expect(output).not.toContain('Efficiency & Optimizations');
|
||||
expect(output).not.toContain('Model'); // The table header
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('renders a table with two models correctly', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 3, totalErrors: 0, totalLatencyMs: 15000 },
|
||||
tokens: {
|
||||
prompt: 1000,
|
||||
candidates: 2000,
|
||||
total: 43234,
|
||||
cached: 500,
|
||||
thoughts: 100,
|
||||
tool: 50,
|
||||
},
|
||||
},
|
||||
'gemini-2.5-flash': {
|
||||
api: { totalRequests: 5, totalErrors: 1, totalLatencyMs: 4500 },
|
||||
tokens: {
|
||||
prompt: 25000,
|
||||
candidates: 15000,
|
||||
total: 150000000,
|
||||
cached: 10000,
|
||||
thoughts: 2000,
|
||||
tool: 1000,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
const output = lastFrame();
|
||||
|
||||
expect(output).toContain('gemini-2.5-pro');
|
||||
expect(output).toContain('gemini-2.5-flash');
|
||||
expect(output).toContain('1,000');
|
||||
expect(output).toContain('25,000');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('renders all sections when all data is present', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 100 },
|
||||
tokens: {
|
||||
prompt: 100,
|
||||
candidates: 100,
|
||||
total: 250,
|
||||
cached: 50,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 2,
|
||||
totalSuccess: 1,
|
||||
totalFail: 1,
|
||||
totalDurationMs: 123,
|
||||
totalDecisions: { accept: 1, reject: 0, modify: 0 },
|
||||
byName: {
|
||||
'test-tool': {
|
||||
count: 2,
|
||||
success: 1,
|
||||
fail: 1,
|
||||
durationMs: 123,
|
||||
decisions: { accept: 1, reject: 0, modify: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
const output = lastFrame();
|
||||
|
||||
expect(output).toContain('Performance');
|
||||
expect(output).toContain('Interaction Summary');
|
||||
expect(output).toContain('User Agreement');
|
||||
expect(output).toContain('Savings Highlight');
|
||||
expect(output).toContain('gemini-2.5-pro');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
describe('Conditional Rendering Tests', () => {
|
||||
it('hides User Agreement when no decisions are made', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 2,
|
||||
totalSuccess: 1,
|
||||
totalFail: 1,
|
||||
totalDurationMs: 123,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 }, // No decisions
|
||||
byName: {
|
||||
'test-tool': {
|
||||
count: 2,
|
||||
success: 1,
|
||||
fail: 1,
|
||||
durationMs: 123,
|
||||
decisions: { accept: 0, reject: 0, modify: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
const output = lastFrame();
|
||||
|
||||
expect(output).toContain('Interaction Summary');
|
||||
expect(output).toContain('Success Rate');
|
||||
expect(output).not.toContain('User Agreement');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('hides Efficiency section when cache is not used', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 100 },
|
||||
tokens: {
|
||||
prompt: 100,
|
||||
candidates: 100,
|
||||
total: 200,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
const output = lastFrame();
|
||||
|
||||
expect(output).not.toContain('Efficiency & Optimizations');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Conditional Color Tests', () => {
|
||||
it('renders success rate in green for high values', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 10,
|
||||
totalSuccess: 10,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('renders success rate in yellow for medium values', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 10,
|
||||
totalSuccess: 9,
|
||||
totalFail: 1,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('renders success rate in red for low values', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 10,
|
||||
totalSuccess: 5,
|
||||
totalFail: 5,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
const { lastFrame } = renderWithMockedStats(metrics);
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -8,90 +8,230 @@ import React from 'react';
|
|||
import { Box, Text } from 'ink';
|
||||
import { Colors } from '../colors.js';
|
||||
import { formatDuration } from '../utils/formatters.js';
|
||||
import { CumulativeStats } from '../contexts/SessionContext.js';
|
||||
import { FormattedStats, StatRow, StatsColumn } from './Stats.js';
|
||||
import { useSessionStats, ModelMetrics } from '../contexts/SessionContext.js';
|
||||
import {
|
||||
getStatusColor,
|
||||
TOOL_SUCCESS_RATE_HIGH,
|
||||
TOOL_SUCCESS_RATE_MEDIUM,
|
||||
USER_AGREEMENT_RATE_HIGH,
|
||||
USER_AGREEMENT_RATE_MEDIUM,
|
||||
} from '../utils/displayUtils.js';
|
||||
import { computeSessionStats } from '../utils/computeStats.js';
|
||||
|
||||
// --- Constants ---
|
||||
// A more flexible and powerful StatRow component
|
||||
interface StatRowProps {
|
||||
title: string;
|
||||
children: React.ReactNode; // Use children to allow for complex, colored values
|
||||
}
|
||||
|
||||
const COLUMN_WIDTH = '48%';
|
||||
const StatRow: React.FC<StatRowProps> = ({ title, children }) => (
|
||||
<Box>
|
||||
{/* Fixed width for the label creates a clean "gutter" for alignment */}
|
||||
<Box width={28}>
|
||||
<Text color={Colors.LightBlue}>{title}</Text>
|
||||
</Box>
|
||||
{children}
|
||||
</Box>
|
||||
);
|
||||
|
||||
// --- Prop and Data Structures ---
|
||||
// A SubStatRow for indented, secondary information
|
||||
interface SubStatRowProps {
|
||||
title: string;
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
const SubStatRow: React.FC<SubStatRowProps> = ({ title, children }) => (
|
||||
<Box paddingLeft={2}>
|
||||
{/* Adjust width for the "» " prefix */}
|
||||
<Box width={26}>
|
||||
<Text>» {title}</Text>
|
||||
</Box>
|
||||
{children}
|
||||
</Box>
|
||||
);
|
||||
|
||||
// A Section component to group related stats
|
||||
interface SectionProps {
|
||||
title: string;
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
const Section: React.FC<SectionProps> = ({ title, children }) => (
|
||||
<Box flexDirection="column" width="100%" marginBottom={1}>
|
||||
<Text bold>{title}</Text>
|
||||
{children}
|
||||
</Box>
|
||||
);
|
||||
|
||||
const ModelUsageTable: React.FC<{
|
||||
models: Record<string, ModelMetrics>;
|
||||
totalCachedTokens: number;
|
||||
cacheEfficiency: number;
|
||||
}> = ({ models, totalCachedTokens, cacheEfficiency }) => {
|
||||
const nameWidth = 25;
|
||||
const requestsWidth = 8;
|
||||
const inputTokensWidth = 15;
|
||||
const outputTokensWidth = 15;
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
{/* Header */}
|
||||
<Box>
|
||||
<Box width={nameWidth}>
|
||||
<Text bold>Model Usage</Text>
|
||||
</Box>
|
||||
<Box width={requestsWidth} justifyContent="flex-end">
|
||||
<Text bold>Reqs</Text>
|
||||
</Box>
|
||||
<Box width={inputTokensWidth} justifyContent="flex-end">
|
||||
<Text bold>Input Tokens</Text>
|
||||
</Box>
|
||||
<Box width={outputTokensWidth} justifyContent="flex-end">
|
||||
<Text bold>Output Tokens</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
{/* Divider */}
|
||||
<Box
|
||||
borderStyle="round"
|
||||
borderBottom={true}
|
||||
borderTop={false}
|
||||
borderLeft={false}
|
||||
borderRight={false}
|
||||
width={nameWidth + requestsWidth + inputTokensWidth + outputTokensWidth}
|
||||
></Box>
|
||||
|
||||
{/* Rows */}
|
||||
{Object.entries(models).map(([name, modelMetrics]) => (
|
||||
<Box key={name}>
|
||||
<Box width={nameWidth}>
|
||||
<Text>{name.replace('-001', '')}</Text>
|
||||
</Box>
|
||||
<Box width={requestsWidth} justifyContent="flex-end">
|
||||
<Text>{modelMetrics.api.totalRequests}</Text>
|
||||
</Box>
|
||||
<Box width={inputTokensWidth} justifyContent="flex-end">
|
||||
<Text color={Colors.AccentYellow}>
|
||||
{modelMetrics.tokens.prompt.toLocaleString()}
|
||||
</Text>
|
||||
</Box>
|
||||
<Box width={outputTokensWidth} justifyContent="flex-end">
|
||||
<Text color={Colors.AccentYellow}>
|
||||
{modelMetrics.tokens.candidates.toLocaleString()}
|
||||
</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
))}
|
||||
{cacheEfficiency > 0 && (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Text>
|
||||
<Text color={Colors.AccentGreen}>Savings Highlight:</Text>{' '}
|
||||
{totalCachedTokens.toLocaleString()} ({cacheEfficiency.toFixed(1)}
|
||||
%) of input tokens were served from the cache, reducing costs.
|
||||
</Text>
|
||||
<Box height={1} />
|
||||
<Text color={Colors.Gray}>
|
||||
» Tip: For a full token breakdown, run `/stats model`.
|
||||
</Text>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
interface StatsDisplayProps {
|
||||
stats: CumulativeStats;
|
||||
lastTurnStats: CumulativeStats;
|
||||
duration: string;
|
||||
}
|
||||
|
||||
// --- Main Component ---
|
||||
export const StatsDisplay: React.FC<StatsDisplayProps> = ({ duration }) => {
|
||||
const { stats } = useSessionStats();
|
||||
const { metrics } = stats;
|
||||
const { models, tools } = metrics;
|
||||
const computed = computeSessionStats(metrics);
|
||||
|
||||
export const StatsDisplay: React.FC<StatsDisplayProps> = ({
|
||||
stats,
|
||||
lastTurnStats,
|
||||
duration,
|
||||
}) => {
|
||||
const lastTurnFormatted: FormattedStats = {
|
||||
inputTokens: lastTurnStats.promptTokenCount,
|
||||
outputTokens: lastTurnStats.candidatesTokenCount,
|
||||
toolUseTokens: lastTurnStats.toolUsePromptTokenCount,
|
||||
thoughtsTokens: lastTurnStats.thoughtsTokenCount,
|
||||
cachedTokens: lastTurnStats.cachedContentTokenCount,
|
||||
totalTokens: lastTurnStats.totalTokenCount,
|
||||
const successThresholds = {
|
||||
green: TOOL_SUCCESS_RATE_HIGH,
|
||||
yellow: TOOL_SUCCESS_RATE_MEDIUM,
|
||||
};
|
||||
|
||||
const cumulativeFormatted: FormattedStats = {
|
||||
inputTokens: stats.promptTokenCount,
|
||||
outputTokens: stats.candidatesTokenCount,
|
||||
toolUseTokens: stats.toolUsePromptTokenCount,
|
||||
thoughtsTokens: stats.thoughtsTokenCount,
|
||||
cachedTokens: stats.cachedContentTokenCount,
|
||||
totalTokens: stats.totalTokenCount,
|
||||
const agreementThresholds = {
|
||||
green: USER_AGREEMENT_RATE_HIGH,
|
||||
yellow: USER_AGREEMENT_RATE_MEDIUM,
|
||||
};
|
||||
const successColor = getStatusColor(computed.successRate, successThresholds);
|
||||
const agreementColor = getStatusColor(
|
||||
computed.agreementRate,
|
||||
agreementThresholds,
|
||||
);
|
||||
|
||||
return (
|
||||
<Box
|
||||
borderStyle="round"
|
||||
borderColor="gray"
|
||||
borderColor={Colors.Gray}
|
||||
flexDirection="column"
|
||||
paddingY={1}
|
||||
paddingX={2}
|
||||
>
|
||||
<Text bold color={Colors.AccentPurple}>
|
||||
Stats
|
||||
Session Stats
|
||||
</Text>
|
||||
<Box height={1} />
|
||||
|
||||
<Box flexDirection="row" justifyContent="space-between" marginTop={1}>
|
||||
<StatsColumn
|
||||
title="Last Turn"
|
||||
stats={lastTurnFormatted}
|
||||
width={COLUMN_WIDTH}
|
||||
{tools.totalCalls > 0 && (
|
||||
<Section title="Interaction Summary">
|
||||
<StatRow title="Tool Calls:">
|
||||
<Text>
|
||||
{tools.totalCalls} ({' '}
|
||||
<Text color={Colors.AccentGreen}>✔ {tools.totalSuccess}</Text>{' '}
|
||||
<Text color={Colors.AccentRed}>✖ {tools.totalFail}</Text> )
|
||||
</Text>
|
||||
</StatRow>
|
||||
<StatRow title="Success Rate:">
|
||||
<Text color={successColor}>{computed.successRate.toFixed(1)}%</Text>
|
||||
</StatRow>
|
||||
{computed.totalDecisions > 0 && (
|
||||
<StatRow title="User Agreement:">
|
||||
<Text color={agreementColor}>
|
||||
{computed.agreementRate.toFixed(1)}%{' '}
|
||||
<Text color={Colors.Gray}>
|
||||
({computed.totalDecisions} reviewed)
|
||||
</Text>
|
||||
</Text>
|
||||
</StatRow>
|
||||
)}
|
||||
</Section>
|
||||
)}
|
||||
|
||||
<Section title="Performance">
|
||||
<StatRow title="Wall Time:">
|
||||
<Text>{duration}</Text>
|
||||
</StatRow>
|
||||
<StatRow title="Agent Active:">
|
||||
<Text>{formatDuration(computed.agentActiveTime)}</Text>
|
||||
</StatRow>
|
||||
<SubStatRow title="API Time:">
|
||||
<Text>
|
||||
{formatDuration(computed.totalApiTime)}{' '}
|
||||
<Text color={Colors.Gray}>
|
||||
({computed.apiTimePercent.toFixed(1)}%)
|
||||
</Text>
|
||||
</Text>
|
||||
</SubStatRow>
|
||||
<SubStatRow title="Tool Time:">
|
||||
<Text>
|
||||
{formatDuration(computed.totalToolTime)}{' '}
|
||||
<Text color={Colors.Gray}>
|
||||
({computed.toolTimePercent.toFixed(1)}%)
|
||||
</Text>
|
||||
</Text>
|
||||
</SubStatRow>
|
||||
</Section>
|
||||
|
||||
{Object.keys(models).length > 0 && (
|
||||
<ModelUsageTable
|
||||
models={models}
|
||||
totalCachedTokens={computed.totalCachedTokens}
|
||||
cacheEfficiency={computed.cacheEfficiency}
|
||||
/>
|
||||
<StatsColumn
|
||||
title={`Cumulative (${stats.turnCount} Turns)`}
|
||||
stats={cumulativeFormatted}
|
||||
isCumulative={true}
|
||||
width={COLUMN_WIDTH}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Box flexDirection="row" justifyContent="space-between" marginTop={1}>
|
||||
{/* Left column for "Last Turn" duration */}
|
||||
<Box width={COLUMN_WIDTH} flexDirection="column">
|
||||
<StatRow
|
||||
label="Turn Duration (API)"
|
||||
value={formatDuration(lastTurnStats.apiTimeMs)}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
{/* Right column for "Cumulative" durations */}
|
||||
<Box width={COLUMN_WIDTH} flexDirection="column">
|
||||
<StatRow
|
||||
label="Total duration (API)"
|
||||
value={formatDuration(stats.apiTimeMs)}
|
||||
/>
|
||||
<StatRow label="Total duration (wall)" value={duration} />
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
|
|
@ -0,0 +1,176 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { render } from 'ink-testing-library';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { ToolStatsDisplay } from './ToolStatsDisplay.js';
|
||||
import * as SessionContext from '../contexts/SessionContext.js';
|
||||
import { SessionMetrics } from '../contexts/SessionContext.js';
|
||||
|
||||
// Mock the context to provide controlled data for testing
|
||||
vi.mock('../contexts/SessionContext.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof SessionContext>();
|
||||
return {
|
||||
...actual,
|
||||
useSessionStats: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const useSessionStatsMock = vi.mocked(SessionContext.useSessionStats);
|
||||
|
||||
const renderWithMockedStats = (metrics: SessionMetrics) => {
|
||||
useSessionStatsMock.mockReturnValue({
|
||||
stats: {
|
||||
sessionStartTime: new Date(),
|
||||
metrics,
|
||||
lastPromptTokenCount: 0,
|
||||
},
|
||||
});
|
||||
|
||||
return render(<ToolStatsDisplay />);
|
||||
};
|
||||
|
||||
describe('<ToolStatsDisplay />', () => {
|
||||
it('should render "no tool calls" message when there are no active tools', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
expect(lastFrame()).toContain(
|
||||
'No tool calls have been made in this session.',
|
||||
);
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should display stats for a single tool correctly', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 1,
|
||||
totalSuccess: 1,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 100,
|
||||
totalDecisions: { accept: 1, reject: 0, modify: 0 },
|
||||
byName: {
|
||||
'test-tool': {
|
||||
count: 1,
|
||||
success: 1,
|
||||
fail: 0,
|
||||
durationMs: 100,
|
||||
decisions: { accept: 1, reject: 0, modify: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const output = lastFrame();
|
||||
expect(output).toContain('test-tool');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should display stats for multiple tools correctly', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 3,
|
||||
totalSuccess: 2,
|
||||
totalFail: 1,
|
||||
totalDurationMs: 300,
|
||||
totalDecisions: { accept: 1, reject: 1, modify: 1 },
|
||||
byName: {
|
||||
'tool-a': {
|
||||
count: 2,
|
||||
success: 1,
|
||||
fail: 1,
|
||||
durationMs: 200,
|
||||
decisions: { accept: 1, reject: 1, modify: 0 },
|
||||
},
|
||||
'tool-b': {
|
||||
count: 1,
|
||||
success: 1,
|
||||
fail: 0,
|
||||
durationMs: 100,
|
||||
decisions: { accept: 0, reject: 0, modify: 1 },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const output = lastFrame();
|
||||
expect(output).toContain('tool-a');
|
||||
expect(output).toContain('tool-b');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should handle large values without wrapping or overlapping', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 999999999,
|
||||
totalSuccess: 888888888,
|
||||
totalFail: 111111111,
|
||||
totalDurationMs: 987654321,
|
||||
totalDecisions: {
|
||||
accept: 123456789,
|
||||
reject: 98765432,
|
||||
modify: 12345,
|
||||
},
|
||||
byName: {
|
||||
'long-named-tool-for-testing-wrapping-and-such': {
|
||||
count: 999999999,
|
||||
success: 888888888,
|
||||
fail: 111111111,
|
||||
durationMs: 987654321,
|
||||
decisions: {
|
||||
accept: 123456789,
|
||||
reject: 98765432,
|
||||
modify: 12345,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(lastFrame()).toMatchSnapshot();
|
||||
});
|
||||
|
||||
it('should handle zero decisions gracefully', () => {
|
||||
const { lastFrame } = renderWithMockedStats({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 1,
|
||||
totalSuccess: 1,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 100,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {
|
||||
'test-tool': {
|
||||
count: 1,
|
||||
success: 1,
|
||||
fail: 0,
|
||||
durationMs: 100,
|
||||
decisions: { accept: 0, reject: 0, modify: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const output = lastFrame();
|
||||
expect(output).toContain('Total Reviewed Suggestions:');
|
||||
expect(output).toContain('0');
|
||||
expect(output).toContain('Overall Agreement Rate:');
|
||||
expect(output).toContain('--');
|
||||
expect(output).toMatchSnapshot();
|
||||
});
|
||||
});
|
|
@ -0,0 +1,208 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { Box, Text } from 'ink';
|
||||
import { Colors } from '../colors.js';
|
||||
import { formatDuration } from '../utils/formatters.js';
|
||||
import {
|
||||
getStatusColor,
|
||||
TOOL_SUCCESS_RATE_HIGH,
|
||||
TOOL_SUCCESS_RATE_MEDIUM,
|
||||
USER_AGREEMENT_RATE_HIGH,
|
||||
USER_AGREEMENT_RATE_MEDIUM,
|
||||
} from '../utils/displayUtils.js';
|
||||
import { useSessionStats } from '../contexts/SessionContext.js';
|
||||
import { ToolCallStats } from '@google/gemini-cli-core';
|
||||
|
||||
const TOOL_NAME_COL_WIDTH = 25;
|
||||
const CALLS_COL_WIDTH = 8;
|
||||
const SUCCESS_RATE_COL_WIDTH = 15;
|
||||
const AVG_DURATION_COL_WIDTH = 15;
|
||||
|
||||
const StatRow: React.FC<{
|
||||
name: string;
|
||||
stats: ToolCallStats;
|
||||
}> = ({ name, stats }) => {
|
||||
const successRate = stats.count > 0 ? (stats.success / stats.count) * 100 : 0;
|
||||
const avgDuration = stats.count > 0 ? stats.durationMs / stats.count : 0;
|
||||
const successColor = getStatusColor(successRate, {
|
||||
green: TOOL_SUCCESS_RATE_HIGH,
|
||||
yellow: TOOL_SUCCESS_RATE_MEDIUM,
|
||||
});
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<Box width={TOOL_NAME_COL_WIDTH}>
|
||||
<Text color={Colors.LightBlue}>{name}</Text>
|
||||
</Box>
|
||||
<Box width={CALLS_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text>{stats.count}</Text>
|
||||
</Box>
|
||||
<Box width={SUCCESS_RATE_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text color={successColor}>{successRate.toFixed(1)}%</Text>
|
||||
</Box>
|
||||
<Box width={AVG_DURATION_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text>{formatDuration(avgDuration)}</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export const ToolStatsDisplay: React.FC = () => {
|
||||
const { stats } = useSessionStats();
|
||||
const { tools } = stats.metrics;
|
||||
const activeTools = Object.entries(tools.byName).filter(
|
||||
([, metrics]) => metrics.count > 0,
|
||||
);
|
||||
|
||||
if (activeTools.length === 0) {
|
||||
return (
|
||||
<Box
|
||||
borderStyle="round"
|
||||
borderColor={Colors.Gray}
|
||||
paddingY={1}
|
||||
paddingX={2}
|
||||
>
|
||||
<Text>No tool calls have been made in this session.</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
const totalDecisions = Object.values(tools.byName).reduce(
|
||||
(acc, tool) => {
|
||||
acc.accept += tool.decisions.accept;
|
||||
acc.reject += tool.decisions.reject;
|
||||
acc.modify += tool.decisions.modify;
|
||||
return acc;
|
||||
},
|
||||
{ accept: 0, reject: 0, modify: 0 },
|
||||
);
|
||||
|
||||
const totalReviewed =
|
||||
totalDecisions.accept + totalDecisions.reject + totalDecisions.modify;
|
||||
const agreementRate =
|
||||
totalReviewed > 0 ? (totalDecisions.accept / totalReviewed) * 100 : 0;
|
||||
const agreementColor = getStatusColor(agreementRate, {
|
||||
green: USER_AGREEMENT_RATE_HIGH,
|
||||
yellow: USER_AGREEMENT_RATE_MEDIUM,
|
||||
});
|
||||
|
||||
return (
|
||||
<Box
|
||||
borderStyle="round"
|
||||
borderColor={Colors.Gray}
|
||||
flexDirection="column"
|
||||
paddingY={1}
|
||||
paddingX={2}
|
||||
width={70}
|
||||
>
|
||||
<Text bold color={Colors.AccentPurple}>
|
||||
Tool Stats For Nerds
|
||||
</Text>
|
||||
<Box height={1} />
|
||||
|
||||
{/* Header */}
|
||||
<Box>
|
||||
<Box width={TOOL_NAME_COL_WIDTH}>
|
||||
<Text bold>Tool Name</Text>
|
||||
</Box>
|
||||
<Box width={CALLS_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text bold>Calls</Text>
|
||||
</Box>
|
||||
<Box width={SUCCESS_RATE_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text bold>Success Rate</Text>
|
||||
</Box>
|
||||
<Box width={AVG_DURATION_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text bold>Avg Duration</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
{/* Divider */}
|
||||
<Box
|
||||
borderStyle="single"
|
||||
borderBottom={true}
|
||||
borderTop={false}
|
||||
borderLeft={false}
|
||||
borderRight={false}
|
||||
width="100%"
|
||||
/>
|
||||
|
||||
{/* Tool Rows */}
|
||||
{activeTools.map(([name, stats]) => (
|
||||
<StatRow key={name} name={name} stats={stats as ToolCallStats} />
|
||||
))}
|
||||
|
||||
<Box height={1} />
|
||||
|
||||
{/* User Decision Summary */}
|
||||
<Text bold>User Decision Summary</Text>
|
||||
<Box>
|
||||
<Box
|
||||
width={TOOL_NAME_COL_WIDTH + CALLS_COL_WIDTH + SUCCESS_RATE_COL_WIDTH}
|
||||
>
|
||||
<Text color={Colors.LightBlue}>Total Reviewed Suggestions:</Text>
|
||||
</Box>
|
||||
<Box width={AVG_DURATION_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text>{totalReviewed}</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
<Box>
|
||||
<Box
|
||||
width={TOOL_NAME_COL_WIDTH + CALLS_COL_WIDTH + SUCCESS_RATE_COL_WIDTH}
|
||||
>
|
||||
<Text> » Accepted:</Text>
|
||||
</Box>
|
||||
<Box width={AVG_DURATION_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text color={Colors.AccentGreen}>{totalDecisions.accept}</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
<Box>
|
||||
<Box
|
||||
width={TOOL_NAME_COL_WIDTH + CALLS_COL_WIDTH + SUCCESS_RATE_COL_WIDTH}
|
||||
>
|
||||
<Text> » Rejected:</Text>
|
||||
</Box>
|
||||
<Box width={AVG_DURATION_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text color={Colors.AccentRed}>{totalDecisions.reject}</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
<Box>
|
||||
<Box
|
||||
width={TOOL_NAME_COL_WIDTH + CALLS_COL_WIDTH + SUCCESS_RATE_COL_WIDTH}
|
||||
>
|
||||
<Text> » Modified:</Text>
|
||||
</Box>
|
||||
<Box width={AVG_DURATION_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text color={Colors.AccentYellow}>{totalDecisions.modify}</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
{/* Divider */}
|
||||
<Box
|
||||
borderStyle="single"
|
||||
borderBottom={true}
|
||||
borderTop={false}
|
||||
borderLeft={false}
|
||||
borderRight={false}
|
||||
width="100%"
|
||||
/>
|
||||
|
||||
<Box>
|
||||
<Box
|
||||
width={TOOL_NAME_COL_WIDTH + CALLS_COL_WIDTH + SUCCESS_RATE_COL_WIDTH}
|
||||
>
|
||||
<Text> Overall Agreement Rate:</Text>
|
||||
</Box>
|
||||
<Box width={AVG_DURATION_COL_WIDTH} justifyContent="flex-end">
|
||||
<Text bold color={totalReviewed > 0 ? agreementColor : undefined}>
|
||||
{totalReviewed > 0 ? `${agreementRate.toFixed(1)}%` : '--'}
|
||||
</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
|
@ -0,0 +1,121 @@
|
|||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
||||
|
||||
exports[`<ModelStatsDisplay /> > should display a single model correctly 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Model Stats For Nerds │
|
||||
│ │
|
||||
│ Metric gemini-2.5-pro │
|
||||
│ ────────────────────────────────────────────────────────────────────────────────────────────── │
|
||||
│ API │
|
||||
│ Requests 1 │
|
||||
│ Errors 0 (0.0%) │
|
||||
│ Avg Latency 100ms │
|
||||
│ │
|
||||
│ Tokens │
|
||||
│ Total 30 │
|
||||
│ ↳ Prompt 10 │
|
||||
│ ↳ Cached 5 (50.0%) │
|
||||
│ ↳ Thoughts 2 │
|
||||
│ ↳ Tool 1 │
|
||||
│ ↳ Output 20 │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ModelStatsDisplay /> > should display conditional rows if at least one model has data 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Model Stats For Nerds │
|
||||
│ │
|
||||
│ Metric gemini-2.5-pro gemini-2.5-flash │
|
||||
│ ────────────────────────────────────────────────────────────────────────────────────────────── │
|
||||
│ API │
|
||||
│ Requests 1 1 │
|
||||
│ Errors 0 (0.0%) 0 (0.0%) │
|
||||
│ Avg Latency 100ms 50ms │
|
||||
│ │
|
||||
│ Tokens │
|
||||
│ Total 30 15 │
|
||||
│ ↳ Prompt 10 5 │
|
||||
│ ↳ Cached 5 (50.0%) 0 (0.0%) │
|
||||
│ ↳ Thoughts 2 0 │
|
||||
│ ↳ Tool 0 3 │
|
||||
│ ↳ Output 20 10 │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ModelStatsDisplay /> > should display stats for multiple models correctly 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Model Stats For Nerds │
|
||||
│ │
|
||||
│ Metric gemini-2.5-pro gemini-2.5-flash │
|
||||
│ ────────────────────────────────────────────────────────────────────────────────────────────── │
|
||||
│ API │
|
||||
│ Requests 10 20 │
|
||||
│ Errors 1 (10.0%) 2 (10.0%) │
|
||||
│ Avg Latency 100ms 25ms │
|
||||
│ │
|
||||
│ Tokens │
|
||||
│ Total 300 600 │
|
||||
│ ↳ Prompt 100 200 │
|
||||
│ ↳ Cached 50 (50.0%) 100 (50.0%) │
|
||||
│ ↳ Thoughts 10 20 │
|
||||
│ ↳ Tool 5 10 │
|
||||
│ ↳ Output 200 400 │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ModelStatsDisplay /> > should handle large values without wrapping or overlapping 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Model Stats For Nerds │
|
||||
│ │
|
||||
│ Metric gemini-2.5-pro │
|
||||
│ ────────────────────────────────────────────────────────────────────────────────────────────── │
|
||||
│ API │
|
||||
│ Requests 999,999,999 │
|
||||
│ Errors 123,456,789 (12.3%) │
|
||||
│ Avg Latency 0ms │
|
||||
│ │
|
||||
│ Tokens │
|
||||
│ Total 999,999,999 │
|
||||
│ ↳ Prompt 987,654,321 │
|
||||
│ ↳ Cached 123,456,789 (12.5%) │
|
||||
│ ↳ Thoughts 111,111,111 │
|
||||
│ ↳ Tool 222,222,222 │
|
||||
│ ↳ Output 123,456,789 │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ModelStatsDisplay /> > should not display conditional rows if no model has data for them 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Model Stats For Nerds │
|
||||
│ │
|
||||
│ Metric gemini-2.5-pro │
|
||||
│ ────────────────────────────────────────────────────────────────────────────────────────────── │
|
||||
│ API │
|
||||
│ Requests 1 │
|
||||
│ Errors 0 (0.0%) │
|
||||
│ Avg Latency 100ms │
|
||||
│ │
|
||||
│ Tokens │
|
||||
│ Total 30 │
|
||||
│ ↳ Prompt 10 │
|
||||
│ ↳ Output 20 │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ModelStatsDisplay /> > should render "no API calls" message when there are no active models 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ No API calls have been made in this session. │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
|
@ -1,43 +1,45 @@
|
|||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
||||
|
||||
exports[`<SessionSummaryDisplay /> > renders correctly with given stats and duration 1`] = `
|
||||
exports[`<SessionSummaryDisplay /> > correctly sums and displays stats from multiple models 1`] = `
|
||||
"╭─────────────────────────────────────╮
|
||||
│ │
|
||||
│ Agent powering down. Goodbye! │
|
||||
│ │
|
||||
│ │
|
||||
│ Cumulative Stats (10 Turns) │
|
||||
│ Cumulative Stats (15 API calls) │
|
||||
│ │
|
||||
│ Input Tokens 1,000 │
|
||||
│ Output Tokens 2,000 │
|
||||
│ Tool Use Tokens 200 │
|
||||
│ Thoughts Tokens 300 │
|
||||
│ Cached Tokens 500 (14.3%) │
|
||||
│ Input Tokens 1,500 │
|
||||
│ Output Tokens 3,000 │
|
||||
│ Tool Use Tokens 220 │
|
||||
│ Thoughts Tokens 350 │
|
||||
│ Cached Tokens 600 (12.0%) │
|
||||
│ ───────────────────────────────── │
|
||||
│ Total Tokens 3,500 │
|
||||
│ Total Tokens 5,000 │
|
||||
│ │
|
||||
│ Total duration (API) 50.2s │
|
||||
│ Total duration (API) 1m 2s │
|
||||
│ Total duration (Tools) 0s │
|
||||
│ Total duration (wall) 1h 23m 45s │
|
||||
│ │
|
||||
╰─────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<SessionSummaryDisplay /> > renders zero state correctly 1`] = `
|
||||
"╭─────────────────────────────────╮
|
||||
│ │
|
||||
│ Agent powering down. Goodbye! │
|
||||
│ │
|
||||
│ │
|
||||
│ Cumulative Stats (0 Turns) │
|
||||
│ │
|
||||
│ Input Tokens 0 │
|
||||
│ Output Tokens 0 │
|
||||
│ Thoughts Tokens 0 │
|
||||
│ ────────────────────────── │
|
||||
│ Total Tokens 0 │
|
||||
│ │
|
||||
│ Total duration (API) 0s │
|
||||
│ Total duration (wall) 0s │
|
||||
│ │
|
||||
╰─────────────────────────────────╯"
|
||||
"╭─────────────────────────────────────╮
|
||||
│ │
|
||||
│ Agent powering down. Goodbye! │
|
||||
│ │
|
||||
│ │
|
||||
│ Cumulative Stats (0 API calls) │
|
||||
│ │
|
||||
│ Input Tokens 0 │
|
||||
│ Output Tokens 0 │
|
||||
│ Thoughts Tokens 0 │
|
||||
│ ───────────────────────────────── │
|
||||
│ Total Tokens 0 │
|
||||
│ │
|
||||
│ Total duration (API) 0s │
|
||||
│ Total duration (Tools) 0s │
|
||||
│ Total duration (wall) 1h 23m 45s │
|
||||
│ │
|
||||
╰─────────────────────────────────────╯"
|
||||
`;
|
||||
|
|
|
@ -1,41 +1,163 @@
|
|||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
||||
|
||||
exports[`<StatsDisplay /> > renders correctly with given stats and duration 1`] = `
|
||||
exports[`<StatsDisplay /> > Conditional Color Tests > renders success rate in green for high values 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Stats │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Last Turn Cumulative (10 Turns) │
|
||||
│ Interaction Summary │
|
||||
│ Tool Calls: 10 ( ✔ 10 ✖ 0 ) │
|
||||
│ Success Rate: 100.0% │
|
||||
│ │
|
||||
│ Input Tokens 100 Input Tokens 1,000 │
|
||||
│ Output Tokens 200 Output Tokens 2,000 │
|
||||
│ Tool Use Tokens 20 Tool Use Tokens 200 │
|
||||
│ Thoughts Tokens 30 Thoughts Tokens 300 │
|
||||
│ Cached Tokens 50 Cached Tokens 500 (14.3%) │
|
||||
│ ───────────────────────────────────────────── ───────────────────────────────────────────── │
|
||||
│ Total Tokens 350 Total Tokens 3,500 │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 0s │
|
||||
│ » API Time: 0s (0.0%) │
|
||||
│ » Tool Time: 0s (0.0%) │
|
||||
│ │
|
||||
│ Turn Duration (API) 1.2s Total duration (API) 50.2s │
|
||||
│ Total duration (wall) 1h 23m 45s │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<StatsDisplay /> > renders zero state correctly 1`] = `
|
||||
exports[`<StatsDisplay /> > Conditional Color Tests > renders success rate in red for low values 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Stats │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Last Turn Cumulative (0 Turns) │
|
||||
│ Interaction Summary │
|
||||
│ Tool Calls: 10 ( ✔ 5 ✖ 5 ) │
|
||||
│ Success Rate: 50.0% │
|
||||
│ │
|
||||
│ Input Tokens 0 Input Tokens 0 │
|
||||
│ Output Tokens 0 Output Tokens 0 │
|
||||
│ Thoughts Tokens 0 Thoughts Tokens 0 │
|
||||
│ ───────────────────────────────────────────── ───────────────────────────────────────────── │
|
||||
│ Total Tokens 0 Total Tokens 0 │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 0s │
|
||||
│ » API Time: 0s (0.0%) │
|
||||
│ » Tool Time: 0s (0.0%) │
|
||||
│ │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<StatsDisplay /> > Conditional Color Tests > renders success rate in yellow for medium values 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Interaction Summary │
|
||||
│ Tool Calls: 10 ( ✔ 9 ✖ 1 ) │
|
||||
│ Success Rate: 90.0% │
|
||||
│ │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 0s │
|
||||
│ » API Time: 0s (0.0%) │
|
||||
│ » Tool Time: 0s (0.0%) │
|
||||
│ │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<StatsDisplay /> > Conditional Rendering Tests > hides Efficiency section when cache is not used 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 100ms │
|
||||
│ » API Time: 100ms (100.0%) │
|
||||
│ » Tool Time: 0s (0.0%) │
|
||||
│ │
|
||||
│ │
|
||||
│ Model Usage Reqs Input Tokens Output Tokens │
|
||||
│ ─────────────────────────────────────────────────────────────── │
|
||||
│ gemini-2.5-pro 1 100 100 │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<StatsDisplay /> > Conditional Rendering Tests > hides User Agreement when no decisions are made 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Interaction Summary │
|
||||
│ Tool Calls: 2 ( ✔ 1 ✖ 1 ) │
|
||||
│ Success Rate: 50.0% │
|
||||
│ │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 123ms │
|
||||
│ » API Time: 0s (0.0%) │
|
||||
│ » Tool Time: 123ms (100.0%) │
|
||||
│ │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<StatsDisplay /> > renders a table with two models correctly 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 19.5s │
|
||||
│ » API Time: 19.5s (100.0%) │
|
||||
│ » Tool Time: 0s (0.0%) │
|
||||
│ │
|
||||
│ │
|
||||
│ Model Usage Reqs Input Tokens Output Tokens │
|
||||
│ ─────────────────────────────────────────────────────────────── │
|
||||
│ gemini-2.5-pro 3 1,000 2,000 │
|
||||
│ gemini-2.5-flash 5 25,000 15,000 │
|
||||
│ │
|
||||
│ Savings Highlight: 10,500 (40.4%) of input tokens were served from the cache, reducing costs. │
|
||||
│ │
|
||||
│ » Tip: For a full token breakdown, run \`/stats model\`. │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<StatsDisplay /> > renders all sections when all data is present 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Interaction Summary │
|
||||
│ Tool Calls: 2 ( ✔ 1 ✖ 1 ) │
|
||||
│ Success Rate: 50.0% │
|
||||
│ User Agreement: 100.0% (1 reviewed) │
|
||||
│ │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 223ms │
|
||||
│ » API Time: 100ms (44.8%) │
|
||||
│ » Tool Time: 123ms (55.2%) │
|
||||
│ │
|
||||
│ │
|
||||
│ Model Usage Reqs Input Tokens Output Tokens │
|
||||
│ ─────────────────────────────────────────────────────────────── │
|
||||
│ gemini-2.5-pro 1 100 100 │
|
||||
│ │
|
||||
│ Savings Highlight: 50 (50.0%) of input tokens were served from the cache, reducing costs. │
|
||||
│ │
|
||||
│ » Tip: For a full token breakdown, run \`/stats model\`. │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<StatsDisplay /> > renders only the Performance section in its zero state 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Session Stats │
|
||||
│ │
|
||||
│ Performance │
|
||||
│ Wall Time: 1s │
|
||||
│ Agent Active: 0s │
|
||||
│ » API Time: 0s (0.0%) │
|
||||
│ » Tool Time: 0s (0.0%) │
|
||||
│ │
|
||||
│ Turn Duration (API) 0s Total duration (API) 0s │
|
||||
│ Total duration (wall) 0s │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
|
|
@ -0,0 +1,91 @@
|
|||
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
||||
|
||||
exports[`<ToolStatsDisplay /> > should display stats for a single tool correctly 1`] = `
|
||||
"╭────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Tool Stats For Nerds │
|
||||
│ │
|
||||
│ Tool Name Calls Success Rate Avg Duration │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ test-tool 1 100.0% 100ms │
|
||||
│ │
|
||||
│ User Decision Summary │
|
||||
│ Total Reviewed Suggestions: 1 │
|
||||
│ » Accepted: 1 │
|
||||
│ » Rejected: 0 │
|
||||
│ » Modified: 0 │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ Overall Agreement Rate: 100.0% │
|
||||
│ │
|
||||
╰────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ToolStatsDisplay /> > should display stats for multiple tools correctly 1`] = `
|
||||
"╭────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Tool Stats For Nerds │
|
||||
│ │
|
||||
│ Tool Name Calls Success Rate Avg Duration │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ tool-a 2 50.0% 100ms │
|
||||
│ tool-b 1 100.0% 100ms │
|
||||
│ │
|
||||
│ User Decision Summary │
|
||||
│ Total Reviewed Suggestions: 3 │
|
||||
│ » Accepted: 1 │
|
||||
│ » Rejected: 1 │
|
||||
│ » Modified: 1 │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ Overall Agreement Rate: 33.3% │
|
||||
│ │
|
||||
╰────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ToolStatsDisplay /> > should handle large values without wrapping or overlapping 1`] = `
|
||||
"╭────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Tool Stats For Nerds │
|
||||
│ │
|
||||
│ Tool Name Calls Success Rate Avg Duration │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ long-named-tool-for-testi99999999 88.9% 1ms │
|
||||
│ ng-wrapping-and-such 9 │
|
||||
│ │
|
||||
│ User Decision Summary │
|
||||
│ Total Reviewed Suggestions: 222234566 │
|
||||
│ » Accepted: 123456789 │
|
||||
│ » Rejected: 98765432 │
|
||||
│ » Modified: 12345 │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ Overall Agreement Rate: 55.6% │
|
||||
│ │
|
||||
╰────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ToolStatsDisplay /> > should handle zero decisions gracefully 1`] = `
|
||||
"╭────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ Tool Stats For Nerds │
|
||||
│ │
|
||||
│ Tool Name Calls Success Rate Avg Duration │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ test-tool 1 100.0% 100ms │
|
||||
│ │
|
||||
│ User Decision Summary │
|
||||
│ Total Reviewed Suggestions: 0 │
|
||||
│ » Accepted: 0 │
|
||||
│ » Rejected: 0 │
|
||||
│ » Modified: 0 │
|
||||
│ ──────────────────────────────────────────────────────────────── │
|
||||
│ Overall Agreement Rate: -- │
|
||||
│ │
|
||||
╰────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
||||
|
||||
exports[`<ToolStatsDisplay /> > should render "no tool calls" message when there are no active tools 1`] = `
|
||||
"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ │
|
||||
│ No tool calls have been made in this session. │
|
||||
│ │
|
||||
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯"
|
||||
`;
|
|
@ -8,28 +8,13 @@ import { type MutableRefObject } from 'react';
|
|||
import { render } from 'ink-testing-library';
|
||||
import { renderHook } from '@testing-library/react';
|
||||
import { act } from 'react-dom/test-utils';
|
||||
import { SessionStatsProvider, useSessionStats } from './SessionContext.js';
|
||||
import {
|
||||
SessionStatsProvider,
|
||||
useSessionStats,
|
||||
SessionMetrics,
|
||||
} from './SessionContext.js';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
||||
|
||||
// Mock data that simulates what the Gemini API would return.
|
||||
const mockMetadata1: GenerateContentResponseUsageMetadata = {
|
||||
promptTokenCount: 100,
|
||||
candidatesTokenCount: 200,
|
||||
totalTokenCount: 300,
|
||||
cachedContentTokenCount: 50,
|
||||
toolUsePromptTokenCount: 10,
|
||||
thoughtsTokenCount: 20,
|
||||
};
|
||||
|
||||
const mockMetadata2: GenerateContentResponseUsageMetadata = {
|
||||
promptTokenCount: 10,
|
||||
candidatesTokenCount: 20,
|
||||
totalTokenCount: 30,
|
||||
cachedContentTokenCount: 5,
|
||||
toolUsePromptTokenCount: 1,
|
||||
thoughtsTokenCount: 2,
|
||||
};
|
||||
import { uiTelemetryService } from '@google/gemini-cli-core';
|
||||
|
||||
/**
|
||||
* A test harness component that uses the hook and exposes the context value
|
||||
|
@ -60,13 +45,11 @@ describe('SessionStatsContext', () => {
|
|||
const stats = contextRef.current?.stats;
|
||||
|
||||
expect(stats?.sessionStartTime).toBeInstanceOf(Date);
|
||||
expect(stats?.currentTurn).toBeDefined();
|
||||
expect(stats?.cumulative.turnCount).toBe(0);
|
||||
expect(stats?.cumulative.totalTokenCount).toBe(0);
|
||||
expect(stats?.cumulative.promptTokenCount).toBe(0);
|
||||
expect(stats?.metrics).toBeDefined();
|
||||
expect(stats?.metrics.models).toEqual({});
|
||||
});
|
||||
|
||||
it('should increment turnCount when startNewTurn is called', () => {
|
||||
it('should update metrics when the uiTelemetryService emits an update', () => {
|
||||
const contextRef: MutableRefObject<
|
||||
ReturnType<typeof useSessionStats> | undefined
|
||||
> = { current: undefined };
|
||||
|
@ -77,150 +60,60 @@ describe('SessionStatsContext', () => {
|
|||
</SessionStatsProvider>,
|
||||
);
|
||||
|
||||
const newMetrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-pro': {
|
||||
api: {
|
||||
totalRequests: 1,
|
||||
totalErrors: 0,
|
||||
totalLatencyMs: 123,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 100,
|
||||
candidates: 200,
|
||||
total: 300,
|
||||
cached: 50,
|
||||
thoughts: 20,
|
||||
tool: 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 1,
|
||||
totalSuccess: 1,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 456,
|
||||
totalDecisions: {
|
||||
accept: 1,
|
||||
reject: 0,
|
||||
modify: 0,
|
||||
},
|
||||
byName: {
|
||||
'test-tool': {
|
||||
count: 1,
|
||||
success: 1,
|
||||
fail: 0,
|
||||
durationMs: 456,
|
||||
decisions: {
|
||||
accept: 1,
|
||||
reject: 0,
|
||||
modify: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
act(() => {
|
||||
contextRef.current?.startNewTurn();
|
||||
uiTelemetryService.emit('update', {
|
||||
metrics: newMetrics,
|
||||
lastPromptTokenCount: 100,
|
||||
});
|
||||
});
|
||||
|
||||
const stats = contextRef.current?.stats;
|
||||
expect(stats?.currentTurn.totalTokenCount).toBe(0);
|
||||
expect(stats?.cumulative.turnCount).toBe(1);
|
||||
// Ensure token counts are unaffected
|
||||
expect(stats?.cumulative.totalTokenCount).toBe(0);
|
||||
});
|
||||
|
||||
it('should aggregate token usage correctly when addUsage is called', () => {
|
||||
const contextRef: MutableRefObject<
|
||||
ReturnType<typeof useSessionStats> | undefined
|
||||
> = { current: undefined };
|
||||
|
||||
render(
|
||||
<SessionStatsProvider>
|
||||
<TestHarness contextRef={contextRef} />
|
||||
</SessionStatsProvider>,
|
||||
);
|
||||
|
||||
act(() => {
|
||||
contextRef.current?.addUsage({ ...mockMetadata1, apiTimeMs: 123 });
|
||||
});
|
||||
|
||||
const stats = contextRef.current?.stats;
|
||||
|
||||
// Check that token counts are updated
|
||||
expect(stats?.cumulative.totalTokenCount).toBe(
|
||||
mockMetadata1.totalTokenCount ?? 0,
|
||||
);
|
||||
expect(stats?.cumulative.promptTokenCount).toBe(
|
||||
mockMetadata1.promptTokenCount ?? 0,
|
||||
);
|
||||
expect(stats?.cumulative.apiTimeMs).toBe(123);
|
||||
|
||||
// Check that turn count is NOT incremented
|
||||
expect(stats?.cumulative.turnCount).toBe(0);
|
||||
|
||||
// Check that currentTurn is updated
|
||||
expect(stats?.currentTurn?.totalTokenCount).toEqual(
|
||||
mockMetadata1.totalTokenCount,
|
||||
);
|
||||
expect(stats?.currentTurn?.apiTimeMs).toBe(123);
|
||||
});
|
||||
|
||||
it('should correctly track a full logical turn with multiple API calls', () => {
|
||||
const contextRef: MutableRefObject<
|
||||
ReturnType<typeof useSessionStats> | undefined
|
||||
> = { current: undefined };
|
||||
|
||||
render(
|
||||
<SessionStatsProvider>
|
||||
<TestHarness contextRef={contextRef} />
|
||||
</SessionStatsProvider>,
|
||||
);
|
||||
|
||||
// 1. User starts a new turn
|
||||
act(() => {
|
||||
contextRef.current?.startNewTurn();
|
||||
});
|
||||
|
||||
// 2. First API call (e.g., prompt with a tool request)
|
||||
act(() => {
|
||||
contextRef.current?.addUsage({ ...mockMetadata1, apiTimeMs: 100 });
|
||||
});
|
||||
|
||||
// 3. Second API call (e.g., sending tool response back)
|
||||
act(() => {
|
||||
contextRef.current?.addUsage({ ...mockMetadata2, apiTimeMs: 50 });
|
||||
});
|
||||
|
||||
const stats = contextRef.current?.stats;
|
||||
|
||||
// Turn count should only be 1
|
||||
expect(stats?.cumulative.turnCount).toBe(1);
|
||||
|
||||
// --- Check Cumulative Stats ---
|
||||
// These fields should be the SUM of both calls
|
||||
expect(stats?.cumulative.totalTokenCount).toBe(300 + 30);
|
||||
expect(stats?.cumulative.candidatesTokenCount).toBe(200 + 20);
|
||||
expect(stats?.cumulative.thoughtsTokenCount).toBe(20 + 2);
|
||||
expect(stats?.cumulative.apiTimeMs).toBe(100 + 50);
|
||||
|
||||
// These fields should be the SUM of both calls
|
||||
expect(stats?.cumulative.promptTokenCount).toBe(100 + 10);
|
||||
expect(stats?.cumulative.cachedContentTokenCount).toBe(50 + 5);
|
||||
expect(stats?.cumulative.toolUsePromptTokenCount).toBe(10 + 1);
|
||||
|
||||
// --- Check Current Turn Stats ---
|
||||
// All fields should be the SUM of both calls for the turn
|
||||
expect(stats?.currentTurn.totalTokenCount).toBe(300 + 30);
|
||||
expect(stats?.currentTurn.candidatesTokenCount).toBe(200 + 20);
|
||||
expect(stats?.currentTurn.thoughtsTokenCount).toBe(20 + 2);
|
||||
expect(stats?.currentTurn.promptTokenCount).toBe(100 + 10);
|
||||
expect(stats?.currentTurn.cachedContentTokenCount).toBe(50 + 5);
|
||||
expect(stats?.currentTurn.toolUsePromptTokenCount).toBe(10 + 1);
|
||||
expect(stats?.currentTurn.apiTimeMs).toBe(100 + 50);
|
||||
});
|
||||
|
||||
it('should overwrite currentResponse with each API call', () => {
|
||||
const contextRef: MutableRefObject<
|
||||
ReturnType<typeof useSessionStats> | undefined
|
||||
> = { current: undefined };
|
||||
|
||||
render(
|
||||
<SessionStatsProvider>
|
||||
<TestHarness contextRef={contextRef} />
|
||||
</SessionStatsProvider>,
|
||||
);
|
||||
|
||||
// 1. First API call
|
||||
act(() => {
|
||||
contextRef.current?.addUsage({ ...mockMetadata1, apiTimeMs: 100 });
|
||||
});
|
||||
|
||||
let stats = contextRef.current?.stats;
|
||||
|
||||
// currentResponse should match the first call
|
||||
expect(stats?.currentResponse.totalTokenCount).toBe(300);
|
||||
expect(stats?.currentResponse.apiTimeMs).toBe(100);
|
||||
|
||||
// 2. Second API call
|
||||
act(() => {
|
||||
contextRef.current?.addUsage({ ...mockMetadata2, apiTimeMs: 50 });
|
||||
});
|
||||
|
||||
stats = contextRef.current?.stats;
|
||||
|
||||
// currentResponse should now match the second call
|
||||
expect(stats?.currentResponse.totalTokenCount).toBe(30);
|
||||
expect(stats?.currentResponse.apiTimeMs).toBe(50);
|
||||
|
||||
// 3. Start a new turn
|
||||
act(() => {
|
||||
contextRef.current?.startNewTurn();
|
||||
});
|
||||
|
||||
stats = contextRef.current?.stats;
|
||||
|
||||
// currentResponse should be reset
|
||||
expect(stats?.currentResponse.totalTokenCount).toBe(0);
|
||||
expect(stats?.currentResponse.apiTimeMs).toBe(0);
|
||||
expect(stats?.metrics).toEqual(newMetrics);
|
||||
expect(stats?.lastPromptTokenCount).toBe(100);
|
||||
});
|
||||
|
||||
it('should throw an error when useSessionStats is used outside of a provider', () => {
|
||||
|
|
|
@ -9,39 +9,43 @@ import React, {
|
|||
useContext,
|
||||
useState,
|
||||
useMemo,
|
||||
useCallback,
|
||||
useEffect,
|
||||
} from 'react';
|
||||
|
||||
import { type GenerateContentResponseUsageMetadata } from '@google/genai';
|
||||
import {
|
||||
uiTelemetryService,
|
||||
SessionMetrics,
|
||||
ModelMetrics,
|
||||
} from '@google/gemini-cli-core';
|
||||
|
||||
// --- Interface Definitions ---
|
||||
|
||||
export interface CumulativeStats {
|
||||
turnCount: number;
|
||||
promptTokenCount: number;
|
||||
candidatesTokenCount: number;
|
||||
totalTokenCount: number;
|
||||
cachedContentTokenCount: number;
|
||||
toolUsePromptTokenCount: number;
|
||||
thoughtsTokenCount: number;
|
||||
apiTimeMs: number;
|
||||
}
|
||||
export type { SessionMetrics, ModelMetrics };
|
||||
|
||||
interface SessionStatsState {
|
||||
sessionStartTime: Date;
|
||||
cumulative: CumulativeStats;
|
||||
currentTurn: CumulativeStats;
|
||||
currentResponse: CumulativeStats;
|
||||
metrics: SessionMetrics;
|
||||
lastPromptTokenCount: number;
|
||||
}
|
||||
|
||||
export interface ComputedSessionStats {
|
||||
totalApiTime: number;
|
||||
totalToolTime: number;
|
||||
agentActiveTime: number;
|
||||
apiTimePercent: number;
|
||||
toolTimePercent: number;
|
||||
cacheEfficiency: number;
|
||||
totalDecisions: number;
|
||||
successRate: number;
|
||||
agreementRate: number;
|
||||
totalCachedTokens: number;
|
||||
totalPromptTokens: number;
|
||||
}
|
||||
|
||||
// Defines the final "value" of our context, including the state
|
||||
// and the functions to update it.
|
||||
interface SessionStatsContextValue {
|
||||
stats: SessionStatsState;
|
||||
startNewTurn: () => void;
|
||||
addUsage: (
|
||||
metadata: GenerateContentResponseUsageMetadata & { apiTimeMs?: number },
|
||||
) => void;
|
||||
}
|
||||
|
||||
// --- Context Definition ---
|
||||
|
@ -50,27 +54,6 @@ const SessionStatsContext = createContext<SessionStatsContextValue | undefined>(
|
|||
undefined,
|
||||
);
|
||||
|
||||
// --- Helper Functions ---
|
||||
|
||||
/**
|
||||
* A small, reusable helper function to sum token counts.
|
||||
* It unconditionally adds all token values from the source to the target.
|
||||
* @param target The object to add the tokens to (e.g., cumulative, currentTurn).
|
||||
* @param source The metadata object from the API response.
|
||||
*/
|
||||
const addTokens = (
|
||||
target: CumulativeStats,
|
||||
source: GenerateContentResponseUsageMetadata & { apiTimeMs?: number },
|
||||
) => {
|
||||
target.candidatesTokenCount += source.candidatesTokenCount ?? 0;
|
||||
target.thoughtsTokenCount += source.thoughtsTokenCount ?? 0;
|
||||
target.totalTokenCount += source.totalTokenCount ?? 0;
|
||||
target.apiTimeMs += source.apiTimeMs ?? 0;
|
||||
target.promptTokenCount += source.promptTokenCount ?? 0;
|
||||
target.cachedContentTokenCount += source.cachedContentTokenCount ?? 0;
|
||||
target.toolUsePromptTokenCount += source.toolUsePromptTokenCount ?? 0;
|
||||
};
|
||||
|
||||
// --- Provider Component ---
|
||||
|
||||
export const SessionStatsProvider: React.FC<{ children: React.ReactNode }> = ({
|
||||
|
@ -78,110 +61,42 @@ export const SessionStatsProvider: React.FC<{ children: React.ReactNode }> = ({
|
|||
}) => {
|
||||
const [stats, setStats] = useState<SessionStatsState>({
|
||||
sessionStartTime: new Date(),
|
||||
cumulative: {
|
||||
turnCount: 0,
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
},
|
||||
currentTurn: {
|
||||
turnCount: 0,
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
},
|
||||
currentResponse: {
|
||||
turnCount: 0,
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
},
|
||||
metrics: uiTelemetryService.getMetrics(),
|
||||
lastPromptTokenCount: 0,
|
||||
});
|
||||
|
||||
// A single, internal worker function to handle all metadata aggregation.
|
||||
const aggregateTokens = useCallback(
|
||||
(
|
||||
metadata: GenerateContentResponseUsageMetadata & { apiTimeMs?: number },
|
||||
) => {
|
||||
setStats((prevState) => {
|
||||
const newCumulative = { ...prevState.cumulative };
|
||||
const newCurrentTurn = { ...prevState.currentTurn };
|
||||
const newCurrentResponse = {
|
||||
turnCount: 0,
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
};
|
||||
useEffect(() => {
|
||||
const handleUpdate = ({
|
||||
metrics,
|
||||
lastPromptTokenCount,
|
||||
}: {
|
||||
metrics: SessionMetrics;
|
||||
lastPromptTokenCount: number;
|
||||
}) => {
|
||||
setStats((prevState) => ({
|
||||
...prevState,
|
||||
metrics,
|
||||
lastPromptTokenCount,
|
||||
}));
|
||||
};
|
||||
|
||||
// Add all tokens to the current turn's stats as well as cumulative stats.
|
||||
addTokens(newCurrentTurn, metadata);
|
||||
addTokens(newCumulative, metadata);
|
||||
addTokens(newCurrentResponse, metadata);
|
||||
uiTelemetryService.on('update', handleUpdate);
|
||||
// Set initial state
|
||||
handleUpdate({
|
||||
metrics: uiTelemetryService.getMetrics(),
|
||||
lastPromptTokenCount: uiTelemetryService.getLastPromptTokenCount(),
|
||||
});
|
||||
|
||||
return {
|
||||
...prevState,
|
||||
cumulative: newCumulative,
|
||||
currentTurn: newCurrentTurn,
|
||||
currentResponse: newCurrentResponse,
|
||||
};
|
||||
});
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const startNewTurn = useCallback(() => {
|
||||
setStats((prevState) => ({
|
||||
...prevState,
|
||||
cumulative: {
|
||||
...prevState.cumulative,
|
||||
turnCount: prevState.cumulative.turnCount + 1,
|
||||
},
|
||||
currentTurn: {
|
||||
turnCount: 0, // Reset for the new turn's accumulation.
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
},
|
||||
currentResponse: {
|
||||
turnCount: 0,
|
||||
promptTokenCount: 0,
|
||||
candidatesTokenCount: 0,
|
||||
totalTokenCount: 0,
|
||||
cachedContentTokenCount: 0,
|
||||
toolUsePromptTokenCount: 0,
|
||||
thoughtsTokenCount: 0,
|
||||
apiTimeMs: 0,
|
||||
},
|
||||
}));
|
||||
return () => {
|
||||
uiTelemetryService.off('update', handleUpdate);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const value = useMemo(
|
||||
() => ({
|
||||
stats,
|
||||
startNewTurn,
|
||||
addUsage: aggregateTokens,
|
||||
}),
|
||||
[stats, startNewTurn, aggregateTokens],
|
||||
[stats],
|
||||
);
|
||||
|
||||
return (
|
||||
|
|
|
@ -296,19 +296,9 @@ describe('useSlashCommandProcessor', () => {
|
|||
describe('/stats command', () => {
|
||||
it('should show detailed session statistics', async () => {
|
||||
// Arrange
|
||||
const cumulativeStats = {
|
||||
totalTokenCount: 900,
|
||||
promptTokenCount: 200,
|
||||
candidatesTokenCount: 400,
|
||||
cachedContentTokenCount: 100,
|
||||
turnCount: 1,
|
||||
toolUsePromptTokenCount: 50,
|
||||
thoughtsTokenCount: 150,
|
||||
};
|
||||
mockUseSessionStats.mockReturnValue({
|
||||
stats: {
|
||||
sessionStartTime: new Date('2025-01-01T00:00:00.000Z'),
|
||||
cumulative: cumulativeStats,
|
||||
},
|
||||
});
|
||||
|
||||
|
@ -326,7 +316,6 @@ describe('useSlashCommandProcessor', () => {
|
|||
2, // Called after the user message
|
||||
expect.objectContaining({
|
||||
type: MessageType.STATS,
|
||||
stats: cumulativeStats,
|
||||
duration: '1h 2m 3s',
|
||||
}),
|
||||
expect.any(Number),
|
||||
|
@ -334,6 +323,44 @@ describe('useSlashCommandProcessor', () => {
|
|||
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('should show model-specific statistics when using /stats model', async () => {
|
||||
// Arrange
|
||||
const { handleSlashCommand } = getProcessor();
|
||||
|
||||
// Act
|
||||
await act(async () => {
|
||||
handleSlashCommand('/stats model');
|
||||
});
|
||||
|
||||
// Assert
|
||||
expect(mockAddItem).toHaveBeenNthCalledWith(
|
||||
2, // Called after the user message
|
||||
expect.objectContaining({
|
||||
type: MessageType.MODEL_STATS,
|
||||
}),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it('should show tool-specific statistics when using /stats tools', async () => {
|
||||
// Arrange
|
||||
const { handleSlashCommand } = getProcessor();
|
||||
|
||||
// Act
|
||||
await act(async () => {
|
||||
handleSlashCommand('/stats tools');
|
||||
});
|
||||
|
||||
// Assert
|
||||
expect(mockAddItem).toHaveBeenNthCalledWith(
|
||||
2, // Called after the user message
|
||||
expect.objectContaining({
|
||||
type: MessageType.TOOL_STATS,
|
||||
}),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('/about command', () => {
|
||||
|
@ -598,7 +625,6 @@ describe('useSlashCommandProcessor', () => {
|
|||
},
|
||||
{
|
||||
type: 'quit',
|
||||
stats: expect.any(Object),
|
||||
duration: '1h 2m 3s',
|
||||
id: expect.any(Number),
|
||||
},
|
||||
|
|
|
@ -110,14 +110,19 @@ export const useSlashCommandProcessor = (
|
|||
} else if (message.type === MessageType.STATS) {
|
||||
historyItemContent = {
|
||||
type: 'stats',
|
||||
stats: message.stats,
|
||||
lastTurnStats: message.lastTurnStats,
|
||||
duration: message.duration,
|
||||
};
|
||||
} else if (message.type === MessageType.MODEL_STATS) {
|
||||
historyItemContent = {
|
||||
type: 'model_stats',
|
||||
};
|
||||
} else if (message.type === MessageType.TOOL_STATS) {
|
||||
historyItemContent = {
|
||||
type: 'tool_stats',
|
||||
};
|
||||
} else if (message.type === MessageType.QUIT) {
|
||||
historyItemContent = {
|
||||
type: 'quit',
|
||||
stats: message.stats,
|
||||
duration: message.duration,
|
||||
};
|
||||
} else if (message.type === MessageType.COMPRESSION) {
|
||||
|
@ -262,16 +267,28 @@ export const useSlashCommandProcessor = (
|
|||
{
|
||||
name: 'stats',
|
||||
altName: 'usage',
|
||||
description: 'check session stats',
|
||||
action: (_mainCommand, _subCommand, _args) => {
|
||||
description: 'check session stats. Usage: /stats [model|tools]',
|
||||
action: (_mainCommand, subCommand, _args) => {
|
||||
if (subCommand === 'model') {
|
||||
addMessage({
|
||||
type: MessageType.MODEL_STATS,
|
||||
timestamp: new Date(),
|
||||
});
|
||||
return;
|
||||
} else if (subCommand === 'tools') {
|
||||
addMessage({
|
||||
type: MessageType.TOOL_STATS,
|
||||
timestamp: new Date(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const { sessionStartTime, cumulative, currentTurn } = session.stats;
|
||||
const { sessionStartTime } = session.stats;
|
||||
const wallDuration = now.getTime() - sessionStartTime.getTime();
|
||||
|
||||
addMessage({
|
||||
type: MessageType.STATS,
|
||||
stats: cumulative,
|
||||
lastTurnStats: currentTurn,
|
||||
duration: formatDuration(wallDuration),
|
||||
timestamp: new Date(),
|
||||
});
|
||||
|
@ -805,7 +822,7 @@ export const useSlashCommandProcessor = (
|
|||
description: 'exit the cli',
|
||||
action: async (mainCommand, _subCommand, _args) => {
|
||||
const now = new Date();
|
||||
const { sessionStartTime, cumulative } = session.stats;
|
||||
const { sessionStartTime } = session.stats;
|
||||
const wallDuration = now.getTime() - sessionStartTime.getTime();
|
||||
|
||||
setQuittingMessages([
|
||||
|
@ -816,7 +833,6 @@ export const useSlashCommandProcessor = (
|
|||
},
|
||||
{
|
||||
type: 'quit',
|
||||
stats: cumulative,
|
||||
duration: formatDuration(wallDuration),
|
||||
id: now.getTime(),
|
||||
},
|
||||
|
|
|
@ -604,78 +604,6 @@ describe('useGeminiStream', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('Session Stats Integration', () => {
|
||||
it('should call startNewTurn and addUsage for a simple prompt', async () => {
|
||||
const mockMetadata = { totalTokenCount: 123 };
|
||||
const mockStream = (async function* () {
|
||||
yield { type: 'content', value: 'Response' };
|
||||
yield { type: 'usage_metadata', value: mockMetadata };
|
||||
})();
|
||||
mockSendMessageStream.mockReturnValue(mockStream);
|
||||
|
||||
const { result } = renderTestHook();
|
||||
|
||||
await act(async () => {
|
||||
await result.current.submitQuery('Hello, world!');
|
||||
});
|
||||
|
||||
expect(mockStartNewTurn).toHaveBeenCalledTimes(1);
|
||||
expect(mockAddUsage).toHaveBeenCalledTimes(1);
|
||||
expect(mockAddUsage).toHaveBeenCalledWith(mockMetadata);
|
||||
});
|
||||
|
||||
it('should only call addUsage for a tool continuation prompt', async () => {
|
||||
const mockMetadata = { totalTokenCount: 456 };
|
||||
const mockStream = (async function* () {
|
||||
yield { type: 'content', value: 'Final Answer' };
|
||||
yield { type: 'usage_metadata', value: mockMetadata };
|
||||
})();
|
||||
mockSendMessageStream.mockReturnValue(mockStream);
|
||||
|
||||
const { result } = renderTestHook();
|
||||
|
||||
await act(async () => {
|
||||
await result.current.submitQuery([{ text: 'tool response' }], {
|
||||
isContinuation: true,
|
||||
});
|
||||
});
|
||||
|
||||
expect(mockStartNewTurn).not.toHaveBeenCalled();
|
||||
expect(mockAddUsage).toHaveBeenCalledTimes(1);
|
||||
expect(mockAddUsage).toHaveBeenCalledWith(mockMetadata);
|
||||
});
|
||||
|
||||
it('should not call addUsage if the stream contains no usage metadata', async () => {
|
||||
// Arrange: A stream that yields content but never a usage_metadata event
|
||||
const mockStream = (async function* () {
|
||||
yield { type: 'content', value: 'Some response text' };
|
||||
})();
|
||||
mockSendMessageStream.mockReturnValue(mockStream);
|
||||
|
||||
const { result } = renderTestHook();
|
||||
|
||||
await act(async () => {
|
||||
await result.current.submitQuery('Query with no usage data');
|
||||
});
|
||||
|
||||
expect(mockStartNewTurn).toHaveBeenCalledTimes(1);
|
||||
expect(mockAddUsage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not call startNewTurn for a slash command', async () => {
|
||||
mockHandleSlashCommand.mockReturnValue(true);
|
||||
|
||||
const { result } = renderTestHook();
|
||||
|
||||
await act(async () => {
|
||||
await result.current.submitQuery('/stats');
|
||||
});
|
||||
|
||||
expect(mockStartNewTurn).not.toHaveBeenCalled();
|
||||
expect(mockSendMessageStream).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('should not flicker streaming state to Idle between tool completion and submission', async () => {
|
||||
const toolCallResponseParts: PartListUnion = [
|
||||
{ text: 'tool 1 final response' },
|
||||
|
|
|
@ -51,7 +51,6 @@ import {
|
|||
TrackedCompletedToolCall,
|
||||
TrackedCancelledToolCall,
|
||||
} from './useReactToolScheduler.js';
|
||||
import { useSessionStats } from '../contexts/SessionContext.js';
|
||||
|
||||
export function mergePartListUnions(list: PartListUnion[]): PartListUnion {
|
||||
const resultParts: PartListUnion = [];
|
||||
|
@ -101,7 +100,6 @@ export const useGeminiStream = (
|
|||
useStateAndRef<HistoryItemWithoutId | null>(null);
|
||||
const processedMemoryToolsRef = useRef<Set<string>>(new Set());
|
||||
const logger = useLogger();
|
||||
const { startNewTurn, addUsage } = useSessionStats();
|
||||
const gitService = useMemo(() => {
|
||||
if (!config.getProjectRoot()) {
|
||||
return;
|
||||
|
@ -461,9 +459,6 @@ export const useGeminiStream = (
|
|||
case ServerGeminiEventType.ChatCompressed:
|
||||
handleChatCompressionEvent(event.value);
|
||||
break;
|
||||
case ServerGeminiEventType.UsageMetadata:
|
||||
addUsage(event.value);
|
||||
break;
|
||||
case ServerGeminiEventType.ToolCallConfirmation:
|
||||
case ServerGeminiEventType.ToolCallResponse:
|
||||
// do nothing
|
||||
|
@ -486,7 +481,6 @@ export const useGeminiStream = (
|
|||
handleErrorEvent,
|
||||
scheduleToolCalls,
|
||||
handleChatCompressionEvent,
|
||||
addUsage,
|
||||
],
|
||||
);
|
||||
|
||||
|
@ -516,10 +510,6 @@ export const useGeminiStream = (
|
|||
return;
|
||||
}
|
||||
|
||||
if (!options?.isContinuation) {
|
||||
startNewTurn();
|
||||
}
|
||||
|
||||
setIsResponding(true);
|
||||
setInitError(null);
|
||||
|
||||
|
@ -568,7 +558,6 @@ export const useGeminiStream = (
|
|||
setPendingHistoryItem,
|
||||
setInitError,
|
||||
geminiClient,
|
||||
startNewTurn,
|
||||
onAuthError,
|
||||
config,
|
||||
],
|
||||
|
|
|
@ -8,7 +8,6 @@ import {
|
|||
ToolCallConfirmationDetails,
|
||||
ToolResultDisplay,
|
||||
} from '@google/gemini-cli-core';
|
||||
import { CumulativeStats } from './contexts/SessionContext.js';
|
||||
|
||||
// Only defining the state enum needed by the UI
|
||||
export enum StreamingState {
|
||||
|
@ -100,14 +99,19 @@ export type HistoryItemAbout = HistoryItemBase & {
|
|||
|
||||
export type HistoryItemStats = HistoryItemBase & {
|
||||
type: 'stats';
|
||||
stats: CumulativeStats;
|
||||
lastTurnStats: CumulativeStats;
|
||||
duration: string;
|
||||
};
|
||||
|
||||
export type HistoryItemModelStats = HistoryItemBase & {
|
||||
type: 'model_stats';
|
||||
};
|
||||
|
||||
export type HistoryItemToolStats = HistoryItemBase & {
|
||||
type: 'tool_stats';
|
||||
};
|
||||
|
||||
export type HistoryItemQuit = HistoryItemBase & {
|
||||
type: 'quit';
|
||||
stats: CumulativeStats;
|
||||
duration: string;
|
||||
};
|
||||
|
||||
|
@ -140,6 +144,8 @@ export type HistoryItemWithoutId =
|
|||
| HistoryItemAbout
|
||||
| HistoryItemToolGroup
|
||||
| HistoryItemStats
|
||||
| HistoryItemModelStats
|
||||
| HistoryItemToolStats
|
||||
| HistoryItemQuit
|
||||
| HistoryItemCompression;
|
||||
|
||||
|
@ -152,6 +158,8 @@ export enum MessageType {
|
|||
USER = 'user',
|
||||
ABOUT = 'about',
|
||||
STATS = 'stats',
|
||||
MODEL_STATS = 'model_stats',
|
||||
TOOL_STATS = 'tool_stats',
|
||||
QUIT = 'quit',
|
||||
GEMINI = 'gemini',
|
||||
COMPRESSION = 'compression',
|
||||
|
@ -178,15 +186,22 @@ export type Message =
|
|||
| {
|
||||
type: MessageType.STATS;
|
||||
timestamp: Date;
|
||||
stats: CumulativeStats;
|
||||
lastTurnStats: CumulativeStats;
|
||||
duration: string;
|
||||
content?: string;
|
||||
}
|
||||
| {
|
||||
type: MessageType.MODEL_STATS;
|
||||
timestamp: Date;
|
||||
content?: string;
|
||||
}
|
||||
| {
|
||||
type: MessageType.TOOL_STATS;
|
||||
timestamp: Date;
|
||||
content?: string;
|
||||
}
|
||||
| {
|
||||
type: MessageType.QUIT;
|
||||
timestamp: Date;
|
||||
stats: CumulativeStats;
|
||||
duration: string;
|
||||
content?: string;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,247 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
calculateAverageLatency,
|
||||
calculateCacheHitRate,
|
||||
calculateErrorRate,
|
||||
computeSessionStats,
|
||||
} from './computeStats.js';
|
||||
import { ModelMetrics, SessionMetrics } from '../contexts/SessionContext.js';
|
||||
|
||||
describe('calculateErrorRate', () => {
|
||||
it('should return 0 if totalRequests is 0', () => {
|
||||
const metrics: ModelMetrics = {
|
||||
api: { totalRequests: 0, totalErrors: 0, totalLatencyMs: 0 },
|
||||
tokens: {
|
||||
prompt: 0,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
};
|
||||
expect(calculateErrorRate(metrics)).toBe(0);
|
||||
});
|
||||
|
||||
it('should calculate the error rate correctly', () => {
|
||||
const metrics: ModelMetrics = {
|
||||
api: { totalRequests: 10, totalErrors: 2, totalLatencyMs: 0 },
|
||||
tokens: {
|
||||
prompt: 0,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
};
|
||||
expect(calculateErrorRate(metrics)).toBe(20);
|
||||
});
|
||||
});
|
||||
|
||||
describe('calculateAverageLatency', () => {
|
||||
it('should return 0 if totalRequests is 0', () => {
|
||||
const metrics: ModelMetrics = {
|
||||
api: { totalRequests: 0, totalErrors: 0, totalLatencyMs: 1000 },
|
||||
tokens: {
|
||||
prompt: 0,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
};
|
||||
expect(calculateAverageLatency(metrics)).toBe(0);
|
||||
});
|
||||
|
||||
it('should calculate the average latency correctly', () => {
|
||||
const metrics: ModelMetrics = {
|
||||
api: { totalRequests: 10, totalErrors: 0, totalLatencyMs: 1500 },
|
||||
tokens: {
|
||||
prompt: 0,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
};
|
||||
expect(calculateAverageLatency(metrics)).toBe(150);
|
||||
});
|
||||
});
|
||||
|
||||
describe('calculateCacheHitRate', () => {
|
||||
it('should return 0 if prompt tokens is 0', () => {
|
||||
const metrics: ModelMetrics = {
|
||||
api: { totalRequests: 0, totalErrors: 0, totalLatencyMs: 0 },
|
||||
tokens: {
|
||||
prompt: 0,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 100,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
};
|
||||
expect(calculateCacheHitRate(metrics)).toBe(0);
|
||||
});
|
||||
|
||||
it('should calculate the cache hit rate correctly', () => {
|
||||
const metrics: ModelMetrics = {
|
||||
api: { totalRequests: 0, totalErrors: 0, totalLatencyMs: 0 },
|
||||
tokens: {
|
||||
prompt: 200,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 50,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
};
|
||||
expect(calculateCacheHitRate(metrics)).toBe(25);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeSessionStats', () => {
|
||||
it('should return all zeros for initial empty metrics', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const result = computeSessionStats(metrics);
|
||||
|
||||
expect(result).toEqual({
|
||||
totalApiTime: 0,
|
||||
totalToolTime: 0,
|
||||
agentActiveTime: 0,
|
||||
apiTimePercent: 0,
|
||||
toolTimePercent: 0,
|
||||
cacheEfficiency: 0,
|
||||
totalDecisions: 0,
|
||||
successRate: 0,
|
||||
agreementRate: 0,
|
||||
totalPromptTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('should correctly calculate API and tool time percentages', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-pro': {
|
||||
api: { totalRequests: 1, totalErrors: 0, totalLatencyMs: 750 },
|
||||
tokens: {
|
||||
prompt: 10,
|
||||
candidates: 10,
|
||||
total: 20,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 1,
|
||||
totalSuccess: 1,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 250,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const result = computeSessionStats(metrics);
|
||||
|
||||
expect(result.totalApiTime).toBe(750);
|
||||
expect(result.totalToolTime).toBe(250);
|
||||
expect(result.agentActiveTime).toBe(1000);
|
||||
expect(result.apiTimePercent).toBe(75);
|
||||
expect(result.toolTimePercent).toBe(25);
|
||||
});
|
||||
|
||||
it('should correctly calculate cache efficiency', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-pro': {
|
||||
api: { totalRequests: 2, totalErrors: 0, totalLatencyMs: 1000 },
|
||||
tokens: {
|
||||
prompt: 150,
|
||||
candidates: 10,
|
||||
total: 160,
|
||||
cached: 50,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const result = computeSessionStats(metrics);
|
||||
|
||||
expect(result.cacheEfficiency).toBeCloseTo(33.33); // 50 / 150
|
||||
});
|
||||
|
||||
it('should correctly calculate success and agreement rates', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 10,
|
||||
totalSuccess: 8,
|
||||
totalFail: 2,
|
||||
totalDurationMs: 1000,
|
||||
totalDecisions: { accept: 6, reject: 2, modify: 2 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const result = computeSessionStats(metrics);
|
||||
|
||||
expect(result.successRate).toBe(80); // 8 / 10
|
||||
expect(result.agreementRate).toBe(60); // 6 / 10
|
||||
});
|
||||
|
||||
it('should handle division by zero gracefully', () => {
|
||||
const metrics: SessionMetrics = {
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: { accept: 0, reject: 0, modify: 0 },
|
||||
byName: {},
|
||||
},
|
||||
};
|
||||
|
||||
const result = computeSessionStats(metrics);
|
||||
|
||||
expect(result.apiTimePercent).toBe(0);
|
||||
expect(result.toolTimePercent).toBe(0);
|
||||
expect(result.cacheEfficiency).toBe(0);
|
||||
expect(result.successRate).toBe(0);
|
||||
expect(result.agreementRate).toBe(0);
|
||||
});
|
||||
});
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {
|
||||
SessionMetrics,
|
||||
ComputedSessionStats,
|
||||
ModelMetrics,
|
||||
} from '../contexts/SessionContext.js';
|
||||
|
||||
export function calculateErrorRate(metrics: ModelMetrics): number {
|
||||
if (metrics.api.totalRequests === 0) {
|
||||
return 0;
|
||||
}
|
||||
return (metrics.api.totalErrors / metrics.api.totalRequests) * 100;
|
||||
}
|
||||
|
||||
export function calculateAverageLatency(metrics: ModelMetrics): number {
|
||||
if (metrics.api.totalRequests === 0) {
|
||||
return 0;
|
||||
}
|
||||
return metrics.api.totalLatencyMs / metrics.api.totalRequests;
|
||||
}
|
||||
|
||||
export function calculateCacheHitRate(metrics: ModelMetrics): number {
|
||||
if (metrics.tokens.prompt === 0) {
|
||||
return 0;
|
||||
}
|
||||
return (metrics.tokens.cached / metrics.tokens.prompt) * 100;
|
||||
}
|
||||
|
||||
export const computeSessionStats = (
|
||||
metrics: SessionMetrics,
|
||||
): ComputedSessionStats => {
|
||||
const { models, tools } = metrics;
|
||||
const totalApiTime = Object.values(models).reduce(
|
||||
(acc, model) => acc + model.api.totalLatencyMs,
|
||||
0,
|
||||
);
|
||||
const totalToolTime = tools.totalDurationMs;
|
||||
const agentActiveTime = totalApiTime + totalToolTime;
|
||||
const apiTimePercent =
|
||||
agentActiveTime > 0 ? (totalApiTime / agentActiveTime) * 100 : 0;
|
||||
const toolTimePercent =
|
||||
agentActiveTime > 0 ? (totalToolTime / agentActiveTime) * 100 : 0;
|
||||
|
||||
const totalCachedTokens = Object.values(models).reduce(
|
||||
(acc, model) => acc + model.tokens.cached,
|
||||
0,
|
||||
);
|
||||
const totalPromptTokens = Object.values(models).reduce(
|
||||
(acc, model) => acc + model.tokens.prompt,
|
||||
0,
|
||||
);
|
||||
const cacheEfficiency =
|
||||
totalPromptTokens > 0 ? (totalCachedTokens / totalPromptTokens) * 100 : 0;
|
||||
|
||||
const totalDecisions =
|
||||
tools.totalDecisions.accept +
|
||||
tools.totalDecisions.reject +
|
||||
tools.totalDecisions.modify;
|
||||
const successRate =
|
||||
tools.totalCalls > 0 ? (tools.totalSuccess / tools.totalCalls) * 100 : 0;
|
||||
const agreementRate =
|
||||
totalDecisions > 0
|
||||
? (tools.totalDecisions.accept / totalDecisions) * 100
|
||||
: 0;
|
||||
|
||||
return {
|
||||
totalApiTime,
|
||||
totalToolTime,
|
||||
agentActiveTime,
|
||||
apiTimePercent,
|
||||
toolTimePercent,
|
||||
cacheEfficiency,
|
||||
totalDecisions,
|
||||
successRate,
|
||||
agreementRate,
|
||||
totalCachedTokens,
|
||||
totalPromptTokens,
|
||||
};
|
||||
};
|
|
@ -0,0 +1,58 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
getStatusColor,
|
||||
TOOL_SUCCESS_RATE_HIGH,
|
||||
TOOL_SUCCESS_RATE_MEDIUM,
|
||||
USER_AGREEMENT_RATE_HIGH,
|
||||
USER_AGREEMENT_RATE_MEDIUM,
|
||||
CACHE_EFFICIENCY_HIGH,
|
||||
CACHE_EFFICIENCY_MEDIUM,
|
||||
} from './displayUtils.js';
|
||||
import { Colors } from '../colors.js';
|
||||
|
||||
describe('displayUtils', () => {
|
||||
describe('getStatusColor', () => {
|
||||
const thresholds = {
|
||||
green: 80,
|
||||
yellow: 50,
|
||||
};
|
||||
|
||||
it('should return green for values >= green threshold', () => {
|
||||
expect(getStatusColor(90, thresholds)).toBe(Colors.AccentGreen);
|
||||
expect(getStatusColor(80, thresholds)).toBe(Colors.AccentGreen);
|
||||
});
|
||||
|
||||
it('should return yellow for values < green and >= yellow threshold', () => {
|
||||
expect(getStatusColor(79, thresholds)).toBe(Colors.AccentYellow);
|
||||
expect(getStatusColor(50, thresholds)).toBe(Colors.AccentYellow);
|
||||
});
|
||||
|
||||
it('should return red for values < yellow threshold', () => {
|
||||
expect(getStatusColor(49, thresholds)).toBe(Colors.AccentRed);
|
||||
expect(getStatusColor(0, thresholds)).toBe(Colors.AccentRed);
|
||||
});
|
||||
|
||||
it('should return defaultColor for values < yellow threshold when provided', () => {
|
||||
expect(
|
||||
getStatusColor(49, thresholds, { defaultColor: Colors.Foreground }),
|
||||
).toBe(Colors.Foreground);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Threshold Constants', () => {
|
||||
it('should have the correct values', () => {
|
||||
expect(TOOL_SUCCESS_RATE_HIGH).toBe(95);
|
||||
expect(TOOL_SUCCESS_RATE_MEDIUM).toBe(85);
|
||||
expect(USER_AGREEMENT_RATE_HIGH).toBe(75);
|
||||
expect(USER_AGREEMENT_RATE_MEDIUM).toBe(45);
|
||||
expect(CACHE_EFFICIENCY_HIGH).toBe(40);
|
||||
expect(CACHE_EFFICIENCY_MEDIUM).toBe(15);
|
||||
});
|
||||
});
|
||||
});
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { Colors } from '../colors.js';
|
||||
|
||||
// --- Thresholds ---
|
||||
export const TOOL_SUCCESS_RATE_HIGH = 95;
|
||||
export const TOOL_SUCCESS_RATE_MEDIUM = 85;
|
||||
|
||||
export const USER_AGREEMENT_RATE_HIGH = 75;
|
||||
export const USER_AGREEMENT_RATE_MEDIUM = 45;
|
||||
|
||||
export const CACHE_EFFICIENCY_HIGH = 40;
|
||||
export const CACHE_EFFICIENCY_MEDIUM = 15;
|
||||
|
||||
// --- Color Logic ---
|
||||
export const getStatusColor = (
|
||||
value: number,
|
||||
thresholds: { green: number; yellow: number },
|
||||
options: { defaultColor?: string } = {},
|
||||
) => {
|
||||
if (value >= thresholds.green) {
|
||||
return Colors.AccentGreen;
|
||||
}
|
||||
if (value >= thresholds.yellow) {
|
||||
return Colors.AccentYellow;
|
||||
}
|
||||
return options.defaultColor || Colors.AccentRed;
|
||||
};
|
|
@ -27,7 +27,7 @@ export const formatDuration = (milliseconds: number): string => {
|
|||
}
|
||||
|
||||
if (milliseconds < 1000) {
|
||||
return `${milliseconds}ms`;
|
||||
return `${Math.round(milliseconds)}ms`;
|
||||
}
|
||||
|
||||
const totalSeconds = milliseconds / 1000;
|
||||
|
|
|
@ -10,14 +10,8 @@ import {
|
|||
GeminiEventType,
|
||||
ServerGeminiToolCallRequestEvent,
|
||||
ServerGeminiErrorEvent,
|
||||
ServerGeminiUsageMetadataEvent,
|
||||
} from './turn.js';
|
||||
import {
|
||||
GenerateContentResponse,
|
||||
Part,
|
||||
Content,
|
||||
GenerateContentResponseUsageMetadata,
|
||||
} from '@google/genai';
|
||||
import { GenerateContentResponse, Part, Content } from '@google/genai';
|
||||
import { reportError } from '../utils/errorReporting.js';
|
||||
import { GeminiChat } from './geminiChat.js';
|
||||
|
||||
|
@ -55,24 +49,6 @@ describe('Turn', () => {
|
|||
};
|
||||
let mockChatInstance: MockedChatInstance;
|
||||
|
||||
const mockMetadata1: GenerateContentResponseUsageMetadata = {
|
||||
promptTokenCount: 10,
|
||||
candidatesTokenCount: 20,
|
||||
totalTokenCount: 30,
|
||||
cachedContentTokenCount: 5,
|
||||
toolUsePromptTokenCount: 2,
|
||||
thoughtsTokenCount: 3,
|
||||
};
|
||||
|
||||
const mockMetadata2: GenerateContentResponseUsageMetadata = {
|
||||
promptTokenCount: 100,
|
||||
candidatesTokenCount: 200,
|
||||
totalTokenCount: 300,
|
||||
cachedContentTokenCount: 50,
|
||||
toolUsePromptTokenCount: 20,
|
||||
thoughtsTokenCount: 30,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mockChatInstance = {
|
||||
|
@ -245,46 +221,6 @@ describe('Turn', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should yield the last UsageMetadata event from the stream', async () => {
|
||||
const mockResponseStream = (async function* () {
|
||||
yield {
|
||||
candidates: [{ content: { parts: [{ text: 'First response' }] } }],
|
||||
usageMetadata: mockMetadata1,
|
||||
} as unknown as GenerateContentResponse;
|
||||
// Add a small delay to ensure apiTimeMs is > 0
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
yield {
|
||||
functionCalls: [{ name: 'aTool' }],
|
||||
usageMetadata: mockMetadata2,
|
||||
} as unknown as GenerateContentResponse;
|
||||
})();
|
||||
mockSendMessageStream.mockResolvedValue(mockResponseStream);
|
||||
|
||||
const events = [];
|
||||
const reqParts: Part[] = [{ text: 'Test metadata' }];
|
||||
for await (const event of turn.run(
|
||||
reqParts,
|
||||
new AbortController().signal,
|
||||
)) {
|
||||
events.push(event);
|
||||
}
|
||||
|
||||
// There should be a content event, a tool call, and our metadata event
|
||||
expect(events.length).toBe(3);
|
||||
|
||||
const metadataEvent = events[2] as ServerGeminiUsageMetadataEvent;
|
||||
expect(metadataEvent.type).toBe(GeminiEventType.UsageMetadata);
|
||||
|
||||
// The value should be the *last* metadata object received.
|
||||
expect(metadataEvent.value).toEqual(
|
||||
expect.objectContaining(mockMetadata2),
|
||||
);
|
||||
expect(metadataEvent.value.apiTimeMs).toBeGreaterThan(0);
|
||||
|
||||
// Also check the public getter
|
||||
expect(turn.getUsageMetadata()).toEqual(mockMetadata2);
|
||||
});
|
||||
|
||||
it('should handle function calls with undefined name or args', async () => {
|
||||
const mockResponseStream = (async function* () {
|
||||
yield {
|
||||
|
|
|
@ -9,7 +9,6 @@ import {
|
|||
GenerateContentResponse,
|
||||
FunctionCall,
|
||||
FunctionDeclaration,
|
||||
GenerateContentResponseUsageMetadata,
|
||||
} from '@google/genai';
|
||||
import {
|
||||
ToolCallConfirmationDetails,
|
||||
|
@ -48,7 +47,6 @@ export enum GeminiEventType {
|
|||
UserCancelled = 'user_cancelled',
|
||||
Error = 'error',
|
||||
ChatCompressed = 'chat_compressed',
|
||||
UsageMetadata = 'usage_metadata',
|
||||
Thought = 'thought',
|
||||
}
|
||||
|
||||
|
@ -129,11 +127,6 @@ export type ServerGeminiChatCompressedEvent = {
|
|||
value: ChatCompressionInfo | null;
|
||||
};
|
||||
|
||||
export type ServerGeminiUsageMetadataEvent = {
|
||||
type: GeminiEventType.UsageMetadata;
|
||||
value: GenerateContentResponseUsageMetadata & { apiTimeMs?: number };
|
||||
};
|
||||
|
||||
// The original union type, now composed of the individual types
|
||||
export type ServerGeminiStreamEvent =
|
||||
| ServerGeminiContentEvent
|
||||
|
@ -143,14 +136,12 @@ export type ServerGeminiStreamEvent =
|
|||
| ServerGeminiUserCancelledEvent
|
||||
| ServerGeminiErrorEvent
|
||||
| ServerGeminiChatCompressedEvent
|
||||
| ServerGeminiUsageMetadataEvent
|
||||
| ServerGeminiThoughtEvent;
|
||||
|
||||
// A turn manages the agentic loop turn within the server context.
|
||||
export class Turn {
|
||||
readonly pendingToolCalls: ToolCallRequestInfo[];
|
||||
private debugResponses: GenerateContentResponse[];
|
||||
private lastUsageMetadata: GenerateContentResponseUsageMetadata | null = null;
|
||||
|
||||
constructor(private readonly chat: GeminiChat) {
|
||||
this.pendingToolCalls = [];
|
||||
|
@ -161,7 +152,6 @@ export class Turn {
|
|||
req: PartListUnion,
|
||||
signal: AbortSignal,
|
||||
): AsyncGenerator<ServerGeminiStreamEvent> {
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
const responseStream = await this.chat.sendMessageStream({
|
||||
message: req,
|
||||
|
@ -213,19 +203,6 @@ export class Turn {
|
|||
yield event;
|
||||
}
|
||||
}
|
||||
|
||||
if (resp.usageMetadata) {
|
||||
this.lastUsageMetadata =
|
||||
resp.usageMetadata as GenerateContentResponseUsageMetadata;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.lastUsageMetadata) {
|
||||
const durationMs = Date.now() - startTime;
|
||||
yield {
|
||||
type: GeminiEventType.UsageMetadata,
|
||||
value: { ...this.lastUsageMetadata, apiTimeMs: durationMs },
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
const error = toFriendlyError(e);
|
||||
|
@ -286,8 +263,4 @@ export class Turn {
|
|||
getDebugResponses(): GenerateContentResponse[] {
|
||||
return this.debugResponses;
|
||||
}
|
||||
|
||||
getUsageMetadata(): GenerateContentResponseUsageMetadata | null {
|
||||
return this.lastUsageMetadata;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -38,3 +38,4 @@ export {
|
|||
} from './types.js';
|
||||
export { SpanStatusCode, ValueType } from '@opentelemetry/api';
|
||||
export { SemanticAttributes } from '@opentelemetry/semantic-conventions';
|
||||
export * from './uiTelemetry.js';
|
||||
|
|
|
@ -43,15 +43,22 @@ import * as metrics from './metrics.js';
|
|||
import * as sdk from './sdk.js';
|
||||
import { vi, describe, beforeEach, it, expect } from 'vitest';
|
||||
import { GenerateContentResponseUsageMetadata } from '@google/genai';
|
||||
import * as uiTelemetry from './uiTelemetry.js';
|
||||
|
||||
describe('loggers', () => {
|
||||
const mockLogger = {
|
||||
emit: vi.fn(),
|
||||
};
|
||||
const mockUiEvent = {
|
||||
addEvent: vi.fn(),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.spyOn(sdk, 'isTelemetrySdkInitialized').mockReturnValue(true);
|
||||
vi.spyOn(logs, 'getLogger').mockReturnValue(mockLogger);
|
||||
vi.spyOn(uiTelemetry.uiTelemetryService, 'addEvent').mockImplementation(
|
||||
mockUiEvent.addEvent,
|
||||
);
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z'));
|
||||
});
|
||||
|
@ -215,6 +222,7 @@ describe('loggers', () => {
|
|||
cached_content_token_count: 10,
|
||||
thoughts_token_count: 5,
|
||||
tool_token_count: 2,
|
||||
total_token_count: 0,
|
||||
response_text: 'test-response',
|
||||
},
|
||||
});
|
||||
|
@ -233,6 +241,12 @@ describe('loggers', () => {
|
|||
50,
|
||||
'output',
|
||||
);
|
||||
|
||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||
...event,
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('should log an API response with an error', () => {
|
||||
|
@ -263,6 +277,12 @@ describe('loggers', () => {
|
|||
'error.message': 'test-error',
|
||||
},
|
||||
});
|
||||
|
||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||
...event,
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
@ -417,6 +437,12 @@ describe('loggers', () => {
|
|||
true,
|
||||
ToolCallDecision.ACCEPT,
|
||||
);
|
||||
|
||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||
...event,
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
});
|
||||
});
|
||||
it('should log a tool call with a reject decision', () => {
|
||||
const call: ErroredToolCall = {
|
||||
|
@ -471,6 +497,12 @@ describe('loggers', () => {
|
|||
false,
|
||||
ToolCallDecision.REJECT,
|
||||
);
|
||||
|
||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||
...event,
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('should log a tool call with a modify decision', () => {
|
||||
|
@ -527,6 +559,12 @@ describe('loggers', () => {
|
|||
true,
|
||||
ToolCallDecision.MODIFY,
|
||||
);
|
||||
|
||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||
...event,
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('should log a tool call without a decision', () => {
|
||||
|
@ -581,6 +619,12 @@ describe('loggers', () => {
|
|||
true,
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||
...event,
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('should log a failed tool call with an error', () => {
|
||||
|
@ -641,6 +685,12 @@ describe('loggers', () => {
|
|||
false,
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
|
||||
...event,
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
'event.timestamp': '2025-01-01T00:00:00.000Z',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -31,6 +31,7 @@ import {
|
|||
recordToolCallMetrics,
|
||||
} from './metrics.js';
|
||||
import { isTelemetrySdkInitialized } from './sdk.js';
|
||||
import { uiTelemetryService, UiEvent } from './uiTelemetry.js';
|
||||
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
|
||||
|
||||
const shouldLogUserPrompts = (config: Config): boolean =>
|
||||
|
@ -98,6 +99,12 @@ export function logUserPrompt(config: Config, event: UserPromptEvent): void {
|
|||
}
|
||||
|
||||
export function logToolCall(config: Config, event: ToolCallEvent): void {
|
||||
const uiEvent = {
|
||||
...event,
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
'event.timestamp': new Date().toISOString(),
|
||||
} as UiEvent;
|
||||
uiTelemetryService.addEvent(uiEvent);
|
||||
ClearcutLogger.getInstance(config)?.logToolCallEvent(event);
|
||||
if (!isTelemetrySdkInitialized()) return;
|
||||
|
||||
|
@ -150,6 +157,12 @@ export function logApiRequest(config: Config, event: ApiRequestEvent): void {
|
|||
}
|
||||
|
||||
export function logApiError(config: Config, event: ApiErrorEvent): void {
|
||||
const uiEvent = {
|
||||
...event,
|
||||
'event.name': EVENT_API_ERROR,
|
||||
'event.timestamp': new Date().toISOString(),
|
||||
} as UiEvent;
|
||||
uiTelemetryService.addEvent(uiEvent);
|
||||
ClearcutLogger.getInstance(config)?.logApiErrorEvent(event);
|
||||
if (!isTelemetrySdkInitialized()) return;
|
||||
|
||||
|
@ -186,6 +199,12 @@ export function logApiError(config: Config, event: ApiErrorEvent): void {
|
|||
}
|
||||
|
||||
export function logApiResponse(config: Config, event: ApiResponseEvent): void {
|
||||
const uiEvent = {
|
||||
...event,
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
'event.timestamp': new Date().toISOString(),
|
||||
} as UiEvent;
|
||||
uiTelemetryService.addEvent(uiEvent);
|
||||
ClearcutLogger.getInstance(config)?.logApiResponseEvent(event);
|
||||
if (!isTelemetrySdkInitialized()) return;
|
||||
const attributes: LogAttributes = {
|
||||
|
|
|
@ -183,6 +183,7 @@ export class ApiResponseEvent {
|
|||
cached_content_token_count: number;
|
||||
thoughts_token_count: number;
|
||||
tool_token_count: number;
|
||||
total_token_count: number;
|
||||
response_text?: string;
|
||||
|
||||
constructor(
|
||||
|
@ -202,6 +203,7 @@ export class ApiResponseEvent {
|
|||
this.cached_content_token_count = usage_data?.cachedContentTokenCount ?? 0;
|
||||
this.thoughts_token_count = usage_data?.thoughtsTokenCount ?? 0;
|
||||
this.tool_token_count = usage_data?.toolUsePromptTokenCount ?? 0;
|
||||
this.total_token_count = usage_data?.totalTokenCount ?? 0;
|
||||
this.response_text = response_text;
|
||||
this.error = error;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,510 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { UiTelemetryService } from './uiTelemetry.js';
|
||||
import {
|
||||
ApiErrorEvent,
|
||||
ApiResponseEvent,
|
||||
ToolCallEvent,
|
||||
ToolCallDecision,
|
||||
} from './types.js';
|
||||
import {
|
||||
EVENT_API_ERROR,
|
||||
EVENT_API_RESPONSE,
|
||||
EVENT_TOOL_CALL,
|
||||
} from './constants.js';
|
||||
import {
|
||||
CompletedToolCall,
|
||||
ErroredToolCall,
|
||||
SuccessfulToolCall,
|
||||
} from '../core/coreToolScheduler.js';
|
||||
import { Tool, ToolConfirmationOutcome } from '../tools/tools.js';
|
||||
|
||||
const createFakeCompletedToolCall = (
|
||||
name: string,
|
||||
success: boolean,
|
||||
duration = 100,
|
||||
outcome?: ToolConfirmationOutcome,
|
||||
error?: Error,
|
||||
): CompletedToolCall => {
|
||||
const request = {
|
||||
callId: `call_${name}_${Date.now()}`,
|
||||
name,
|
||||
args: { foo: 'bar' },
|
||||
isClientInitiated: false,
|
||||
};
|
||||
|
||||
if (success) {
|
||||
return {
|
||||
status: 'success',
|
||||
request,
|
||||
tool: { name } as Tool, // Mock tool
|
||||
response: {
|
||||
callId: request.callId,
|
||||
responseParts: {
|
||||
functionResponse: {
|
||||
id: request.callId,
|
||||
name,
|
||||
response: { output: 'Success!' },
|
||||
},
|
||||
},
|
||||
error: undefined,
|
||||
resultDisplay: 'Success!',
|
||||
},
|
||||
durationMs: duration,
|
||||
outcome,
|
||||
} as SuccessfulToolCall;
|
||||
} else {
|
||||
return {
|
||||
status: 'error',
|
||||
request,
|
||||
response: {
|
||||
callId: request.callId,
|
||||
responseParts: {
|
||||
functionResponse: {
|
||||
id: request.callId,
|
||||
name,
|
||||
response: { error: 'Tool failed' },
|
||||
},
|
||||
},
|
||||
error: error || new Error('Tool failed'),
|
||||
resultDisplay: 'Failure!',
|
||||
},
|
||||
durationMs: duration,
|
||||
outcome,
|
||||
} as ErroredToolCall;
|
||||
}
|
||||
};
|
||||
|
||||
describe('UiTelemetryService', () => {
|
||||
let service: UiTelemetryService;
|
||||
|
||||
beforeEach(() => {
|
||||
service = new UiTelemetryService();
|
||||
});
|
||||
|
||||
it('should have correct initial metrics', () => {
|
||||
const metrics = service.getMetrics();
|
||||
expect(metrics).toEqual({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: {
|
||||
[ToolCallDecision.ACCEPT]: 0,
|
||||
[ToolCallDecision.REJECT]: 0,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
},
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
expect(service.getLastPromptTokenCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('should emit an update event when an event is added', () => {
|
||||
const spy = vi.fn();
|
||||
service.on('update', spy);
|
||||
|
||||
const event = {
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 500,
|
||||
input_token_count: 10,
|
||||
output_token_count: 20,
|
||||
total_token_count: 30,
|
||||
cached_content_token_count: 5,
|
||||
thoughts_token_count: 2,
|
||||
tool_token_count: 3,
|
||||
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
|
||||
|
||||
service.addEvent(event);
|
||||
|
||||
expect(spy).toHaveBeenCalledOnce();
|
||||
const { metrics, lastPromptTokenCount } = spy.mock.calls[0][0];
|
||||
expect(metrics).toBeDefined();
|
||||
expect(lastPromptTokenCount).toBe(10);
|
||||
});
|
||||
|
||||
describe('API Response Event Processing', () => {
|
||||
it('should process a single ApiResponseEvent', () => {
|
||||
const event = {
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 500,
|
||||
input_token_count: 10,
|
||||
output_token_count: 20,
|
||||
total_token_count: 30,
|
||||
cached_content_token_count: 5,
|
||||
thoughts_token_count: 2,
|
||||
tool_token_count: 3,
|
||||
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
|
||||
|
||||
service.addEvent(event);
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
expect(metrics.models['gemini-2.5-pro']).toEqual({
|
||||
api: {
|
||||
totalRequests: 1,
|
||||
totalErrors: 0,
|
||||
totalLatencyMs: 500,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 10,
|
||||
candidates: 20,
|
||||
total: 30,
|
||||
cached: 5,
|
||||
thoughts: 2,
|
||||
tool: 3,
|
||||
},
|
||||
});
|
||||
expect(service.getLastPromptTokenCount()).toBe(10);
|
||||
});
|
||||
|
||||
it('should aggregate multiple ApiResponseEvents for the same model', () => {
|
||||
const event1 = {
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 500,
|
||||
input_token_count: 10,
|
||||
output_token_count: 20,
|
||||
total_token_count: 30,
|
||||
cached_content_token_count: 5,
|
||||
thoughts_token_count: 2,
|
||||
tool_token_count: 3,
|
||||
} as ApiResponseEvent & {
|
||||
'event.name': typeof EVENT_API_RESPONSE;
|
||||
};
|
||||
const event2 = {
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 600,
|
||||
input_token_count: 15,
|
||||
output_token_count: 25,
|
||||
total_token_count: 40,
|
||||
cached_content_token_count: 10,
|
||||
thoughts_token_count: 4,
|
||||
tool_token_count: 6,
|
||||
} as ApiResponseEvent & {
|
||||
'event.name': typeof EVENT_API_RESPONSE;
|
||||
};
|
||||
|
||||
service.addEvent(event1);
|
||||
service.addEvent(event2);
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
expect(metrics.models['gemini-2.5-pro']).toEqual({
|
||||
api: {
|
||||
totalRequests: 2,
|
||||
totalErrors: 0,
|
||||
totalLatencyMs: 1100,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 25,
|
||||
candidates: 45,
|
||||
total: 70,
|
||||
cached: 15,
|
||||
thoughts: 6,
|
||||
tool: 9,
|
||||
},
|
||||
});
|
||||
expect(service.getLastPromptTokenCount()).toBe(15);
|
||||
});
|
||||
|
||||
it('should handle ApiResponseEvents for different models', () => {
|
||||
const event1 = {
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 500,
|
||||
input_token_count: 10,
|
||||
output_token_count: 20,
|
||||
total_token_count: 30,
|
||||
cached_content_token_count: 5,
|
||||
thoughts_token_count: 2,
|
||||
tool_token_count: 3,
|
||||
} as ApiResponseEvent & {
|
||||
'event.name': typeof EVENT_API_RESPONSE;
|
||||
};
|
||||
const event2 = {
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
model: 'gemini-2.5-flash',
|
||||
duration_ms: 1000,
|
||||
input_token_count: 100,
|
||||
output_token_count: 200,
|
||||
total_token_count: 300,
|
||||
cached_content_token_count: 50,
|
||||
thoughts_token_count: 20,
|
||||
tool_token_count: 30,
|
||||
} as ApiResponseEvent & {
|
||||
'event.name': typeof EVENT_API_RESPONSE;
|
||||
};
|
||||
|
||||
service.addEvent(event1);
|
||||
service.addEvent(event2);
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
expect(metrics.models['gemini-2.5-pro']).toBeDefined();
|
||||
expect(metrics.models['gemini-2.5-flash']).toBeDefined();
|
||||
expect(metrics.models['gemini-2.5-pro'].api.totalRequests).toBe(1);
|
||||
expect(metrics.models['gemini-2.5-flash'].api.totalRequests).toBe(1);
|
||||
expect(service.getLastPromptTokenCount()).toBe(100);
|
||||
});
|
||||
});
|
||||
|
||||
describe('API Error Event Processing', () => {
|
||||
it('should process a single ApiErrorEvent', () => {
|
||||
const event = {
|
||||
'event.name': EVENT_API_ERROR,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 300,
|
||||
error: 'Something went wrong',
|
||||
} as ApiErrorEvent & { 'event.name': typeof EVENT_API_ERROR };
|
||||
|
||||
service.addEvent(event);
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
expect(metrics.models['gemini-2.5-pro']).toEqual({
|
||||
api: {
|
||||
totalRequests: 1,
|
||||
totalErrors: 1,
|
||||
totalLatencyMs: 300,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 0,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should aggregate ApiErrorEvents and ApiResponseEvents', () => {
|
||||
const responseEvent = {
|
||||
'event.name': EVENT_API_RESPONSE,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 500,
|
||||
input_token_count: 10,
|
||||
output_token_count: 20,
|
||||
total_token_count: 30,
|
||||
cached_content_token_count: 5,
|
||||
thoughts_token_count: 2,
|
||||
tool_token_count: 3,
|
||||
} as ApiResponseEvent & {
|
||||
'event.name': typeof EVENT_API_RESPONSE;
|
||||
};
|
||||
const errorEvent = {
|
||||
'event.name': EVENT_API_ERROR,
|
||||
model: 'gemini-2.5-pro',
|
||||
duration_ms: 300,
|
||||
error: 'Something went wrong',
|
||||
} as ApiErrorEvent & { 'event.name': typeof EVENT_API_ERROR };
|
||||
|
||||
service.addEvent(responseEvent);
|
||||
service.addEvent(errorEvent);
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
expect(metrics.models['gemini-2.5-pro']).toEqual({
|
||||
api: {
|
||||
totalRequests: 2,
|
||||
totalErrors: 1,
|
||||
totalLatencyMs: 800,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 10,
|
||||
candidates: 20,
|
||||
total: 30,
|
||||
cached: 5,
|
||||
thoughts: 2,
|
||||
tool: 3,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tool Call Event Processing', () => {
|
||||
it('should process a single successful ToolCallEvent', () => {
|
||||
const toolCall = createFakeCompletedToolCall(
|
||||
'test_tool',
|
||||
true,
|
||||
150,
|
||||
ToolConfirmationOutcome.ProceedOnce,
|
||||
);
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
const { tools } = metrics;
|
||||
|
||||
expect(tools.totalCalls).toBe(1);
|
||||
expect(tools.totalSuccess).toBe(1);
|
||||
expect(tools.totalFail).toBe(0);
|
||||
expect(tools.totalDurationMs).toBe(150);
|
||||
expect(tools.totalDecisions[ToolCallDecision.ACCEPT]).toBe(1);
|
||||
expect(tools.byName['test_tool']).toEqual({
|
||||
count: 1,
|
||||
success: 1,
|
||||
fail: 0,
|
||||
durationMs: 150,
|
||||
decisions: {
|
||||
[ToolCallDecision.ACCEPT]: 1,
|
||||
[ToolCallDecision.REJECT]: 0,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should process a single failed ToolCallEvent', () => {
|
||||
const toolCall = createFakeCompletedToolCall(
|
||||
'test_tool',
|
||||
false,
|
||||
200,
|
||||
ToolConfirmationOutcome.Cancel,
|
||||
);
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
const { tools } = metrics;
|
||||
|
||||
expect(tools.totalCalls).toBe(1);
|
||||
expect(tools.totalSuccess).toBe(0);
|
||||
expect(tools.totalFail).toBe(1);
|
||||
expect(tools.totalDurationMs).toBe(200);
|
||||
expect(tools.totalDecisions[ToolCallDecision.REJECT]).toBe(1);
|
||||
expect(tools.byName['test_tool']).toEqual({
|
||||
count: 1,
|
||||
success: 0,
|
||||
fail: 1,
|
||||
durationMs: 200,
|
||||
decisions: {
|
||||
[ToolCallDecision.ACCEPT]: 0,
|
||||
[ToolCallDecision.REJECT]: 1,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should process a ToolCallEvent with modify decision', () => {
|
||||
const toolCall = createFakeCompletedToolCall(
|
||||
'test_tool',
|
||||
true,
|
||||
250,
|
||||
ToolConfirmationOutcome.ModifyWithEditor,
|
||||
);
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
const { tools } = metrics;
|
||||
|
||||
expect(tools.totalDecisions[ToolCallDecision.MODIFY]).toBe(1);
|
||||
expect(tools.byName['test_tool'].decisions[ToolCallDecision.MODIFY]).toBe(
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
it('should process a ToolCallEvent without a decision', () => {
|
||||
const toolCall = createFakeCompletedToolCall('test_tool', true, 100);
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
const { tools } = metrics;
|
||||
|
||||
expect(tools.totalDecisions).toEqual({
|
||||
[ToolCallDecision.ACCEPT]: 0,
|
||||
[ToolCallDecision.REJECT]: 0,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
});
|
||||
expect(tools.byName['test_tool'].decisions).toEqual({
|
||||
[ToolCallDecision.ACCEPT]: 0,
|
||||
[ToolCallDecision.REJECT]: 0,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('should aggregate multiple ToolCallEvents for the same tool', () => {
|
||||
const toolCall1 = createFakeCompletedToolCall(
|
||||
'test_tool',
|
||||
true,
|
||||
100,
|
||||
ToolConfirmationOutcome.ProceedOnce,
|
||||
);
|
||||
const toolCall2 = createFakeCompletedToolCall(
|
||||
'test_tool',
|
||||
false,
|
||||
150,
|
||||
ToolConfirmationOutcome.Cancel,
|
||||
);
|
||||
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall1))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall2))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
const { tools } = metrics;
|
||||
|
||||
expect(tools.totalCalls).toBe(2);
|
||||
expect(tools.totalSuccess).toBe(1);
|
||||
expect(tools.totalFail).toBe(1);
|
||||
expect(tools.totalDurationMs).toBe(250);
|
||||
expect(tools.totalDecisions[ToolCallDecision.ACCEPT]).toBe(1);
|
||||
expect(tools.totalDecisions[ToolCallDecision.REJECT]).toBe(1);
|
||||
expect(tools.byName['test_tool']).toEqual({
|
||||
count: 2,
|
||||
success: 1,
|
||||
fail: 1,
|
||||
durationMs: 250,
|
||||
decisions: {
|
||||
[ToolCallDecision.ACCEPT]: 1,
|
||||
[ToolCallDecision.REJECT]: 1,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle ToolCallEvents for different tools', () => {
|
||||
const toolCall1 = createFakeCompletedToolCall('tool_A', true, 100);
|
||||
const toolCall2 = createFakeCompletedToolCall('tool_B', false, 200);
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall1))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
service.addEvent({
|
||||
...JSON.parse(JSON.stringify(new ToolCallEvent(toolCall2))),
|
||||
'event.name': EVENT_TOOL_CALL,
|
||||
});
|
||||
|
||||
const metrics = service.getMetrics();
|
||||
const { tools } = metrics;
|
||||
|
||||
expect(tools.totalCalls).toBe(2);
|
||||
expect(tools.totalSuccess).toBe(1);
|
||||
expect(tools.totalFail).toBe(1);
|
||||
expect(tools.byName['tool_A']).toBeDefined();
|
||||
expect(tools.byName['tool_B']).toBeDefined();
|
||||
expect(tools.byName['tool_A'].count).toBe(1);
|
||||
expect(tools.byName['tool_B'].count).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
|
@ -0,0 +1,207 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import {
|
||||
EVENT_API_ERROR,
|
||||
EVENT_API_RESPONSE,
|
||||
EVENT_TOOL_CALL,
|
||||
} from './constants.js';
|
||||
|
||||
import {
|
||||
ApiErrorEvent,
|
||||
ApiResponseEvent,
|
||||
ToolCallEvent,
|
||||
ToolCallDecision,
|
||||
} from './types.js';
|
||||
|
||||
export type UiEvent =
|
||||
| (ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE })
|
||||
| (ApiErrorEvent & { 'event.name': typeof EVENT_API_ERROR })
|
||||
| (ToolCallEvent & { 'event.name': typeof EVENT_TOOL_CALL });
|
||||
|
||||
export interface ToolCallStats {
|
||||
count: number;
|
||||
success: number;
|
||||
fail: number;
|
||||
durationMs: number;
|
||||
decisions: {
|
||||
[ToolCallDecision.ACCEPT]: number;
|
||||
[ToolCallDecision.REJECT]: number;
|
||||
[ToolCallDecision.MODIFY]: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ModelMetrics {
|
||||
api: {
|
||||
totalRequests: number;
|
||||
totalErrors: number;
|
||||
totalLatencyMs: number;
|
||||
};
|
||||
tokens: {
|
||||
prompt: number;
|
||||
candidates: number;
|
||||
total: number;
|
||||
cached: number;
|
||||
thoughts: number;
|
||||
tool: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface SessionMetrics {
|
||||
models: Record<string, ModelMetrics>;
|
||||
tools: {
|
||||
totalCalls: number;
|
||||
totalSuccess: number;
|
||||
totalFail: number;
|
||||
totalDurationMs: number;
|
||||
totalDecisions: {
|
||||
[ToolCallDecision.ACCEPT]: number;
|
||||
[ToolCallDecision.REJECT]: number;
|
||||
[ToolCallDecision.MODIFY]: number;
|
||||
};
|
||||
byName: Record<string, ToolCallStats>;
|
||||
};
|
||||
}
|
||||
|
||||
const createInitialModelMetrics = (): ModelMetrics => ({
|
||||
api: {
|
||||
totalRequests: 0,
|
||||
totalErrors: 0,
|
||||
totalLatencyMs: 0,
|
||||
},
|
||||
tokens: {
|
||||
prompt: 0,
|
||||
candidates: 0,
|
||||
total: 0,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
});
|
||||
|
||||
const createInitialMetrics = (): SessionMetrics => ({
|
||||
models: {},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: {
|
||||
[ToolCallDecision.ACCEPT]: 0,
|
||||
[ToolCallDecision.REJECT]: 0,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
},
|
||||
byName: {},
|
||||
},
|
||||
});
|
||||
|
||||
export class UiTelemetryService extends EventEmitter {
|
||||
#metrics: SessionMetrics = createInitialMetrics();
|
||||
#lastPromptTokenCount = 0;
|
||||
|
||||
addEvent(event: UiEvent) {
|
||||
switch (event['event.name']) {
|
||||
case EVENT_API_RESPONSE:
|
||||
this.processApiResponse(event);
|
||||
break;
|
||||
case EVENT_API_ERROR:
|
||||
this.processApiError(event);
|
||||
break;
|
||||
case EVENT_TOOL_CALL:
|
||||
this.processToolCall(event);
|
||||
break;
|
||||
default:
|
||||
// We should not emit update for any other event metric.
|
||||
return;
|
||||
}
|
||||
|
||||
this.emit('update', {
|
||||
metrics: this.#metrics,
|
||||
lastPromptTokenCount: this.#lastPromptTokenCount,
|
||||
});
|
||||
}
|
||||
|
||||
getMetrics(): SessionMetrics {
|
||||
return this.#metrics;
|
||||
}
|
||||
|
||||
getLastPromptTokenCount(): number {
|
||||
return this.#lastPromptTokenCount;
|
||||
}
|
||||
|
||||
private getOrCreateModelMetrics(modelName: string): ModelMetrics {
|
||||
if (!this.#metrics.models[modelName]) {
|
||||
this.#metrics.models[modelName] = createInitialModelMetrics();
|
||||
}
|
||||
return this.#metrics.models[modelName];
|
||||
}
|
||||
|
||||
private processApiResponse(event: ApiResponseEvent) {
|
||||
const modelMetrics = this.getOrCreateModelMetrics(event.model);
|
||||
|
||||
modelMetrics.api.totalRequests++;
|
||||
modelMetrics.api.totalLatencyMs += event.duration_ms;
|
||||
|
||||
modelMetrics.tokens.prompt += event.input_token_count;
|
||||
modelMetrics.tokens.candidates += event.output_token_count;
|
||||
modelMetrics.tokens.total += event.total_token_count;
|
||||
modelMetrics.tokens.cached += event.cached_content_token_count;
|
||||
modelMetrics.tokens.thoughts += event.thoughts_token_count;
|
||||
modelMetrics.tokens.tool += event.tool_token_count;
|
||||
|
||||
this.#lastPromptTokenCount = event.input_token_count;
|
||||
}
|
||||
|
||||
private processApiError(event: ApiErrorEvent) {
|
||||
const modelMetrics = this.getOrCreateModelMetrics(event.model);
|
||||
modelMetrics.api.totalRequests++;
|
||||
modelMetrics.api.totalErrors++;
|
||||
modelMetrics.api.totalLatencyMs += event.duration_ms;
|
||||
}
|
||||
|
||||
private processToolCall(event: ToolCallEvent) {
|
||||
const { tools } = this.#metrics;
|
||||
tools.totalCalls++;
|
||||
tools.totalDurationMs += event.duration_ms;
|
||||
|
||||
if (event.success) {
|
||||
tools.totalSuccess++;
|
||||
} else {
|
||||
tools.totalFail++;
|
||||
}
|
||||
|
||||
if (!tools.byName[event.function_name]) {
|
||||
tools.byName[event.function_name] = {
|
||||
count: 0,
|
||||
success: 0,
|
||||
fail: 0,
|
||||
durationMs: 0,
|
||||
decisions: {
|
||||
[ToolCallDecision.ACCEPT]: 0,
|
||||
[ToolCallDecision.REJECT]: 0,
|
||||
[ToolCallDecision.MODIFY]: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const toolStats = tools.byName[event.function_name];
|
||||
toolStats.count++;
|
||||
toolStats.durationMs += event.duration_ms;
|
||||
if (event.success) {
|
||||
toolStats.success++;
|
||||
} else {
|
||||
toolStats.fail++;
|
||||
}
|
||||
|
||||
if (event.decision) {
|
||||
tools.totalDecisions[event.decision]++;
|
||||
toolStats.decisions[event.decision]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const uiTelemetryService = new UiTelemetryService();
|
Loading…
Reference in New Issue