From 7cc6b8c270a67803f9387eeead0d3d7ac914303a Mon Sep 17 00:00:00 2001 From: Richie Foreman Date: Tue, 12 Aug 2025 14:31:59 -0400 Subject: [PATCH] chore(usage telemetry): Freshen up Clearcut logging (#6013) Co-authored-by: christine betts Co-authored-by: Jacob Richman Co-authored-by: matt korwel --- eslint.config.js | 7 +- package-lock.json | 557 ++++++++++++++++++ package.json | 8 +- .../cli/src/config/config.integration.test.ts | 21 + packages/core/src/config/config.test.ts | 42 +- packages/core/src/mocks/msw.ts | 9 + .../clearcut-logger/clearcut-logger.test.ts | 253 ++++---- .../clearcut-logger/clearcut-logger.ts | 306 +++++----- packages/core/src/test-utils/config.ts | 36 ++ 9 files changed, 969 insertions(+), 270 deletions(-) create mode 100644 packages/core/src/mocks/msw.ts create mode 100644 packages/core/src/test-utils/config.ts diff --git a/eslint.config.js b/eslint.config.js index f35d4f35..fc751418 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -118,7 +118,12 @@ export default tseslint.config( 'import/no-internal-modules': [ 'error', { - allow: ['react-dom/test-utils', 'memfs/lib/volume.js', 'yargs/**'], + allow: [ + 'react-dom/test-utils', + 'memfs/lib/volume.js', + 'yargs/**', + 'msw/node', + ], }, ], 'import/no-relative-packages': 'error', diff --git a/package-lock.json b/package-lock.json index bcce33b5..acdd67f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,9 @@ "workspaces": [ "packages/*" ], + "dependencies": { + "node-fetch": "^3.3.2" + }, "bin": { "gemini": "bundle/gemini.js" }, @@ -37,6 +40,7 @@ "memfs": "^4.17.2", "mnemonist": "^0.40.3", "mock-fs": "^5.5.0", + "msw": "^2.10.4", "prettier": "^3.5.3", "react-devtools-core": "^4.28.5", "tsx": "^4.20.3", @@ -202,6 +206,53 @@ "node": ">=18" } }, + "node_modules/@bundled-es-modules/cookie": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/cookie/-/cookie-2.0.1.tgz", + "integrity": "sha512-8o+5fRPLNbjbdGRRmJj3h6Hh1AQJf2dk3qQ/5ZFb+PXkRNiSoMGGUKlsgLfrxneb72axVJyIYji64E2+nNfYyw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cookie": "^0.7.2" + } + }, + "node_modules/@bundled-es-modules/statuses": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz", + "integrity": "sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==", + "dev": true, + "license": "ISC", + "dependencies": { + "statuses": "^2.0.1" + } + }, + "node_modules/@bundled-es-modules/tough-cookie": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/tough-cookie/-/tough-cookie-0.1.6.tgz", + "integrity": "sha512-dvMHbL464C0zI+Yqxbz6kZ5TOEp7GLW+pry/RWndAR8MJQAXZ2rPmIs8tziTZjeIyhSNZgZbCePtfSbdWqStJw==", + "dev": true, + "license": "ISC", + "dependencies": { + "@types/tough-cookie": "^4.0.5", + "tough-cookie": "^4.1.4" + } + }, + "node_modules/@bundled-es-modules/tough-cookie/node_modules/tough-cookie": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", + "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@csstools/color-helpers": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.0.2.tgz", @@ -1041,6 +1092,173 @@ "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==", "license": "ISC" }, + "node_modules/@inquirer/confirm": { + "version": "5.1.14", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.14.tgz", + "integrity": "sha512-5yR4IBfe0kXe59r1YCTG8WXkUbl7Z35HK87Sw+WUyGD8wNUx7JvY7laahzeytyE1oLn74bQnL7hstctQxisQ8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.1.15", + "@inquirer/type": "^3.0.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "10.1.15", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.15.tgz", + "integrity": "sha512-8xrp836RZvKkpNbVvgWUlxjT4CraKk2q+I3Ksy+seI2zkcE+y6wNs1BVhgcv8VyImFecUhdQrYLdW32pAjwBdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/figures": "^1.0.13", + "@inquirer/type": "^3.0.8", + "ansi-escapes": "^4.3.2", + "cli-width": "^4.1.0", + "mute-stream": "^2.0.0", + "signal-exit": "^4.1.0", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core/node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@inquirer/core/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/core/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@inquirer/core/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/core/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/core/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@inquirer/core/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.13.tgz", + "integrity": "sha512-lGPVU3yO9ZNqA7vTYz26jny41lE7yoQansmqdMLBEfqaGsmdg7V3W9mK9Pvb5IL4EVZ9GnSDGMO/cJXud5dMaw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/type": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.8.tgz", + "integrity": "sha512-lg9Whz8onIHRthWaN1Q9EGLa/0LFJjyM8mEUbL1eTi6yMGvBf8gvyDLtxSXztQsxMvhxxNpJYrwa1YHdq+w4Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1239,6 +1457,24 @@ "node": ">=18" } }, + "node_modules/@mswjs/interceptors": { + "version": "0.39.5", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.39.5.tgz", + "integrity": "sha512-B9nHSJYtsv79uo7QdkZ/b/WoKm20IkVSmTc/WCKarmDtFwM0dRx2ouEniqwNkzCSLn3fydzKmnMzjtfdOWt3VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1277,6 +1513,31 @@ "node": ">= 8" } }, + "node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "dev": true, + "license": "MIT" + }, "node_modules/@opentelemetry/api": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", @@ -2224,6 +2485,13 @@ "@types/node": "*" } }, + "node_modules/@types/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/cors": { "version": "2.8.19", "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", @@ -2491,12 +2759,26 @@ "integrity": "sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg==", "license": "MIT" }, + "node_modules/@types/statuses": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.6.tgz", + "integrity": "sha512-xMAgYwceFhRA2zY+XbEA7mxYbA093wdiW8Vu6gZPGWy9cmOyU9XesH1tNcEWsKFd5Vzrqx5T3D38PWx1FIIXkA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/tinycolor2": { "version": "1.4.6", "resolved": "https://registry.npmjs.org/@types/tinycolor2/-/tinycolor2-1.4.6.tgz", "integrity": "sha512-iEN8J0BoMnsWBqjVbWH/c0G0Hh7O21lpR2/+PrvAVgWdzL7eexIFm4JN/Wn10PTcmNdtS6U67r499mlWMXOxNw==", "license": "MIT" }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -3779,6 +4061,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -4069,6 +4361,15 @@ "devOptional": true, "license": "MIT" }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", @@ -5323,6 +5624,29 @@ "reusify": "^1.0.4" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/figures": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/figures/-/figures-6.1.0.tgz", @@ -5462,6 +5786,18 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -5930,6 +6266,16 @@ "dev": true, "license": "MIT" }, + "node_modules/graphql": { + "version": "16.11.0", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz", + "integrity": "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, "node_modules/gtoken": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", @@ -6092,6 +6438,13 @@ "node": ">= 0.4" } }, + "node_modules/headers-polyfill": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz", + "integrity": "sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/highlight.js": { "version": "11.11.1", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz", @@ -6951,6 +7304,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "dev": true, + "license": "MIT" + }, "node_modules/is-npm": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-6.0.0.tgz", @@ -7906,6 +8266,81 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msw": { + "version": "2.10.4", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.10.4.tgz", + "integrity": "sha512-6R1or/qyele7q3RyPwNuvc0IxO8L8/Aim6Sz5ncXEgcWUNxSKE+udriTOWHtpMwmfkLYlacA2y7TIx4cL5lgHA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@bundled-es-modules/cookie": "^2.0.1", + "@bundled-es-modules/statuses": "^1.0.1", + "@bundled-es-modules/tough-cookie": "^0.1.6", + "@inquirer/confirm": "^5.0.0", + "@mswjs/interceptors": "^0.39.1", + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/until": "^2.1.0", + "@types/cookie": "^0.6.0", + "@types/statuses": "^2.0.4", + "graphql": "^16.8.1", + "headers-polyfill": "^4.0.2", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "path-to-regexp": "^6.3.0", + "picocolors": "^1.1.1", + "strict-event-emitter": "^0.5.1", + "type-fest": "^4.26.1", + "yargs": "^17.7.2" + }, + "bin": { + "msw": "cli/index.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/mswjs" + }, + "peerDependencies": { + "typescript": ">= 4.8.x" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/msw/node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/msw/node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mute-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", + "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -7948,6 +8383,44 @@ "dev": true, "license": "MIT" }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/normalize-package-data": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.2.tgz", @@ -8392,6 +8865,13 @@ "node": ">= 0.8.0" } }, + "node_modules/outvariant": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz", + "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==", + "dev": true, + "license": "MIT" + }, "node_modules/own-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/own-keys/-/own-keys-1.0.1.tgz", @@ -8870,6 +9350,19 @@ "node": ">= 0.10" } }, + "node_modules/psl": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", + "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "funding": { + "url": "https://github.com/sponsors/lupomontero" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -8909,6 +9402,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true, + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -9246,6 +9746,13 @@ "node": ">=0.10.5" } }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, + "license": "MIT" + }, "node_modules/resolve": { "version": "1.22.10", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", @@ -9933,6 +10440,13 @@ "node": ">= 0.4" } }, + "node_modules/strict-event-emitter": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", + "dev": true, + "license": "MIT" + }, "node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", @@ -10806,6 +11320,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -10985,6 +11509,17 @@ "punycode": "^2.1.0" } }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -11242,6 +11777,15 @@ "node": ">=18" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -11703,6 +12247,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/yoctocolors-cjs": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz", + "integrity": "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/yoga-layout": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/yoga-layout/-/yoga-layout-3.2.1.tgz", diff --git a/package.json b/package.json index 637fc445..0e27676f 100644 --- a/package.json +++ b/package.json @@ -79,13 +79,17 @@ "json": "^11.0.0", "lodash": "^4.17.21", "memfs": "^4.17.2", + "mnemonist": "^0.40.3", "mock-fs": "^5.5.0", + "msw": "^2.10.4", "prettier": "^3.5.3", "react-devtools-core": "^4.28.5", "tsx": "^4.20.3", "typescript-eslint": "^8.30.1", "vitest": "^3.2.4", - "yargs": "^17.7.2", - "mnemonist": "^0.40.3" + "yargs": "^17.7.2" + }, + "dependencies": { + "node-fetch": "^3.3.2" } } diff --git a/packages/cli/src/config/config.integration.test.ts b/packages/cli/src/config/config.integration.test.ts index 5d83986e..87a74578 100644 --- a/packages/cli/src/config/config.integration.test.ts +++ b/packages/cli/src/config/config.integration.test.ts @@ -13,6 +13,25 @@ import { ConfigParameters, ContentGeneratorConfig, } from '@google/gemini-cli-core'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; + +export const server = setupServer(); + +// TODO(richieforeman): Consider moving this to test setup globally. +beforeAll(() => { + server.listen({}); +}); + +afterEach(() => { + server.resetHandlers(); +}); + +afterAll(() => { + server.close(); +}); + +const CLEARCUT_URL = 'https://play.googleapis.com/log'; const TEST_CONTENT_GENERATOR_CONFIG: ContentGeneratorConfig = { apiKey: 'test-key', @@ -37,6 +56,8 @@ describe('Configuration Integration Tests', () => { let originalEnv: NodeJS.ProcessEnv; beforeEach(() => { + server.resetHandlers(http.post(CLEARCUT_URL, () => HttpResponse.text())); + tempDir = fs.mkdtempSync(path.join(tmpdir(), 'gemini-cli-test-')); originalEnv = { ...process.env }; process.env.GEMINI_API_KEY = 'test-api-key'; diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index 8e6ca38f..6c57d058 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -4,7 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { describe, it, expect, vi, beforeEach, Mock } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { Mock } from 'vitest'; import { Config, ConfigParameters, SandboxConfig } from './config.js'; import * as path from 'path'; import { setGeminiMdFilename as mockSetGeminiMdFilename } from '../tools/memoryTool.js'; @@ -18,6 +19,7 @@ import { } from '../core/contentGenerator.js'; import { GeminiClient } from '../core/client.js'; import { GitService } from '../services/gitService.js'; +import { ClearcutLogger } from '../telemetry/clearcut-logger/clearcut-logger.js'; vi.mock('fs', async (importOriginal) => { const actual = await importOriginal(); @@ -119,11 +121,16 @@ describe('Server Config (config.ts)', () => { telemetry: TELEMETRY_SETTINGS, sessionId: SESSION_ID, model: MODEL, + usageStatisticsEnabled: false, }; beforeEach(() => { // Reset mocks if necessary vi.clearAllMocks(); + vi.spyOn( + ClearcutLogger.prototype, + 'logStartSessionEvent', + ).mockImplementation(() => undefined); }); describe('initialize', () => { @@ -372,6 +379,39 @@ describe('Server Config (config.ts)', () => { expect(fileService).toBeDefined(); }); + describe('Usage Statistics', () => { + it('defaults usage statistics to enabled if not specified', () => { + const config = new Config({ + ...baseParams, + usageStatisticsEnabled: undefined, + }); + + expect(config.getUsageStatisticsEnabled()).toBe(true); + }); + + it.each([{ enabled: true }, { enabled: false }])( + 'sets usage statistics based on the provided value (enabled: $enabled)', + ({ enabled }) => { + const config = new Config({ + ...baseParams, + usageStatisticsEnabled: enabled, + }); + expect(config.getUsageStatisticsEnabled()).toBe(enabled); + }, + ); + + it('logs the session start event', () => { + new Config({ + ...baseParams, + usageStatisticsEnabled: true, + }); + + expect( + ClearcutLogger.prototype.logStartSessionEvent, + ).toHaveBeenCalledOnce(); + }); + }); + describe('Telemetry Settings', () => { it('should return default telemetry target if not provided', () => { const params: ConfigParameters = { diff --git a/packages/core/src/mocks/msw.ts b/packages/core/src/mocks/msw.ts new file mode 100644 index 00000000..4bf93138 --- /dev/null +++ b/packages/core/src/mocks/msw.ts @@ -0,0 +1,9 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { setupServer } from 'msw/node'; + +export const server = setupServer(); diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts index f955eb5a..96129ad3 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts @@ -4,33 +4,49 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest'; -import * as https from 'https'; -import { ClientRequest, IncomingMessage } from 'http'; -import { Readable, Writable } from 'stream'; - import { - ClearcutLogger, - LogResponse, - LogEventEntry, -} from './clearcut-logger.js'; -import { Config } from '../../config/config.js'; + vi, + describe, + it, + expect, + afterEach, + beforeAll, + afterAll, +} from 'vitest'; + +import { ClearcutLogger, LogEventEntry, TEST_ONLY } from './clearcut-logger.js'; +import { ConfigParameters } from '../../config/config.js'; import * as userAccount from '../../utils/user_account.js'; import * as userId from '../../utils/user_id.js'; +import { EventMetadataKey } from './event-metadata-key.js'; +import { makeFakeConfig } from '../../test-utils/config.js'; +import { http, HttpResponse } from 'msw'; +import { server } from '../../mocks/msw.js'; -// Mock dependencies -vi.mock('https-proxy-agent'); -vi.mock('https'); vi.mock('../../utils/user_account'); vi.mock('../../utils/user_id'); -const mockHttps = vi.mocked(https); const mockUserAccount = vi.mocked(userAccount); const mockUserId = vi.mocked(userId); +// TODO(richieforeman): Consider moving this to test setup globally. +beforeAll(() => { + server.listen({}); +}); + +afterEach(() => { + server.resetHandlers(); +}); + +afterAll(() => { + server.close(); +}); + describe('ClearcutLogger', () => { - let mockConfig: Config; - let logger: ClearcutLogger | undefined; + const NEXT_WAIT_MS = 1234; + const CLEARCUT_URL = 'https://play.googleapis.com/log'; + const MOCK_DATE = new Date('2025-01-02T00:00:00.000Z'); + const EXAMPLE_RESPONSE = `["${NEXT_WAIT_MS}",null,[[["ANDROID_BACKUP",0],["BATTERY_STATS",0],["SMART_SETUP",0],["TRON",0]],-3334737594024971225],[]]`; // A helper to get the internal events array for testing const getEvents = (l: ClearcutLogger): LogEventEntry[][] => @@ -38,32 +54,37 @@ describe('ClearcutLogger', () => { const getEventsSize = (l: ClearcutLogger): number => l['events'].size; - const getMaxEvents = (l: ClearcutLogger): number => l['max_events']; - - const getMaxRetryEvents = (l: ClearcutLogger): number => - l['max_retry_events']; - const requeueFailedEvents = (l: ClearcutLogger, events: LogEventEntry[][]) => l['requeueFailedEvents'](events); - beforeEach(() => { + function setup({ + config = {} as Partial, + lifetimeGoogleAccounts = 1, + cachedGoogleAccount = 'test@google.com', + installationId = 'test-installation-id', + } = {}) { + server.resetHandlers( + http.post(CLEARCUT_URL, () => HttpResponse.text(EXAMPLE_RESPONSE)), + ); + vi.useFakeTimers(); - vi.setSystemTime(new Date()); + vi.setSystemTime(MOCK_DATE); - mockConfig = { - getUsageStatisticsEnabled: vi.fn().mockReturnValue(true), - getDebugMode: vi.fn().mockReturnValue(false), - getSessionId: vi.fn().mockReturnValue('test-session-id'), - getProxy: vi.fn().mockReturnValue(undefined), - } as unknown as Config; + const loggerConfig = makeFakeConfig({ + ...config, + }); + ClearcutLogger.clearInstance(); - mockUserAccount.getCachedGoogleAccount.mockReturnValue('test@google.com'); - mockUserAccount.getLifetimeGoogleAccounts.mockReturnValue(1); - mockUserId.getInstallationId.mockReturnValue('test-installation-id'); + mockUserAccount.getCachedGoogleAccount.mockReturnValue(cachedGoogleAccount); + mockUserAccount.getLifetimeGoogleAccounts.mockReturnValue( + lifetimeGoogleAccounts, + ); + mockUserId.getInstallationId.mockReturnValue(installationId); - logger = ClearcutLogger.getInstance(mockConfig); - expect(logger).toBeDefined(); - }); + const logger = ClearcutLogger.getInstance(loggerConfig); + + return { logger, loggerConfig }; + } afterEach(() => { ClearcutLogger.clearInstance(); @@ -71,109 +92,131 @@ describe('ClearcutLogger', () => { vi.restoreAllMocks(); }); - it('should not return an instance if usage statistics are disabled', () => { - ClearcutLogger.clearInstance(); - vi.spyOn(mockConfig, 'getUsageStatisticsEnabled').mockReturnValue(false); - const disabledLogger = ClearcutLogger.getInstance(mockConfig); - expect(disabledLogger).toBeUndefined(); + describe('getInstance', () => { + it.each([ + { usageStatisticsEnabled: false, expectedValue: undefined }, + { + usageStatisticsEnabled: true, + expectedValue: expect.any(ClearcutLogger), + }, + ])( + 'returns an instance if usage statistics are enabled', + ({ usageStatisticsEnabled, expectedValue }) => { + ClearcutLogger.clearInstance(); + const { logger } = setup({ + config: { + usageStatisticsEnabled, + }, + }); + expect(logger).toEqual(expectedValue); + }, + ); + + it('is a singleton', () => { + ClearcutLogger.clearInstance(); + const { loggerConfig } = setup(); + const logger1 = ClearcutLogger.getInstance(loggerConfig); + const logger2 = ClearcutLogger.getInstance(loggerConfig); + expect(logger1).toBe(logger2); + }); + }); + + describe('createLogEvent', () => { + it('logs the total number of google accounts', () => { + const { logger } = setup({ + lifetimeGoogleAccounts: 9001, + }); + + const event = logger?.createLogEvent('abc', []); + + expect(event?.event_metadata[0][0]).toEqual({ + gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT, + value: '9001', + }); + }); + + it('logs the current surface', () => { + const { logger } = setup({}); + + const event = logger?.createLogEvent('abc', []); + + expect(event?.event_metadata[0][1]).toEqual({ + gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE, + value: 'SURFACE_NOT_SET', + }); + }); }); describe('enqueueLogEvent', () => { it('should add events to the queue', () => { + const { logger } = setup(); logger!.enqueueLogEvent({ test: 'event1' }); expect(getEventsSize(logger!)).toBe(1); }); it('should evict the oldest event when the queue is full', () => { - const maxEvents = getMaxEvents(logger!); + const { logger } = setup(); - for (let i = 0; i < maxEvents; i++) { + for (let i = 0; i < TEST_ONLY.MAX_EVENTS; i++) { logger!.enqueueLogEvent({ event_id: i }); } - expect(getEventsSize(logger!)).toBe(maxEvents); + expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_EVENTS); const firstEvent = JSON.parse( getEvents(logger!)[0][0].source_extension_json, ); expect(firstEvent.event_id).toBe(0); // This should push out the first event - logger!.enqueueLogEvent({ event_id: maxEvents }); + logger!.enqueueLogEvent({ event_id: TEST_ONLY.MAX_EVENTS }); - expect(getEventsSize(logger!)).toBe(maxEvents); + expect(getEventsSize(logger!)).toBe(TEST_ONLY.MAX_EVENTS); const newFirstEvent = JSON.parse( getEvents(logger!)[0][0].source_extension_json, ); expect(newFirstEvent.event_id).toBe(1); const lastEvent = JSON.parse( - getEvents(logger!)[maxEvents - 1][0].source_extension_json, + getEvents(logger!)[TEST_ONLY.MAX_EVENTS - 1][0].source_extension_json, ); - expect(lastEvent.event_id).toBe(maxEvents); + expect(lastEvent.event_id).toBe(TEST_ONLY.MAX_EVENTS); }); }); describe('flushToClearcut', () => { - let mockRequest: Writable; - let mockResponse: Readable & Partial; - - beforeEach(() => { - mockRequest = new Writable({ - write(chunk, encoding, callback) { - callback(); + it('allows for usage with a configured proxy agent', async () => { + const { logger } = setup({ + config: { + proxy: 'http://mycoolproxy.whatever.com:3128', }, }); - vi.spyOn(mockRequest, 'on'); - vi.spyOn(mockRequest, 'end').mockReturnThis(); - vi.spyOn(mockRequest, 'destroy').mockReturnThis(); - mockResponse = new Readable({ read() {} }) as Readable & - Partial; + logger!.enqueueLogEvent({ event_id: 1 }); - mockHttps.request.mockImplementation( - ( - _options: string | https.RequestOptions | URL, - ...args: unknown[] - ): ClientRequest => { - const callback = args.find((arg) => typeof arg === 'function') as - | ((res: IncomingMessage) => void) - | undefined; + const response = await logger!.flushToClearcut(); - if (callback) { - callback(mockResponse as IncomingMessage); - } - return mockRequest as ClientRequest; - }, - ); + expect(response.nextRequestWaitMs).toBe(NEXT_WAIT_MS); }); it('should clear events on successful flush', async () => { - mockResponse.statusCode = 200; - const mockResponseBody = { nextRequestWaitMs: 1000 }; - // Encoded protobuf for {nextRequestWaitMs: 1000} which is `08 E8 07` - const encodedResponse = Buffer.from([8, 232, 7]); + const { logger } = setup(); logger!.enqueueLogEvent({ event_id: 1 }); - const flushPromise = logger!.flushToClearcut(); + const response = await logger!.flushToClearcut(); - mockResponse.push(encodedResponse); - mockResponse.push(null); // End the stream - - const response: LogResponse = await flushPromise; - - expect(getEventsSize(logger!)).toBe(0); - expect(response.nextRequestWaitMs).toBe( - mockResponseBody.nextRequestWaitMs, - ); + expect(getEvents(logger!)).toEqual([]); + expect(response.nextRequestWaitMs).toBe(NEXT_WAIT_MS); }); it('should handle a network error and requeue events', async () => { + const { logger } = setup(); + + server.resetHandlers(http.post(CLEARCUT_URL, () => HttpResponse.error())); logger!.enqueueLogEvent({ event_id: 1 }); logger!.enqueueLogEvent({ event_id: 2 }); expect(getEventsSize(logger!)).toBe(2); - const flushPromise = logger!.flushToClearcut(); - mockRequest.emit('error', new Error('Network error')); - await flushPromise; + const x = logger!.flushToClearcut(); + await x; expect(getEventsSize(logger!)).toBe(2); const events = getEvents(logger!); @@ -181,18 +224,28 @@ describe('ClearcutLogger', () => { }); it('should handle an HTTP error and requeue events', async () => { - mockResponse.statusCode = 500; - mockResponse.statusMessage = 'Internal Server Error'; + const { logger } = setup(); + + server.resetHandlers( + http.post( + CLEARCUT_URL, + () => + new HttpResponse( + { 'the system is down': true }, + { + status: 500, + }, + ), + ), + ); logger!.enqueueLogEvent({ event_id: 1 }); logger!.enqueueLogEvent({ event_id: 2 }); - expect(getEventsSize(logger!)).toBe(2); - const flushPromise = logger!.flushToClearcut(); - mockResponse.emit('end'); // End the response to trigger promise resolution - await flushPromise; + expect(getEvents(logger!).length).toBe(2); + await logger!.flushToClearcut(); - expect(getEventsSize(logger!)).toBe(2); + expect(getEvents(logger!).length).toBe(2); const events = getEvents(logger!); expect(JSON.parse(events[0][0].source_extension_json).event_id).toBe(1); }); @@ -200,7 +253,8 @@ describe('ClearcutLogger', () => { describe('requeueFailedEvents logic', () => { it('should limit the number of requeued events to max_retry_events', () => { - const maxRetryEvents = getMaxRetryEvents(logger!); + const { logger } = setup(); + const maxRetryEvents = TEST_ONLY.MAX_RETRY_EVENTS; const eventsToLogCount = maxRetryEvents + 5; const eventsToSend: LogEventEntry[][] = []; for (let i = 0; i < eventsToLogCount; i++) { @@ -225,7 +279,8 @@ describe('ClearcutLogger', () => { }); it('should not requeue more events than available space in the queue', () => { - const maxEvents = getMaxEvents(logger!); + const { logger } = setup(); + const maxEvents = TEST_ONLY.MAX_EVENTS; const spaceToLeave = 5; const initialEventCount = maxEvents - spaceToLeave; for (let i = 0; i < initialEventCount; i++) { diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index 1e67d1cf..a41f832d 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -4,10 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Buffer } from 'buffer'; -import * as https from 'https'; import { HttpsProxyAgent } from 'https-proxy-agent'; - import { StartSessionEvent, EndSessionEvent, @@ -56,19 +53,25 @@ export interface LogEventEntry { source_extension_json: string; } -export type EventValue = { +export interface EventValue { gemini_cli_key: EventMetadataKey | string; value: string; -}; +} -export type LogEvent = { - console_type: string; +export interface LogEvent { + console_type: 'GEMINI_CLI'; application: number; event_name: string; event_metadata: EventValue[][]; client_email?: string; client_install_id?: string; -}; +} + +export interface LogRequest { + log_source_name: 'CONCORD'; + request_time_ms: number; + log_event: LogEventEntry[][]; +} /** * Determine the surface that the user is currently using. Surface is effectively the @@ -89,22 +92,59 @@ function determineSurface(): string { } } +/** + * Clearcut URL to send logging events to. + */ +const CLEARCUT_URL = 'https://play.googleapis.com/log?format=json&hasfast=true'; + +/** + * Interval in which buffered events are sent to clearcut. + */ +const FLUSH_INTERVAL_MS = 1000 * 60; + +/** + * Maximum amount of events to keep in memory. Events added after this amount + * are dropped until the next flush to clearcut, which happens periodically as + * defined by {@link FLUSH_INTERVAL_MS}. + */ +const MAX_EVENTS = 1000; + +/** + * Maximum events to retry after a failed clearcut flush + */ +const MAX_RETRY_EVENTS = 100; + // Singleton class for batch posting log events to Clearcut. When a new event comes in, the elapsed time // is checked and events are flushed to Clearcut if at least a minute has passed since the last flush. export class ClearcutLogger { private static instance: ClearcutLogger; private config?: Config; + + /** + * Queue of pending events that need to be flushed to the server. New events + * are added to this queue and then flushed on demand (via `flushToClearcut`) + */ private readonly events: FixedDeque; - private last_flush_time: number = Date.now(); - private flush_interval_ms: number = 1000 * 60; // Wait at least a minute before flushing events. - private readonly max_events: number = 1000; // Maximum events to keep in memory - private readonly max_retry_events: number = 100; // Maximum failed events to retry - private flushing: boolean = false; // Prevent concurrent flush operations - private pendingFlush: boolean = false; // Track if a flush was requested during an ongoing flush + + /** + * The last time that the events were successfully flushed to the server. + */ + private lastFlushTime: number = Date.now(); + + /** + * the value is true when there is a pending flush happening. This prevents + * concurrent flush operations. + */ + private flushing: boolean = false; + + /** + * This value is true when a flush was requested during an ongoing flush. + */ + private pendingFlush: boolean = false; private constructor(config?: Config) { this.config = config; - this.events = new FixedDeque(Array, this.max_events); + this.events = new FixedDeque(Array, MAX_EVENTS); } static getInstance(config?: Config): ClearcutLogger | undefined { @@ -125,7 +165,7 @@ export class ClearcutLogger { enqueueLogEvent(event: object): void { try { // Manually handle overflow for FixedDeque, which throws when full. - const wasAtCapacity = this.events.size >= this.max_events; + const wasAtCapacity = this.events.size >= MAX_EVENTS; if (wasAtCapacity) { this.events.shift(); // Evict oldest element to make space. @@ -150,31 +190,14 @@ export class ClearcutLogger { } } - addDefaultFields(data: EventValue[]): void { - const totalAccounts = getLifetimeGoogleAccounts(); - const surface = determineSurface(); - const defaultLogMetadata = [ - { - gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT, - value: totalAccounts.toString(), - }, - { - gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE, - value: surface, - }, - ]; - data.push(...defaultLogMetadata); - } - createLogEvent(name: string, data: EventValue[]): LogEvent { const email = getCachedGoogleAccount(); - // Add default fields that should exist for all logs - this.addDefaultFields(data); + data = addDefaultFields(data); const logEvent: LogEvent = { console_type: 'GEMINI_CLI', - application: 102, + application: 102, // GEMINI_CLI event_name: name, event_metadata: [data], }; @@ -190,7 +213,7 @@ export class ClearcutLogger { } flushIfNeeded(): void { - if (Date.now() - this.last_flush_time < this.flush_interval_ms) { + if (Date.now() - this.lastFlushTime < FLUSH_INTERVAL_MS) { return; } @@ -217,140 +240,67 @@ export class ClearcutLogger { const eventsToSend = this.events.toArray() as LogEventEntry[][]; this.events.clear(); - return new Promise<{ buffer: Buffer; statusCode?: number }>( - (resolve, reject) => { - const request = [ - { - log_source_name: 'CONCORD', - request_time_ms: Date.now(), - log_event: eventsToSend, - }, - ]; - const body = safeJsonStringify(request); - const options = { - hostname: 'play.googleapis.com', - path: '/log', - method: 'POST', - headers: { 'Content-Length': Buffer.byteLength(body) }, - timeout: 30000, // 30-second timeout - }; - const bufs: Buffer[] = []; - const req = https.request( - { - ...options, - agent: this.getProxyAgent(), - }, - (res) => { - res.on('error', reject); // Handle stream errors - res.on('data', (buf) => bufs.push(buf)); - res.on('end', () => { - try { - const buffer = Buffer.concat(bufs); - // Check if we got a successful response - if ( - res.statusCode && - res.statusCode >= 200 && - res.statusCode < 300 - ) { - resolve({ buffer, statusCode: res.statusCode }); - } else { - // HTTP error - reject with status code for retry handling - reject( - new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`), - ); - } - } catch (e) { - reject(e); - } - }); - }, - ); - req.on('error', (e) => { - // Network-level error - reject(e); - }); - req.on('timeout', () => { - if (!req.destroyed) { - req.destroy(new Error('Request timeout after 30 seconds')); - } - }); - req.end(body); + const request: LogRequest[] = [ + { + log_source_name: 'CONCORD', + request_time_ms: Date.now(), + log_event: eventsToSend, }, - ) - .then(({ buffer }) => { - try { - this.last_flush_time = Date.now(); - return this.decodeLogResponse(buffer) || {}; - } catch (error: unknown) { - console.error('Error decoding log response:', error); - return {}; - } - }) - .catch((error: unknown) => { - // Handle both network-level and HTTP-level errors + ]; + + let result: LogResponse = {}; + + try { + const response = await fetch(CLEARCUT_URL, { + method: 'POST', + body: safeJsonStringify(request), + headers: { + 'Content-Type': 'application/json', + }, + }); + + const responseBody = await response.text(); + + if (response.status >= 200 && response.status < 300) { + this.lastFlushTime = Date.now(); + const nextRequestWaitMs = Number(JSON.parse(responseBody)[0]); + result = { + ...result, + nextRequestWaitMs, + }; + } else { if (this.config?.getDebugMode()) { - console.error('Error flushing log events:', error); + console.error( + `Error flushing log events: HTTP ${response.status}: ${response.statusText}`, + ); } // Re-queue failed events for retry this.requeueFailedEvents(eventsToSend); + } + } catch (e: unknown) { + if (this.config?.getDebugMode()) { + console.error('Error flushing log events:', e as Error); + } - // Return empty response to maintain the Promise contract - return {}; - }) - .finally(() => { - this.flushing = false; + // Re-queue failed events for retry + this.requeueFailedEvents(eventsToSend); + } - // If a flush was requested while we were flushing, flush again - if (this.pendingFlush) { - this.pendingFlush = false; - // Fire and forget the pending flush - this.flushToClearcut().catch((error) => { - if (this.config?.getDebugMode()) { - console.debug('Error in pending flush to Clearcut:', error); - } - }); + this.flushing = false; + + // If a flush was requested while we were flushing, flush again + if (this.pendingFlush) { + this.pendingFlush = false; + // Fire and forget the pending flush + this.flushToClearcut().catch((error) => { + if (this.config?.getDebugMode()) { + console.debug('Error in pending flush to Clearcut:', error); } }); - } - - // Visible for testing. Decodes protobuf-encoded response from Clearcut server. - decodeLogResponse(buf: Buffer): LogResponse | undefined { - // TODO(obrienowen): return specific errors to facilitate debugging. - if (buf.length < 1) { - return undefined; } - // The first byte of the buffer is `field<<3 | type`. We're looking for field - // 1, with type varint, represented by type=0. If the first byte isn't 8, that - // means field 1 is missing or the message is corrupted. Either way, we return - // undefined. - if (buf.readUInt8(0) !== 8) { - return undefined; - } - - let ms = BigInt(0); - let cont = true; - - // In each byte, the most significant bit is the continuation bit. If it's - // set, we keep going. The lowest 7 bits, are data bits. They are concatenated - // in reverse order to form the final number. - for (let i = 1; cont && i < buf.length; i++) { - const byte = buf.readUInt8(i); - ms |= BigInt(byte & 0x7f) << BigInt(7 * (i - 1)); - cont = (byte & 0x80) !== 0; - } - - if (cont) { - // We have fallen off the buffer without seeing a terminating byte. The - // message is corrupted. - return undefined; - } - - const returnVal = { - nextRequestWaitMs: Number(ms), - }; - return returnVal; + return result; } logStartSessionEvent(event: StartSessionEvent): void { @@ -752,24 +702,21 @@ export class ClearcutLogger { private requeueFailedEvents(eventsToSend: LogEventEntry[][]): void { // Add the events back to the front of the queue to be retried, but limit retry queue size - const eventsToRetry = eventsToSend.slice(-this.max_retry_events); // Keep only the most recent events + const eventsToRetry = eventsToSend.slice(-MAX_RETRY_EVENTS); // Keep only the most recent events // Log a warning if we're dropping events - if ( - eventsToSend.length > this.max_retry_events && - this.config?.getDebugMode() - ) { + if (eventsToSend.length > MAX_RETRY_EVENTS && this.config?.getDebugMode()) { console.warn( `ClearcutLogger: Dropping ${ - eventsToSend.length - this.max_retry_events + eventsToSend.length - MAX_RETRY_EVENTS } events due to retry queue limit. Total events: ${ eventsToSend.length - }, keeping: ${this.max_retry_events}`, + }, keeping: ${MAX_RETRY_EVENTS}`, ); } // Determine how many events can be re-queued - const availableSpace = this.max_events - this.events.size; + const availableSpace = MAX_EVENTS - this.events.size; const numEventsToRequeue = Math.min(eventsToRetry.length, availableSpace); if (numEventsToRequeue === 0) { @@ -792,7 +739,7 @@ export class ClearcutLogger { this.events.unshift(eventsToRequeue[i]); } // Clear any potential overflow - while (this.events.size > this.max_events) { + while (this.events.size > MAX_EVENTS) { this.events.pop(); } @@ -803,3 +750,28 @@ export class ClearcutLogger { } } } + +/** + * Adds default fields to data, and returns a new data array. This fields + * should exist on all log events. + */ +function addDefaultFields(data: EventValue[]): EventValue[] { + const totalAccounts = getLifetimeGoogleAccounts(); + const surface = determineSurface(); + const defaultLogMetadata: EventValue[] = [ + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_GOOGLE_ACCOUNTS_COUNT, + value: `${totalAccounts}`, + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_SURFACE, + value: surface, + }, + ]; + return [...data, ...defaultLogMetadata]; +} + +export const TEST_ONLY = { + MAX_RETRY_EVENTS, + MAX_EVENTS, +}; diff --git a/packages/core/src/test-utils/config.ts b/packages/core/src/test-utils/config.ts new file mode 100644 index 00000000..08faf8c3 --- /dev/null +++ b/packages/core/src/test-utils/config.ts @@ -0,0 +1,36 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Config, ConfigParameters } from '../config/config.js'; + +/** + * Default parameters used for {@link FAKE_CONFIG} + */ +export const DEFAULT_CONFIG_PARAMETERS: ConfigParameters = { + usageStatisticsEnabled: true, + debugMode: false, + sessionId: 'test-session-id', + proxy: undefined, + model: 'gemini-9001-super-duper', + targetDir: '/', + cwd: '/', +}; + +/** + * Produces a config. Default paramters are set to + * {@link DEFAULT_CONFIG_PARAMETERS}, optionally, fields can be specified to + * override those defaults. + */ +export function makeFakeConfig( + config: Partial = { + ...DEFAULT_CONFIG_PARAMETERS, + }, +): Config { + return new Config({ + ...DEFAULT_CONFIG_PARAMETERS, + ...config, + }); +}