Rollforward AST changes to unblock Sandboxing (#863)
This commit is contained in:
matt korwel
GitHub
parent
ccdd1df039
commit
37edbd8c18
|
|
@ -16,14 +16,5 @@ esbuild
|
|||
banner: {
|
||||
js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url); globalThis.__filename = require('url').fileURLToPath(import.meta.url); globalThis.__dirname = require('path').dirname(globalThis.__filename);`,
|
||||
},
|
||||
external: [
|
||||
'tree-sitter',
|
||||
'tree-sitter-c-sharp',
|
||||
'tree-sitter-go',
|
||||
'tree-sitter-java',
|
||||
'tree-sitter-python',
|
||||
'tree-sitter-rust',
|
||||
'tree-sitter-typescript',
|
||||
],
|
||||
})
|
||||
.catch(() => process.exit(1));
|
||||
|
|
|
|||
|
|
@ -10,15 +10,6 @@
|
|||
"workspaces": [
|
||||
"packages/*"
|
||||
],
|
||||
"dependencies": {
|
||||
"tree-sitter": "^0.21.0",
|
||||
"tree-sitter-c-sharp": "^0.21.0",
|
||||
"tree-sitter-go": "^0.21.0",
|
||||
"tree-sitter-java": "^0.21.0",
|
||||
"tree-sitter-python": "^0.21.0",
|
||||
"tree-sitter-rust": "^0.21.0",
|
||||
"tree-sitter-typescript": "^0.21.0"
|
||||
},
|
||||
"bin": {
|
||||
"gemini": "bundle/gemini.js"
|
||||
},
|
||||
|
|
@ -7093,15 +7084,6 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/node-addon-api": {
|
||||
"version": "8.3.1",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.3.1.tgz",
|
||||
"integrity": "sha512-lytcDEdxKjGJPTLEfW4mYMigRezMlyJY8W4wxJK8zE533Jlb8L8dRuObJFWg2P+AuOIxoCgKF+2Oq4d4Zd0OUA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
|
|
@ -7144,17 +7126,6 @@
|
|||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp-build": {
|
||||
"version": "4.8.4",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
|
||||
"integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"node-gyp-build": "bin.js",
|
||||
"node-gyp-build-optional": "optional.js",
|
||||
"node-gyp-build-test": "build-test.js"
|
||||
}
|
||||
},
|
||||
"node_modules/normalize-package-data": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.2.tgz",
|
||||
|
|
@ -9171,143 +9142,6 @@
|
|||
"tslib": "2"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter": {
|
||||
"version": "0.21.1",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.21.1.tgz",
|
||||
"integrity": "sha512-7dxoA6kYvtgWw80265MyqJlkRl4yawIjO7S5MigytjELkX43fV2WsAXzsNfO7sBpPPCF5Gp0+XzHk0DwLCq3xQ==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-addon-api": "^8.0.0",
|
||||
"node-gyp-build": "^4.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-c-sharp": {
|
||||
"version": "0.21.3",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-c-sharp/-/tree-sitter-c-sharp-0.21.3.tgz",
|
||||
"integrity": "sha512-TVsl5EhmqetO/mhzDPVnMK6TPFnpNMKP0OTNuAQIprshk5Hx672ODRxoIoG5WqvUUlsnBu8J0zmn35hmJqelsA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-addon-api": "^8.0.0",
|
||||
"node-gyp-build": "^4.8.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tree-sitter": "^0.21.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"tree_sitter": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-go": {
|
||||
"version": "0.21.2",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-go/-/tree-sitter-go-0.21.2.tgz",
|
||||
"integrity": "sha512-aMFwjsB948nWhURiIxExK8QX29JYKs96P/IfXVvluVMRJZpL04SREHsdOZHYqJr1whkb7zr3/gWHqqvlkczmvw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-addon-api": "^8.1.0",
|
||||
"node-gyp-build": "^4.8.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tree-sitter": "^0.21.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"tree_sitter": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-java": {
|
||||
"version": "0.21.0",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.21.0.tgz",
|
||||
"integrity": "sha512-CKJiTo1uc3SUsgEcaZgufGx8my6dzihy8JR/JsJH40Tj3uSe2/eFLk+0q+fpbosGAyY4YiXJtEoFB2O4bS2yOw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-addon-api": "^8.0.0",
|
||||
"node-gyp-build": "^4.8.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tree-sitter": "^0.21.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"tree_sitter": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-python": {
|
||||
"version": "0.21.0",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-python/-/tree-sitter-python-0.21.0.tgz",
|
||||
"integrity": "sha512-IUKx7JcTVbByUx1iHGFS/QsIjx7pqwTMHL9bl/NGyhyyydbfNrpruo2C7W6V4KZrbkkCOlX8QVrCoGOFW5qecg==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-addon-api": "^7.1.0",
|
||||
"node-gyp-build": "^4.8.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tree-sitter": "^0.21.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"tree_sitter": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-python/node_modules/node-addon-api": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz",
|
||||
"integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tree-sitter-rust": {
|
||||
"version": "0.21.0",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-rust/-/tree-sitter-rust-0.21.0.tgz",
|
||||
"integrity": "sha512-unVr73YLn3VC4Qa/GF0Nk+Wom6UtI526p5kz9Rn2iZSqwIFedyCZ3e0fKCEmUJLIPGrTb/cIEdu3ZUNGzfZx7A==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-addon-api": "^7.1.0",
|
||||
"node-gyp-build": "^4.8.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tree-sitter": "^0.21.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"tree_sitter": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-rust/node_modules/node-addon-api": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz",
|
||||
"integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tree-sitter-typescript": {
|
||||
"version": "0.21.2",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-typescript/-/tree-sitter-typescript-0.21.2.tgz",
|
||||
"integrity": "sha512-/RyNK41ZpkA8PuPZimR6pGLvNR1p0ibRUJwwQn4qAjyyLEIQD/BNlwS3NSxWtGsAWZe9gZ44VK1mWx2+eQVldg==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-addon-api": "^8.0.0",
|
||||
"node-gyp-build": "^4.8.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tree-sitter": "^0.21.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"tree_sitter": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/ts-api-utils": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz",
|
||||
|
|
|
|||
14
package.json
14
package.json
|
|
@ -25,7 +25,7 @@
|
|||
"auth:docker": "gcloud auth configure-docker us-west1-docker.pkg.dev",
|
||||
"auth": "npm run auth:npm && npm run auth:docker",
|
||||
"prerelease:dev": "npm run prerelease:version --workspaces && npm run prerelease:deps --workspaces",
|
||||
"bundle": "npm run generate && node esbuild.config.js",
|
||||
"bundle": "npm run generate && node esbuild.config.js && bash scripts/copy_bundle_assets.sh",
|
||||
"build:cli": "npm run build --workspace packages/cli",
|
||||
"build:core": "npm run build --workspace packages/core",
|
||||
"build:packages": "npm run build:core && npm run build:cli",
|
||||
|
|
@ -42,8 +42,7 @@
|
|||
"files": [
|
||||
"bundle/",
|
||||
"README.md",
|
||||
"LICENSE",
|
||||
"node_modules/tree-sitter-*/build/Release/*.node"
|
||||
"LICENSE"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@types/micromatch": "^4.0.9",
|
||||
|
|
@ -65,14 +64,5 @@
|
|||
"react-devtools-core": "^4.28.5",
|
||||
"typescript-eslint": "^8.30.1",
|
||||
"yargs": "^17.7.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"tree-sitter": "^0.21.0",
|
||||
"tree-sitter-c-sharp": "^0.21.0",
|
||||
"tree-sitter-go": "^0.21.0",
|
||||
"tree-sitter-java": "^0.21.0",
|
||||
"tree-sitter-python": "^0.21.0",
|
||||
"tree-sitter-rust": "^0.21.0",
|
||||
"tree-sitter-typescript": "^0.21.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import process from 'node:process';
|
|||
import * as os from 'node:os';
|
||||
import { ContentGeneratorConfig } from '../core/contentGenerator.js';
|
||||
import { ToolRegistry } from '../tools/tool-registry.js';
|
||||
import { CodeParserTool } from '../tools/code_parser.js'; // Added CodeParserTool
|
||||
import { LSTool } from '../tools/ls.js';
|
||||
import { ReadFileTool } from '../tools/read-file.js';
|
||||
import { GrepTool } from '../tools/grep.js';
|
||||
|
|
@ -350,7 +349,6 @@ export function createToolRegistry(config: Config): Promise<ToolRegistry> {
|
|||
registerCoreTool(ShellTool, config);
|
||||
registerCoreTool(MemoryTool);
|
||||
registerCoreTool(WebSearchTool, config);
|
||||
registerCoreTool(CodeParserTool, targetDir, config); // Added CodeParserTool
|
||||
return (async () => {
|
||||
await registry.discoverTools();
|
||||
return registry;
|
||||
|
|
|
|||
|
|
@ -1,782 +0,0 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {
|
||||
vi,
|
||||
describe,
|
||||
it,
|
||||
expect,
|
||||
beforeEach,
|
||||
afterEach,
|
||||
Mocked,
|
||||
} from 'vitest';
|
||||
import { CodeParserTool, CodeParserToolParams } from './code_parser.js';
|
||||
import { Config } from '../config/config.js';
|
||||
import fs from 'fs/promises';
|
||||
import { Stats, PathLike } from 'fs'; // Added Stats import
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import actualFs from 'fs'; // For actual fs operations in setup
|
||||
|
||||
// Mock fs/promises
|
||||
vi.mock('fs/promises');
|
||||
|
||||
// Mock tree-sitter and its language grammars
|
||||
const mockTreeSitterParse = vi.fn();
|
||||
const mockSetLanguage = vi.fn();
|
||||
|
||||
vi.mock('tree-sitter', () => ({
|
||||
default: vi.fn().mockImplementation(() => ({
|
||||
setLanguage: mockSetLanguage,
|
||||
parse: mockTreeSitterParse,
|
||||
})),
|
||||
}));
|
||||
|
||||
const mockPythonGrammar = vi.hoisted(() => ({ name: 'python' }));
|
||||
const mockJavaGrammar = vi.hoisted(() => ({ name: 'java' }));
|
||||
const mockGoGrammar = vi.hoisted(() => ({ name: 'go' }));
|
||||
const mockCSharpGrammar = vi.hoisted(() => ({ name: 'csharp' }));
|
||||
const mockTypeScriptGrammar = vi.hoisted(() => ({ name: 'typescript' }));
|
||||
const mockTSXGrammar = vi.hoisted(() => ({ name: 'tsx' }));
|
||||
const mockRustGrammar = vi.hoisted(() => ({ name: 'rust' })); // Added for Rust
|
||||
|
||||
vi.mock('tree-sitter-python', () => ({ default: mockPythonGrammar }));
|
||||
vi.mock('tree-sitter-java', () => ({ default: mockJavaGrammar }));
|
||||
vi.mock('tree-sitter-go', () => ({ default: mockGoGrammar }));
|
||||
vi.mock('tree-sitter-c-sharp', () => ({ default: mockCSharpGrammar }));
|
||||
vi.mock('tree-sitter-typescript', () => ({
|
||||
default: {
|
||||
typescript: mockTypeScriptGrammar,
|
||||
tsx: mockTSXGrammar,
|
||||
},
|
||||
}));
|
||||
vi.mock('tree-sitter-rust', () => ({ default: mockRustGrammar })); // Added for Rust
|
||||
|
||||
describe('CodeParserTool', () => {
|
||||
let tempRootDir: string;
|
||||
let tool: CodeParserTool;
|
||||
let mockConfig: Config;
|
||||
const abortSignal = new AbortController().signal;
|
||||
|
||||
// Use Mocked type for fs/promises
|
||||
let mockFs: Mocked<typeof fs>;
|
||||
|
||||
beforeEach(() => {
|
||||
const tempDirPrefix = path.join(os.tmpdir(), 'code-parser-tool-root-');
|
||||
tempRootDir = actualFs.mkdtempSync(tempDirPrefix);
|
||||
tempRootDir = path.resolve(tempRootDir);
|
||||
|
||||
mockConfig = { get: vi.fn() } as unknown as Config;
|
||||
tool = new CodeParserTool(tempRootDir, mockConfig);
|
||||
mockFs = fs as Mocked<typeof fs>;
|
||||
|
||||
mockTreeSitterParse.mockReset();
|
||||
mockSetLanguage.mockReset();
|
||||
mockFs.stat.mockReset();
|
||||
mockFs.readFile.mockReset();
|
||||
mockFs.readdir.mockReset();
|
||||
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(mock_ast)' },
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (actualFs.existsSync(tempRootDir)) {
|
||||
actualFs.rmSync(tempRootDir, { recursive: true, force: true });
|
||||
}
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('constructor and schema', () => {
|
||||
it('should have correct name', () => {
|
||||
expect(tool.name).toBe('code_parser');
|
||||
});
|
||||
|
||||
it('should have correct schema definition', () => {
|
||||
const schema = tool.schema.parameters!;
|
||||
expect(schema.type).toBe('object');
|
||||
expect(schema.properties).toHaveProperty('path');
|
||||
expect(schema.properties!.path.type).toBe('string');
|
||||
expect(schema.properties!.path.description).toContain('absolute path');
|
||||
expect(schema.properties).toHaveProperty('languages');
|
||||
expect(schema.properties!.languages.type).toBe('array');
|
||||
expect(schema.properties!.languages.description).toContain('go');
|
||||
expect(schema.properties!.languages.description).toContain('csharp');
|
||||
expect(schema.properties!.languages.description).toContain('typescript');
|
||||
expect(schema.properties!.languages.description).toContain('tsx');
|
||||
expect(schema.properties!.languages.description).toContain('javascript');
|
||||
expect(schema.properties!.languages.description).toContain('rust'); // Added for Rust
|
||||
expect(schema.required).toEqual(['path']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateToolParams', () => {
|
||||
it('should return null for valid path with languages', () => {
|
||||
const params: CodeParserToolParams = {
|
||||
path: path.join(tempRootDir, 'dir'),
|
||||
languages: [
|
||||
'python',
|
||||
'go',
|
||||
'csharp',
|
||||
'typescript',
|
||||
'tsx',
|
||||
'javascript',
|
||||
],
|
||||
};
|
||||
expect(tool.validateToolParams(params)).toBeNull();
|
||||
});
|
||||
|
||||
it('should return error for relative path', () => {
|
||||
const params: CodeParserToolParams = { path: 'file.py' };
|
||||
expect(tool.validateToolParams(params)).toMatch(/Path must be absolute/);
|
||||
});
|
||||
|
||||
it('should return error for path outside root directory', () => {
|
||||
const outsidePath = path.resolve(
|
||||
os.tmpdir(),
|
||||
'some-other-dir',
|
||||
'file.py',
|
||||
);
|
||||
if (outsidePath.startsWith(tempRootDir)) {
|
||||
console.warn(
|
||||
'Skipping outside root test due to overlapping temp/outside paths',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const params: CodeParserToolParams = { path: outsidePath };
|
||||
expect(tool.validateToolParams(params)).toMatch(
|
||||
/Path must be within the root directory/,
|
||||
);
|
||||
});
|
||||
|
||||
it('should return error if languages is not an array of strings', () => {
|
||||
const params = {
|
||||
path: path.join(tempRootDir, 'file.py'),
|
||||
languages: [123],
|
||||
} as unknown as CodeParserToolParams;
|
||||
expect(tool.validateToolParams(params)).toBe(
|
||||
'Languages parameter must be an array of strings.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDescription', () => {
|
||||
it('should return "Parse <shortened_relative_path>"', () => {
|
||||
const filePath = path.join(tempRootDir, 'src', 'app', 'main.py');
|
||||
const params: CodeParserToolParams = { path: filePath };
|
||||
expect(tool.getDescription(params)).toBe('Parse src/app/main.py');
|
||||
});
|
||||
});
|
||||
|
||||
describe('execute', () => {
|
||||
// --- Error Handling Tests ---
|
||||
it('should return validation error if params are invalid', async () => {
|
||||
const params: CodeParserToolParams = { path: 'relative/path.txt' };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error: Invalid parameters provided. Reason: Path must be absolute/,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Error: Failed to execute tool.');
|
||||
});
|
||||
|
||||
it('should return error if target path does not exist', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'nonexistent.py');
|
||||
mockFs.stat.mockRejectedValue({
|
||||
code: 'ENOENT',
|
||||
} as NodeJS.ErrnoException);
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error: Path not found or inaccessible/,
|
||||
);
|
||||
expect(result.returnDisplay).toMatch(
|
||||
/Error: Path not found or inaccessible/,
|
||||
);
|
||||
});
|
||||
|
||||
it('should return error if target path is not a file or directory', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'neither_file_nor_dir');
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => false,
|
||||
isDirectory: () => false,
|
||||
size: 0,
|
||||
} as Stats);
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error: Path is not a file or directory/,
|
||||
);
|
||||
expect(result.returnDisplay).toMatch(
|
||||
/Error: Path is not a file or directory/,
|
||||
);
|
||||
});
|
||||
|
||||
it('should return error if no supported languages are specified or available', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'file.py');
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: 100,
|
||||
} as Stats);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const originalGetLanguageParser = (tool as any).getLanguageParser;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(tool as any).getLanguageParser = vi.fn().mockReturnValue(undefined);
|
||||
|
||||
const params: CodeParserToolParams = {
|
||||
path: targetPath,
|
||||
languages: ['fantasy-lang'],
|
||||
};
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error: No supported languages specified for parsing/,
|
||||
);
|
||||
expect(result.returnDisplay).toMatch(
|
||||
/Error: No supported languages to parse/,
|
||||
);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(tool as any).getLanguageParser = originalGetLanguageParser; // Restore
|
||||
});
|
||||
|
||||
// --- Single File Parsing Tests ---
|
||||
it('should parse a single Python file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'test.py');
|
||||
const fileContent = 'print("hello")';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(python_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockPythonGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(python_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a single Java file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'Test.java');
|
||||
const fileContent = 'class Test {}';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(java_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockJavaGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(java_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a single Go file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'main.go');
|
||||
const fileContent = 'package main\nfunc main(){}';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(go_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockGoGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(go_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a single C# file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'Program.cs');
|
||||
const fileContent =
|
||||
'namespace HelloWorld { class Program { static void Main(string[] args) { System.Console.WriteLine("Hello World!"); } } }';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(csharp_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockCSharpGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(csharp_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a single TypeScript (.ts) file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'app.ts');
|
||||
const fileContent = 'const x: number = 10;';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(ts_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockTypeScriptGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(ts_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a single TSX (.tsx) file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'component.tsx');
|
||||
const fileContent = 'const Comp = () => <div />;';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(tsx_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockTSXGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(tsx_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a single JavaScript (.js) file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'script.js');
|
||||
const fileContent = 'console.log("hello");';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(js_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockTypeScriptGrammar); // Uses TypeScript grammar for JS
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(js_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a single Rust (.rs) file successfully', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'main.rs');
|
||||
const fileContent = 'fn main() { println!("Hello, Rust!"); }';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(rust_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockRustGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(rust_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should parse a JavaScript JSX (.jsx) file successfully (using tsx parser)', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'component.jsx');
|
||||
const fileContent = 'const Comp = () => <div />;';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockReturnValue({
|
||||
rootNode: { toString: () => '(jsx_ast)' },
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockSetLanguage).toHaveBeenCalledWith(mockTSXGrammar);
|
||||
expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
|
||||
expect(result.llmContent).toBe(
|
||||
`Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(jsx_ast)\n\n`,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Parsed 1 file(s).');
|
||||
});
|
||||
|
||||
it('should return error for unsupported file type if specified directly', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'notes.txt');
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: 10,
|
||||
} as Stats);
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error: File .* is not of a supported language type/,
|
||||
);
|
||||
expect(result.returnDisplay).toMatch(
|
||||
/Error: Unsupported file type or language/,
|
||||
);
|
||||
});
|
||||
|
||||
it('should skip file if it exceeds maxFileSize', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'large.py');
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: 1024 * 1024 + 1,
|
||||
} as Stats); // 1MB + 1 byte
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(mockFs.readFile).not.toHaveBeenCalled();
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error: Could not parse file .*large.py/,
|
||||
);
|
||||
expect(result.returnDisplay).toBe('Error: Failed to parse file.');
|
||||
});
|
||||
|
||||
it('should return error if parsing a supported file fails internally', async () => {
|
||||
const targetPath = path.join(tempRootDir, 'broken.py');
|
||||
const fileContent = 'print("hello")';
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: fileContent.length,
|
||||
} as Stats);
|
||||
mockFs.readFile.mockResolvedValue(fileContent);
|
||||
mockTreeSitterParse.mockImplementation(() => {
|
||||
throw new Error('TreeSitterCrashed');
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: targetPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error: Could not parse file .*broken.py/,
|
||||
);
|
||||
expect(result.returnDisplay).toMatch(/Error: Failed to parse file./);
|
||||
});
|
||||
|
||||
// --- Directory Parsing Tests ---
|
||||
it('should parse supported files in a directory (including Go, C#, TS, JS, TSX)', async () => {
|
||||
const dirPath = path.join(tempRootDir, 'src');
|
||||
const files = [
|
||||
'main.py',
|
||||
'helper.java',
|
||||
'service.go',
|
||||
'App.cs',
|
||||
'logic.ts',
|
||||
'ui.tsx',
|
||||
'utils.js',
|
||||
'main.rs', // Added Rust file
|
||||
'config.txt',
|
||||
];
|
||||
const pythonContent = 'import os';
|
||||
const javaContent = 'public class Helper {}';
|
||||
const goContent = 'package main';
|
||||
const csharpContent = 'public class App {}';
|
||||
const tsContent = 'let val: number = 1;';
|
||||
const tsxContent = 'const MyComp = () => <p />;';
|
||||
const jsContent = 'function hello() {}';
|
||||
const rustContent = 'fn start() {}'; // Added Rust content
|
||||
|
||||
mockFs.stat.mockImplementation(async (p) => {
|
||||
if (p === dirPath)
|
||||
return { isFile: () => false, isDirectory: () => true } as Stats;
|
||||
if (p === path.join(dirPath, 'main.py'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: pythonContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'helper.java'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: javaContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'service.go'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: goContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'App.cs'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: csharpContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'logic.ts'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: tsContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'ui.tsx'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: tsxContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'utils.js'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: jsContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'main.rs'))
|
||||
// Added for Rust
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: rustContent.length,
|
||||
} as Stats;
|
||||
if (p === path.join(dirPath, 'config.txt'))
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: 10,
|
||||
} as Stats;
|
||||
throw { code: 'ENOENT' };
|
||||
});
|
||||
mockFs.readdir.mockImplementation(
|
||||
vi.fn(async (p: PathLike): Promise<string[]> => {
|
||||
const dirPath = path.join(tempRootDir, 'src'); // Path for this specific test
|
||||
if (p === dirPath) {
|
||||
return files; // files is in scope for this test
|
||||
}
|
||||
throw new Error(
|
||||
`fs.readdir mock: Unhandled path ${p} in test 'should parse supported files in a directory'`,
|
||||
);
|
||||
}) as unknown as typeof fs.readdir,
|
||||
);
|
||||
mockFs.readFile.mockImplementation(async (p) => {
|
||||
if (p === path.join(dirPath, 'main.py')) return pythonContent;
|
||||
if (p === path.join(dirPath, 'helper.java')) return javaContent;
|
||||
if (p === path.join(dirPath, 'service.go')) return goContent;
|
||||
if (p === path.join(dirPath, 'App.cs')) return csharpContent;
|
||||
if (p === path.join(dirPath, 'logic.ts')) return tsContent;
|
||||
if (p === path.join(dirPath, 'ui.tsx')) return tsxContent;
|
||||
if (p === path.join(dirPath, 'utils.js')) return jsContent;
|
||||
if (p === path.join(dirPath, 'main.rs')) return rustContent; // Added for Rust
|
||||
return '';
|
||||
});
|
||||
mockTreeSitterParse.mockImplementation((content) => {
|
||||
if (content === pythonContent)
|
||||
return { rootNode: { toString: () => '(py_ast_dir)' } };
|
||||
if (content === javaContent)
|
||||
return { rootNode: { toString: () => '(java_ast_dir)' } };
|
||||
if (content === goContent)
|
||||
return { rootNode: { toString: () => '(go_ast_dir)' } };
|
||||
if (content === csharpContent)
|
||||
return { rootNode: { toString: () => '(csharp_ast_dir)' } };
|
||||
if (content === tsContent)
|
||||
return { rootNode: { toString: () => '(ts_ast_dir)' } };
|
||||
if (content === tsxContent)
|
||||
return { rootNode: { toString: () => '(tsx_ast_dir)' } };
|
||||
if (content === jsContent)
|
||||
return { rootNode: { toString: () => '(js_ast_dir)' } };
|
||||
if (content === rustContent)
|
||||
// Added for Rust
|
||||
return { rootNode: { toString: () => '(rust_ast_dir)' } };
|
||||
return { rootNode: { toString: () => '(other_ast)' } };
|
||||
});
|
||||
|
||||
const params: CodeParserToolParams = { path: dirPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(result.llmContent).toContain(
|
||||
`-------------${path.join(dirPath, 'main.py')}-------------\n(py_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
`-------------${path.join(dirPath, 'helper.java')}-------------\n(java_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
`-------------${path.join(dirPath, 'service.go')}-------------\n(go_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
`-------------${path.join(dirPath, 'App.cs')}-------------\n(csharp_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
`-------------${path.join(dirPath, 'logic.ts')}-------------\n(ts_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
`-------------${path.join(dirPath, 'ui.tsx')}-------------\n(tsx_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
`-------------${path.join(dirPath, 'utils.js')}-------------\n(js_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
// Added for Rust
|
||||
`-------------${path.join(dirPath, 'main.rs')}-------------\n(rust_ast_dir)\n`,
|
||||
);
|
||||
expect(result.llmContent).not.toContain('config.txt');
|
||||
expect(result.returnDisplay).toBe('Parsed 8 file(s).'); // Updated count
|
||||
});
|
||||
|
||||
it('should only parse languages specified in the languages parameter for directory', async () => {
|
||||
const dirPath = path.join(tempRootDir, 'mixed_lang_project');
|
||||
const files = [
|
||||
'script.py',
|
||||
'Main.java',
|
||||
'another.py',
|
||||
'app.go',
|
||||
'Logic.cs',
|
||||
'index.ts',
|
||||
'view.tsx',
|
||||
'helper.js',
|
||||
'main.rs', // Added Rust file
|
||||
];
|
||||
mockFs.stat.mockImplementation(async (p) => {
|
||||
if (p === dirPath)
|
||||
return { isFile: () => false, isDirectory: () => true } as Stats;
|
||||
return {
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
size: 10,
|
||||
} as Stats;
|
||||
});
|
||||
mockFs.readdir.mockImplementation(
|
||||
vi.fn(async (p: PathLike): Promise<string[]> => {
|
||||
// dirPath and files are in scope for this specific test
|
||||
if (p === dirPath) {
|
||||
return files;
|
||||
}
|
||||
throw new Error(
|
||||
`fs.readdir mock: Unhandled path ${p} in test 'should only parse languages specified'`,
|
||||
);
|
||||
}) as unknown as typeof fs.readdir,
|
||||
);
|
||||
mockFs.readFile.mockResolvedValue('content');
|
||||
|
||||
const params: CodeParserToolParams = {
|
||||
path: dirPath,
|
||||
languages: [
|
||||
'java',
|
||||
'go',
|
||||
'csharp',
|
||||
'typescript',
|
||||
'tsx',
|
||||
'javascript',
|
||||
'rust',
|
||||
], // Added rust
|
||||
};
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(result.llmContent).toContain(path.join(dirPath, 'Main.java'));
|
||||
expect(result.llmContent).toContain(path.join(dirPath, 'app.go'));
|
||||
expect(result.llmContent).toContain(path.join(dirPath, 'Logic.cs'));
|
||||
expect(result.llmContent).toContain(path.join(dirPath, 'index.ts'));
|
||||
expect(result.llmContent).toContain(path.join(dirPath, 'view.tsx'));
|
||||
expect(result.llmContent).toContain(path.join(dirPath, 'helper.js'));
|
||||
expect(result.llmContent).toContain(path.join(dirPath, 'main.rs')); // Added for Rust
|
||||
expect(result.llmContent).not.toContain('script.py');
|
||||
expect(result.llmContent).not.toContain('another.py');
|
||||
expect(result.returnDisplay).toBe('Parsed 7 file(s).'); // Updated count
|
||||
});
|
||||
|
||||
it('should return "Directory is empty" for an empty directory', async () => {
|
||||
const dirPath = path.join(tempRootDir, 'empty_dir');
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => false,
|
||||
isDirectory: () => true,
|
||||
} as Stats);
|
||||
mockFs.readdir.mockResolvedValue([]);
|
||||
|
||||
const params: CodeParserToolParams = { path: dirPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(result.llmContent).toBe(`Directory ${dirPath} is empty.`);
|
||||
expect(result.returnDisplay).toBe('Directory is empty.');
|
||||
});
|
||||
|
||||
it('should handle error if fs.readdir fails', async () => {
|
||||
const dirPath = path.join(tempRootDir, 'unreadable_dir');
|
||||
mockFs.stat.mockResolvedValue({
|
||||
isFile: () => false,
|
||||
isDirectory: () => true,
|
||||
} as Stats);
|
||||
mockFs.readdir.mockRejectedValue(new Error('Permission denied'));
|
||||
|
||||
const params: CodeParserToolParams = { path: dirPath };
|
||||
const result = await tool.execute(params, abortSignal);
|
||||
|
||||
expect(result.llmContent).toMatch(
|
||||
/Error listing or processing directory/,
|
||||
);
|
||||
expect(result.returnDisplay).toMatch(
|
||||
/Error: Failed to process directory./,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('requiresConfirmation', () => {
|
||||
it('should return null', async () => {
|
||||
const params: CodeParserToolParams = { path: 'anypath' };
|
||||
expect(await tool.requiresConfirmation(params)).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,386 +0,0 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import Parser from 'tree-sitter';
|
||||
import Python from 'tree-sitter-python';
|
||||
import Java from 'tree-sitter-java';
|
||||
import Go from 'tree-sitter-go';
|
||||
import CSharp from 'tree-sitter-c-sharp';
|
||||
import TreeSitterTypeScript from 'tree-sitter-typescript';
|
||||
import Rust from 'tree-sitter-rust'; // Added
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { BaseTool, ToolResult, ToolCallConfirmationDetails } from './tools.js';
|
||||
import { SchemaValidator } from '../utils/schemaValidator.js';
|
||||
import { makeRelative, shortenPath } from '../utils/paths.js'; // Removed isWithinRoot
|
||||
import { Config } from '../config/config.js';
|
||||
|
||||
type TreeSitterLanguage = Parameters<typeof Parser.prototype.setLanguage>[0];
|
||||
|
||||
export interface CodeParserToolParams {
|
||||
path: string;
|
||||
ignore?: string[];
|
||||
languages?: string[];
|
||||
}
|
||||
|
||||
export class CodeParserTool extends BaseTool<CodeParserToolParams, ToolResult> {
|
||||
static readonly Name = 'code_parser';
|
||||
|
||||
private parser: Parser;
|
||||
|
||||
constructor(
|
||||
private rootDirectory: string,
|
||||
private config: Config,
|
||||
) {
|
||||
super(
|
||||
CodeParserTool.Name,
|
||||
'CodeParser',
|
||||
'Parses the code in the specified directory path or a single file to generate AST representations. This should be used to get a better understanding of the codebase when refactoring and building out new features.',
|
||||
{
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
description:
|
||||
'The absolute path to the directory or file to parse (must be absolute, not relative)',
|
||||
},
|
||||
languages: {
|
||||
type: 'array',
|
||||
description:
|
||||
'Optional: specific languages to parse (e.g., ["python", "java", "go", "csharp", "typescript", "tsx", "javascript", "rust"]). Defaults to supported languages.',
|
||||
items: {
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['path'],
|
||||
type: 'object',
|
||||
},
|
||||
);
|
||||
this.rootDirectory = path.resolve(rootDirectory);
|
||||
this.parser = new Parser();
|
||||
}
|
||||
|
||||
// Added private isWithinRoot method
|
||||
private isWithinRoot(dirpath: string): boolean {
|
||||
const normalizedPath = path.normalize(dirpath);
|
||||
const normalizedRoot = path.normalize(this.rootDirectory);
|
||||
const rootWithSep = normalizedRoot.endsWith(path.sep)
|
||||
? normalizedRoot
|
||||
: normalizedRoot + path.sep;
|
||||
return (
|
||||
normalizedPath === normalizedRoot ||
|
||||
normalizedPath.startsWith(rootWithSep)
|
||||
);
|
||||
}
|
||||
|
||||
private getLanguageParser(language: string): TreeSitterLanguage | undefined {
|
||||
switch (language.toLowerCase()) {
|
||||
case 'python':
|
||||
return Python;
|
||||
case 'java':
|
||||
return Java;
|
||||
case 'go':
|
||||
return Go;
|
||||
case 'csharp':
|
||||
return CSharp;
|
||||
case 'typescript':
|
||||
return TreeSitterTypeScript.typescript;
|
||||
case 'tsx':
|
||||
return TreeSitterTypeScript.tsx;
|
||||
case 'javascript': // Use TypeScript parser for JS as it handles modern JS well
|
||||
return TreeSitterTypeScript.typescript;
|
||||
case 'rust': // Added
|
||||
return Rust; // Added
|
||||
default:
|
||||
console.warn(
|
||||
`Language '${language}' is not supported by the CodeParserTool.`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
validateToolParams(params: CodeParserToolParams): string | null {
|
||||
if (
|
||||
this.schema.parameters &&
|
||||
!SchemaValidator.validate(
|
||||
this.schema.parameters as Record<string, unknown>,
|
||||
params,
|
||||
)
|
||||
) {
|
||||
return 'Parameters failed schema validation.';
|
||||
}
|
||||
if (!path.isAbsolute(params.path)) {
|
||||
return `Path must be absolute: ${params.path}`;
|
||||
}
|
||||
if (!this.isWithinRoot(params.path)) {
|
||||
// Use the class method
|
||||
return `Path must be within the root directory (${this.rootDirectory}): ${params.path}`;
|
||||
}
|
||||
if (
|
||||
params.languages &&
|
||||
(!Array.isArray(params.languages) ||
|
||||
!params.languages.every((lang) => typeof lang === 'string'))
|
||||
) {
|
||||
return 'Languages parameter must be an array of strings.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
getDescription(params: CodeParserToolParams): string {
|
||||
const relativePath = makeRelative(params.path, this.rootDirectory);
|
||||
return `Parse ${shortenPath(relativePath)}`;
|
||||
}
|
||||
|
||||
private errorResult(llmContent: string, returnDisplay: string): ToolResult {
|
||||
return {
|
||||
llmContent,
|
||||
returnDisplay: `Error: ${returnDisplay}`,
|
||||
};
|
||||
}
|
||||
|
||||
private async parseFile(
|
||||
filePath: string,
|
||||
language: string,
|
||||
maxFileSize?: number,
|
||||
): Promise<string | null> {
|
||||
const langParser = this.getLanguageParser(language);
|
||||
if (!langParser) {
|
||||
return null;
|
||||
}
|
||||
this.parser.setLanguage(langParser);
|
||||
|
||||
try {
|
||||
const stats = await fs.stat(filePath);
|
||||
if (maxFileSize && stats.size > maxFileSize) {
|
||||
console.warn(
|
||||
`File ${filePath} exceeds maxFileSize (${stats.size} > ${maxFileSize}), skipping.`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const fileContent = await fs.readFile(filePath, 'utf8');
|
||||
const tree = this.parser.parse(fileContent);
|
||||
return this.formatTree(tree.rootNode, 0);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Error parsing file ${filePath} with language ${language}:`,
|
||||
error,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to format the AST similar to the Go version
|
||||
private formatTree(node: Parser.SyntaxNode, level: number): string {
|
||||
let formattedTree = '';
|
||||
const indent = ' '.repeat(level);
|
||||
const sexp = node.toString(); // tree-sitter's Node.toString() returns S-expression
|
||||
const maxLength = 100;
|
||||
|
||||
if (sexp.length < maxLength) {
|
||||
// MODIFIED LINE: Removed !sexp.includes('\n')
|
||||
formattedTree += `${indent}${sexp}\n`;
|
||||
return formattedTree;
|
||||
}
|
||||
|
||||
// Expand full format if the S-expression is complex or long
|
||||
formattedTree += `${indent}(${node.type}\n`;
|
||||
|
||||
for (const child of node.namedChildren) {
|
||||
formattedTree += this.formatTree(child, level + 1);
|
||||
}
|
||||
|
||||
// Iterating all children (named and unnamed) to be closer to Go's formatTree.
|
||||
// The original Go code iterates `node.NamedChildCount()` and then `node.ChildCount()`
|
||||
// which implies it processes named children and then all children (including named again).
|
||||
// Here, we iterate named, then iterate all, but skip if already processed as named.
|
||||
// This logic might need further refinement if the exact Go output for unnamed nodes is critical.
|
||||
// For now, focusing on named children as per the Go code's primary loop in formatTree.
|
||||
// If a more exact match for unnamed nodes is needed, the iteration logic for `node.children`
|
||||
// and skipping already processed namedChildren would be added here.
|
||||
|
||||
formattedTree += `${indent})\n`;
|
||||
return formattedTree;
|
||||
}
|
||||
|
||||
private getFileLanguage(filePath: string): string | undefined {
|
||||
const extension = path.extname(filePath).toLowerCase();
|
||||
switch (extension) {
|
||||
case '.py':
|
||||
return 'python';
|
||||
case '.java':
|
||||
return 'java';
|
||||
case '.go':
|
||||
return 'go';
|
||||
case '.cs':
|
||||
return 'csharp';
|
||||
case '.ts':
|
||||
return 'typescript';
|
||||
case '.tsx':
|
||||
return 'tsx';
|
||||
case '.js':
|
||||
return 'javascript';
|
||||
case '.jsx': // Treat jsx as tsx for parsing
|
||||
return 'tsx';
|
||||
case '.mjs':
|
||||
return 'javascript';
|
||||
case '.cjs':
|
||||
return 'javascript';
|
||||
case '.rs': // Added
|
||||
return 'rust'; // Added
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async execute(
|
||||
params: CodeParserToolParams,
|
||||
_signal: AbortSignal,
|
||||
): Promise<ToolResult> {
|
||||
const validationError = this.validateToolParams(params);
|
||||
if (validationError) {
|
||||
return this.errorResult(
|
||||
`Error: Invalid parameters provided. Reason: ${validationError}`,
|
||||
'Failed to execute tool.',
|
||||
);
|
||||
}
|
||||
|
||||
const targetPath = params.path;
|
||||
let stats;
|
||||
try {
|
||||
stats = await fs.stat(targetPath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return this.errorResult(
|
||||
`Error: Path not found or inaccessible: ${targetPath}`,
|
||||
'Path not found or inaccessible.',
|
||||
);
|
||||
}
|
||||
return this.errorResult(
|
||||
`Error: Cannot access path: ${(error as Error).message}`,
|
||||
'Cannot access path.',
|
||||
);
|
||||
}
|
||||
|
||||
const defaultLanguages = [
|
||||
'python',
|
||||
'java',
|
||||
'go',
|
||||
'csharp',
|
||||
'typescript',
|
||||
'tsx',
|
||||
'javascript',
|
||||
'rust', // Added
|
||||
];
|
||||
const languagesToParse = (
|
||||
params.languages && params.languages.length > 0
|
||||
? params.languages
|
||||
: defaultLanguages
|
||||
).map((lang) => lang.toLowerCase());
|
||||
const maxFileSize = 1024 * 1024; // 1MB
|
||||
|
||||
const supportedLanguagesToParse = languagesToParse.filter((lang) =>
|
||||
this.getLanguageParser(lang),
|
||||
);
|
||||
if (supportedLanguagesToParse.length === 0) {
|
||||
const availableLangs =
|
||||
defaultLanguages
|
||||
.filter((lang) => this.getLanguageParser(lang))
|
||||
.join(', ') || 'none configured';
|
||||
return this.errorResult(
|
||||
`Error: No supported languages specified for parsing. Requested: ${languagesToParse.join(', ') || 'default'}. Available: ${availableLangs}.`,
|
||||
'No supported languages to parse.',
|
||||
);
|
||||
}
|
||||
|
||||
let parsedCodeOutput = '';
|
||||
let filesProcessedCount = 0;
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
try {
|
||||
const files = await fs.readdir(targetPath);
|
||||
if (files.length === 0) {
|
||||
return {
|
||||
llmContent: `Directory ${targetPath} is empty.`,
|
||||
returnDisplay: 'Directory is empty.',
|
||||
};
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(targetPath, file);
|
||||
let fileStats;
|
||||
try {
|
||||
fileStats = await fs.stat(filePath);
|
||||
} catch {
|
||||
console.warn(`Could not stat file ${filePath}, skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fileStats.isFile()) {
|
||||
const fileLang = this.getFileLanguage(filePath);
|
||||
if (fileLang && supportedLanguagesToParse.includes(fileLang)) {
|
||||
const ast = await this.parseFile(filePath, fileLang, maxFileSize);
|
||||
if (ast) {
|
||||
parsedCodeOutput += `-------------${filePath}-------------\n`;
|
||||
parsedCodeOutput += ast + '\n';
|
||||
filesProcessedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
return this.errorResult(
|
||||
`Error listing or processing directory ${targetPath}: ${(error as Error).message}`,
|
||||
'Failed to process directory.',
|
||||
);
|
||||
}
|
||||
} else if (stats.isFile()) {
|
||||
const fileLang = this.getFileLanguage(targetPath);
|
||||
if (fileLang && supportedLanguagesToParse.includes(fileLang)) {
|
||||
const ast = await this.parseFile(targetPath, fileLang, maxFileSize);
|
||||
if (ast) {
|
||||
parsedCodeOutput += `-------------${targetPath}-------------\n`;
|
||||
parsedCodeOutput += ast + '\n';
|
||||
filesProcessedCount++;
|
||||
} else {
|
||||
return this.errorResult(
|
||||
`Error: Could not parse file ${targetPath}. Language '${fileLang}' is supported but parsing failed. Check logs.`,
|
||||
'Failed to parse file.',
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return this.errorResult(
|
||||
`Error: File ${targetPath} is not of a supported language type for parsing or language not specified. Supported: ${supportedLanguagesToParse.join(', ')}. Detected extension for language: ${fileLang || 'unknown'}.`,
|
||||
'Unsupported file type or language.',
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return this.errorResult(
|
||||
`Error: Path is not a file or directory: ${targetPath}`,
|
||||
'Path is not a file or directory.',
|
||||
);
|
||||
}
|
||||
|
||||
if (filesProcessedCount === 0) {
|
||||
return {
|
||||
llmContent: `No files were parsed in ${targetPath}. Ensure files match supported languages (${supportedLanguagesToParse.join(', ')}), are not empty or too large, and are not ignored.`,
|
||||
returnDisplay: 'No files parsed.',
|
||||
};
|
||||
}
|
||||
|
||||
const returnDisplay = `Parsed ${filesProcessedCount} file(s).`;
|
||||
return {
|
||||
llmContent: `Parsed code from ${targetPath}:\n${parsedCodeOutput}`,
|
||||
returnDisplay,
|
||||
};
|
||||
}
|
||||
|
||||
async requiresConfirmation(
|
||||
_params: CodeParserToolParams,
|
||||
): Promise<ToolCallConfirmationDetails | null> {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue