mirror of
https://github.com/harivansh-afk/evaluclaude-harness.git
synced 2026-04-15 05:02:09 +00:00
grader, test renderer
This commit is contained in:
parent
9297f0b1ee
commit
e0c36241b0
22 changed files with 1914 additions and 5 deletions
494
package-lock.json
generated
494
package-lock.json
generated
|
|
@ -10,9 +10,12 @@
|
|||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.4",
|
||||
"@anthropic-ai/sdk": "^0.39.0",
|
||||
"commander": "^12.1.0",
|
||||
"glob": "^10.4.0",
|
||||
"handlebars": "^4.7.8",
|
||||
"inquirer": "^9.2.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"tree-sitter": "^0.21.1",
|
||||
"tree-sitter-python": "^0.21.0",
|
||||
"tree-sitter-typescript": "^0.21.0"
|
||||
|
|
@ -22,6 +25,7 @@
|
|||
},
|
||||
"devDependencies": {
|
||||
"@types/inquirer": "^9.0.7",
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
"@types/node": "^20.14.0",
|
||||
"eslint": "^8.57.0",
|
||||
"typescript": "^5.4.5",
|
||||
|
|
@ -53,6 +57,36 @@
|
|||
"zod": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/sdk": {
|
||||
"version": "0.39.0",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.39.0.tgz",
|
||||
"integrity": "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "^18.11.18",
|
||||
"@types/node-fetch": "^2.6.4",
|
||||
"abort-controller": "^3.0.0",
|
||||
"agentkeepalive": "^4.2.1",
|
||||
"form-data-encoder": "1.7.2",
|
||||
"formdata-node": "^4.3.2",
|
||||
"node-fetch": "^2.6.7"
|
||||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/sdk/node_modules/@types/node": {
|
||||
"version": "18.19.130",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz",
|
||||
"integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/sdk/node_modules/undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
"version": "0.21.5",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",
|
||||
|
|
@ -1399,16 +1433,32 @@
|
|||
"rxjs": "^7.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/js-yaml": {
|
||||
"version": "4.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz",
|
||||
"integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "20.19.28",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.28.tgz",
|
||||
"integrity": "sha512-VyKBr25BuFDzBFCK5sUM6ZXiWfqgCTwTAOK8qzGV/m9FCirXYDlmczJ+d5dXBAQALGCdRRdbteKYfJ84NGEusw==",
|
||||
"devOptional": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~6.21.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node-fetch": {
|
||||
"version": "2.6.13",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz",
|
||||
"integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^4.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/through": {
|
||||
"version": "0.0.33",
|
||||
"resolved": "https://registry.npmjs.org/@types/through/-/through-0.0.33.tgz",
|
||||
|
|
@ -1529,6 +1579,18 @@
|
|||
"url": "https://opencollective.com/vitest"
|
||||
}
|
||||
},
|
||||
"node_modules/abort-controller": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
||||
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"event-target-shim": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.5"
|
||||
}
|
||||
},
|
||||
"node_modules/acorn": {
|
||||
"version": "8.15.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
|
||||
|
|
@ -1565,6 +1627,18 @@
|
|||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agentkeepalive": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz",
|
||||
"integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"humanize-ms": "^1.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "6.12.6",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
|
||||
|
|
@ -1637,7 +1711,6 @@
|
|||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
|
||||
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
|
||||
"dev": true,
|
||||
"license": "Python-2.0"
|
||||
},
|
||||
"node_modules/assertion-error": {
|
||||
|
|
@ -1650,6 +1723,12 @@
|
|||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
|
|
@ -1732,6 +1811,19 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/call-bind-apply-helpers": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
|
||||
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"es-errors": "^1.3.0",
|
||||
"function-bind": "^1.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/callsites": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
|
||||
|
|
@ -1856,6 +1948,18 @@
|
|||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "12.1.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
|
||||
|
|
@ -1943,6 +2047,15 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/diff-sequences": {
|
||||
"version": "29.6.3",
|
||||
"resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz",
|
||||
|
|
@ -1966,6 +2079,20 @@
|
|||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"call-bind-apply-helpers": "^1.0.1",
|
||||
"es-errors": "^1.3.0",
|
||||
"gopd": "^1.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/eastasianwidth": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
|
||||
|
|
@ -1978,6 +2105,51 @@
|
|||
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/es-define-property": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
|
||||
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/es-errors": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
|
||||
"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/es-object-atoms": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
|
||||
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"es-errors": "^1.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/es-set-tostringtag": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
|
||||
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"es-errors": "^1.3.0",
|
||||
"get-intrinsic": "^1.2.6",
|
||||
"has-tostringtag": "^1.0.2",
|
||||
"hasown": "^2.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.21.5",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz",
|
||||
|
|
@ -2191,6 +2363,15 @@
|
|||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/event-target-shim": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
||||
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/execa": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz",
|
||||
|
|
@ -2343,6 +2524,41 @@
|
|||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/form-data": {
|
||||
"version": "4.0.5",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
|
||||
"integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"es-set-tostringtag": "^2.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"mime-types": "^2.1.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/form-data-encoder": {
|
||||
"version": "1.7.2",
|
||||
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
|
||||
"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/formdata-node": {
|
||||
"version": "4.4.1",
|
||||
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
|
||||
"integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-domexception": "1.0.0",
|
||||
"web-streams-polyfill": "4.0.0-beta.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 12.20"
|
||||
}
|
||||
},
|
||||
"node_modules/fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
|
|
@ -2365,6 +2581,15 @@
|
|||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/function-bind": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
|
||||
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/get-func-name": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
|
||||
|
|
@ -2375,6 +2600,43 @@
|
|||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/get-intrinsic": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
|
||||
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"call-bind-apply-helpers": "^1.0.2",
|
||||
"es-define-property": "^1.0.1",
|
||||
"es-errors": "^1.3.0",
|
||||
"es-object-atoms": "^1.1.1",
|
||||
"function-bind": "^1.1.2",
|
||||
"get-proto": "^1.0.1",
|
||||
"gopd": "^1.2.0",
|
||||
"has-symbols": "^1.1.0",
|
||||
"hasown": "^2.0.2",
|
||||
"math-intrinsics": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/get-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
|
||||
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"dunder-proto": "^1.0.1",
|
||||
"es-object-atoms": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/get-stream": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz",
|
||||
|
|
@ -2461,6 +2723,18 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/gopd": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
|
||||
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/graphemer": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz",
|
||||
|
|
@ -2468,6 +2742,27 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/handlebars": {
|
||||
"version": "4.7.8",
|
||||
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz",
|
||||
"integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"minimist": "^1.2.5",
|
||||
"neo-async": "^2.6.2",
|
||||
"source-map": "^0.6.1",
|
||||
"wordwrap": "^1.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"handlebars": "bin/handlebars"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.4.7"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"uglify-js": "^3.1.4"
|
||||
}
|
||||
},
|
||||
"node_modules/has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
|
|
@ -2477,6 +2772,45 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/has-symbols": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
|
||||
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/has-tostringtag": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
|
||||
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"has-symbols": "^1.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/hasown": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
|
||||
"integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"function-bind": "^1.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/human-signals": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz",
|
||||
|
|
@ -2487,6 +2821,15 @@
|
|||
"node": ">=16.17.0"
|
||||
}
|
||||
},
|
||||
"node_modules/humanize-ms": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
|
||||
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.7.2",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
|
||||
|
|
@ -2709,7 +3052,6 @@
|
|||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
|
||||
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"argparse": "^2.0.1"
|
||||
|
|
@ -2845,6 +3187,15 @@
|
|||
"@jridgewell/sourcemap-codec": "^1.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/math-intrinsics": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
|
||||
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/merge-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
|
||||
|
|
@ -2852,6 +3203,27 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/mime-db": {
|
||||
"version": "1.52.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
|
||||
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/mime-types": {
|
||||
"version": "2.1.35",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
|
||||
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mime-db": "1.52.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/mimic-fn": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
|
||||
|
|
@ -2874,6 +3246,15 @@
|
|||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/minimist": {
|
||||
"version": "1.2.8",
|
||||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
|
||||
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/minipass": {
|
||||
"version": "7.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
|
||||
|
|
@ -2907,7 +3288,6 @@
|
|||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/mute-stream": {
|
||||
|
|
@ -2945,6 +3325,12 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/neo-async": {
|
||||
"version": "2.6.2",
|
||||
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
|
||||
"integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-addon-api": {
|
||||
"version": "8.5.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.5.0.tgz",
|
||||
|
|
@ -2954,6 +3340,46 @@
|
|||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"deprecated": "Use your platform's native DOMException instead",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp-build": {
|
||||
"version": "4.8.4",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
|
||||
|
|
@ -3550,6 +3976,15 @@
|
|||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
|
||||
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
|
|
@ -3722,6 +4157,12 @@
|
|||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tree-sitter": {
|
||||
"version": "0.21.1",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.21.1.tgz",
|
||||
|
|
@ -3840,11 +4281,23 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uglify-js": {
|
||||
"version": "3.19.3",
|
||||
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz",
|
||||
"integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==",
|
||||
"license": "BSD-2-Clause",
|
||||
"optional": true,
|
||||
"bin": {
|
||||
"uglifyjs": "bin/uglifyjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "6.21.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
|
||||
"devOptional": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uri-js": {
|
||||
|
|
@ -4021,6 +4474,31 @@
|
|||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "4.0.0-beta.3",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
|
||||
"integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
|
|
@ -4063,6 +4541,12 @@
|
|||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wordwrap": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
|
||||
"integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/wrap-ansi": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
|
||||
|
|
|
|||
|
|
@ -27,15 +27,19 @@
|
|||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.4",
|
||||
"@anthropic-ai/sdk": "^0.39.0",
|
||||
"commander": "^12.1.0",
|
||||
"glob": "^10.4.0",
|
||||
"handlebars": "^4.7.8",
|
||||
"inquirer": "^9.2.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"tree-sitter": "^0.21.1",
|
||||
"tree-sitter-python": "^0.21.0",
|
||||
"tree-sitter-typescript": "^0.21.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/inquirer": "^9.0.7",
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
"@types/node": "^20.14.0",
|
||||
"eslint": "^8.57.0",
|
||||
"typescript": "^5.4.5",
|
||||
|
|
|
|||
53
prompts/grader-system.md
Normal file
53
prompts/grader-system.md
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# LLM Rubric Grader
|
||||
|
||||
You are an expert evaluator with deep experience in code quality assessment. Your task is to grade output against a structured rubric with precision and consistency.
|
||||
|
||||
## Your Role
|
||||
|
||||
- You evaluate objectively against the criteria provided
|
||||
- You provide actionable feedback that helps improve quality
|
||||
- You score consistently—the same quality should always receive the same score
|
||||
- You justify every score with specific evidence from the output
|
||||
|
||||
## Evaluation Process
|
||||
|
||||
1. **Read the rubric** — Understand each criterion, its weight, and what good/bad looks like
|
||||
2. **Analyze the output** — Examine it thoroughly before scoring
|
||||
3. **Score independently** — Rate each criterion without letting others influence it
|
||||
4. **Cite evidence** — Every score must reference specific parts of the output
|
||||
5. **Calculate overall** — Compute weighted average accurately
|
||||
|
||||
## Scoring Scale
|
||||
|
||||
| Score | Meaning |
|
||||
|-------|---------|
|
||||
| 0.0 | Complete failure, criterion not addressed |
|
||||
| 0.1-0.3 | Major deficiencies, fundamental issues |
|
||||
| 0.4-0.5 | Below expectations, significant gaps |
|
||||
| 0.6-0.7 | Meets basic requirements, room for improvement |
|
||||
| 0.8-0.9 | Exceeds expectations, minor issues only |
|
||||
| 1.0 | Exemplary, no improvements needed |
|
||||
|
||||
## Critical Rules
|
||||
|
||||
- **Never score 1.0 unless truly perfect** — Reserve it for exceptional cases
|
||||
- **Never score 0.0 unless completely absent** — Even poor attempts get some credit
|
||||
- **Be specific in feedback** — "Could be better" is not helpful; "Variable name 'x' should describe its purpose" is
|
||||
- **Consider context** — A quick script has different quality expectations than a library API
|
||||
|
||||
## Output Format
|
||||
|
||||
Return ONLY valid JSON. No markdown, no explanation outside the JSON.
|
||||
|
||||
```json
|
||||
{
|
||||
"scores": {
|
||||
"criterion_name": {
|
||||
"score": 0.0,
|
||||
"feedback": "Specific, actionable feedback citing evidence"
|
||||
}
|
||||
},
|
||||
"overall": 0.0,
|
||||
"summary": "One-sentence overall assessment"
|
||||
}
|
||||
```
|
||||
33
prompts/grader-user.md
Normal file
33
prompts/grader-user.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Grading Request
|
||||
|
||||
## Rubric: {{RUBRIC_NAME}}
|
||||
|
||||
{{RUBRIC_DESCRIPTION}}
|
||||
|
||||
**Passing Threshold:** {{PASSING_THRESHOLD}}%
|
||||
|
||||
### Criteria
|
||||
|
||||
{{CRITERIA_LIST}}
|
||||
|
||||
---
|
||||
|
||||
## Output to Evaluate
|
||||
|
||||
```
|
||||
{{OUTPUT}}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Your Task
|
||||
|
||||
1. Evaluate the output against each criterion above
|
||||
2. Provide a score (0.0-1.0) and specific feedback for each
|
||||
3. Calculate the weighted overall score
|
||||
4. Return your assessment as JSON
|
||||
|
||||
Remember:
|
||||
- Cite specific evidence from the output for each score
|
||||
- The overall score must equal the weighted average of criterion scores
|
||||
- Feedback should be actionable and specific
|
||||
101
src/cli/commands/grade.ts
Normal file
101
src/cli/commands/grade.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import { Command } from 'commander';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { gradeWithRubric, loadAllRubrics, analyzeCalibration, calibrate } from '../../graders/index.js';
|
||||
import type { CalibrationExample } from '../../graders/types.js';
|
||||
|
||||
export const gradeCommand = new Command('grade')
|
||||
.description('Grade output using LLM rubric')
|
||||
.argument('<input>', 'Path to input file or string to grade')
|
||||
.option('-r, --rubric <name>', 'Rubric name or path', 'code-quality')
|
||||
.option('--rubrics-dir <dir>', 'Directory containing rubric YAML files', 'rubrics')
|
||||
.option('--json', 'Output result as JSON', false)
|
||||
.action(async (input: string, options) => {
|
||||
try {
|
||||
let content: string;
|
||||
|
||||
if (existsSync(input)) {
|
||||
content = readFileSync(input, 'utf-8');
|
||||
} else {
|
||||
content = input;
|
||||
}
|
||||
|
||||
console.log(`Grading with rubric: ${options.rubric}`);
|
||||
|
||||
const result = await gradeWithRubric(content, options.rubric, {
|
||||
rubricsDir: options.rubricsDir,
|
||||
});
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n${result.pass ? '✅ PASS' : '❌ FAIL'}`);
|
||||
console.log(`Score: ${(result.score * 100).toFixed(1)}%`);
|
||||
console.log(`\nSummary: ${result.reason}`);
|
||||
|
||||
console.log('\nCriterion Scores:');
|
||||
for (const cs of result.criterionScores) {
|
||||
const bar = '█'.repeat(Math.round(cs.score * 10)) + '░'.repeat(10 - Math.round(cs.score * 10));
|
||||
console.log(` ${cs.name}: ${bar} ${(cs.score * 100).toFixed(0)}%`);
|
||||
console.log(` ${cs.feedback}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error grading:', error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
export const listRubricsCommand = new Command('rubrics')
|
||||
.description('List available rubrics')
|
||||
.option('--rubrics-dir <dir>', 'Directory containing rubric YAML files', 'rubrics')
|
||||
.action(async (options) => {
|
||||
try {
|
||||
const rubrics = loadAllRubrics(options.rubricsDir);
|
||||
|
||||
if (rubrics.size === 0) {
|
||||
console.log(`No rubrics found in ${options.rubricsDir}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Available rubrics (${rubrics.size}):\n`);
|
||||
|
||||
for (const [name, rubric] of rubrics) {
|
||||
console.log(`📋 ${name}`);
|
||||
console.log(` ${rubric.description}`);
|
||||
console.log(` Threshold: ${(rubric.passingThreshold * 100).toFixed(0)}%`);
|
||||
console.log(` Criteria: ${rubric.criteria.map(c => c.name).join(', ')}`);
|
||||
console.log('');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error listing rubrics:', error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
export const calibrateCommand = new Command('calibrate')
|
||||
.description('Calibrate a rubric against known examples')
|
||||
.argument('<rubric>', 'Rubric name or path')
|
||||
.argument('<examples>', 'Path to calibration examples JSON')
|
||||
.option('--rubrics-dir <dir>', 'Directory containing rubric YAML files', 'rubrics')
|
||||
.action(async (rubricName: string, examplesPath: string, options) => {
|
||||
try {
|
||||
if (!existsSync(examplesPath)) {
|
||||
console.error(`Examples file not found: ${examplesPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const examples: CalibrationExample[] = JSON.parse(readFileSync(examplesPath, 'utf-8'));
|
||||
|
||||
console.log(`Calibrating rubric '${rubricName}' with ${examples.length} examples...`);
|
||||
|
||||
const result = await calibrate(rubricName, examples, {
|
||||
rubricsDir: options.rubricsDir,
|
||||
});
|
||||
|
||||
console.log('\n' + analyzeCalibration(result));
|
||||
} catch (error) {
|
||||
console.error('Error calibrating:', error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
61
src/cli/commands/render.ts
Normal file
61
src/cli/commands/render.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { Command } from 'commander';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { renderSpec, detectFramework, type Framework } from '../../renderers/index.js';
|
||||
import type { EvalSpec } from '../../analyzer/types.js';
|
||||
|
||||
export const renderCommand = new Command('render')
|
||||
.description('Render EvalSpec JSON into runnable test files')
|
||||
.argument('<spec>', 'Path to EvalSpec JSON file')
|
||||
.option('-o, --output <dir>', 'Output directory for test files', './tests/generated')
|
||||
.option('-f, --framework <framework>', 'Test framework (pytest, vitest, jest)')
|
||||
.option('--fixtures', 'Generate fixture stubs', false)
|
||||
.option('--mocks', 'Generate mock stubs', false)
|
||||
.option('--dry-run', 'Preview without writing files', false)
|
||||
.action(async (specPath: string, options) => {
|
||||
try {
|
||||
if (!existsSync(specPath)) {
|
||||
console.error(`Error: Spec file not found: ${specPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const specContent = readFileSync(specPath, 'utf-8');
|
||||
const spec: EvalSpec = JSON.parse(specContent);
|
||||
|
||||
const framework = (options.framework as Framework) || detectFramework(spec);
|
||||
|
||||
console.log(`Rendering ${spec.scenarios.length} scenarios with ${framework}...`);
|
||||
|
||||
const result = await renderSpec(spec, {
|
||||
outputDir: options.output,
|
||||
framework,
|
||||
includeFixtures: options.fixtures,
|
||||
generateMocks: options.mocks,
|
||||
dryRun: options.dryRun,
|
||||
});
|
||||
|
||||
if (options.dryRun) {
|
||||
console.log('\n--- DRY RUN ---\n');
|
||||
for (const file of result.files) {
|
||||
console.log(`📄 ${file.path}`);
|
||||
console.log('---');
|
||||
console.log(file.content);
|
||||
console.log('---\n');
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n✅ Rendered ${result.stats.scenarioCount} scenarios`);
|
||||
console.log(` 📁 ${result.stats.fileCount} test files`);
|
||||
console.log(` 🔍 ${result.stats.assertionCount} assertions`);
|
||||
|
||||
if (result.stats.skippedCount > 0) {
|
||||
console.log(` ⏭️ ${result.stats.skippedCount} scenarios skipped (LLM rubric assertions)`);
|
||||
}
|
||||
|
||||
if (!options.dryRun) {
|
||||
console.log(`\n📂 Output: ${options.output}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error rendering spec:', error instanceof Error ? error.message : error);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
|
@ -3,6 +3,8 @@
|
|||
import { Command } from 'commander';
|
||||
import { introCommand } from './commands/intro.js';
|
||||
import { analyzeCommand } from './commands/analyze.js';
|
||||
import { renderCommand } from './commands/render.js';
|
||||
import { gradeCommand, listRubricsCommand, calibrateCommand } from './commands/grade.js';
|
||||
|
||||
const program = new Command();
|
||||
|
||||
|
|
@ -13,5 +15,9 @@ program
|
|||
|
||||
program.addCommand(introCommand);
|
||||
program.addCommand(analyzeCommand);
|
||||
program.addCommand(renderCommand);
|
||||
program.addCommand(gradeCommand);
|
||||
program.addCommand(listRubricsCommand);
|
||||
program.addCommand(calibrateCommand);
|
||||
|
||||
program.parse(process.argv);
|
||||
|
|
|
|||
22
src/graders/index.ts
Normal file
22
src/graders/index.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
export type {
|
||||
Rubric,
|
||||
RubricCriterion,
|
||||
RubricGradingResult,
|
||||
CriterionScore,
|
||||
GradeRequest,
|
||||
GraderOptions,
|
||||
CalibrationExample,
|
||||
CalibrationSet,
|
||||
CalibrationResult,
|
||||
} from './types.js';
|
||||
|
||||
export {
|
||||
LLMGrader,
|
||||
gradeWithRubric,
|
||||
loadRubric,
|
||||
loadAllRubrics,
|
||||
clearRubricCache,
|
||||
formatRubricForPrompt,
|
||||
calibrate,
|
||||
analyzeCalibration,
|
||||
} from './llm/index.js';
|
||||
68
src/graders/llm/calibrator.ts
Normal file
68
src/graders/llm/calibrator.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import type { Rubric, CalibrationExample, CalibrationResult, GraderOptions } from '../types.js';
|
||||
import { gradeWithRubric } from './grader.js';
|
||||
import { loadRubric } from './rubric-loader.js';
|
||||
|
||||
const AGREEMENT_THRESHOLD = 0.1;
|
||||
const MIN_AGREEMENT_RATE = 0.8;
|
||||
|
||||
export async function calibrate(
|
||||
rubricNameOrDef: string | Rubric,
|
||||
examples: CalibrationExample[],
|
||||
options?: GraderOptions
|
||||
): Promise<CalibrationResult> {
|
||||
const rubric = typeof rubricNameOrDef === 'string'
|
||||
? loadRubric(rubricNameOrDef, options?.rubricsDir)
|
||||
: rubricNameOrDef;
|
||||
|
||||
const results = await Promise.all(
|
||||
examples.map(async (ex) => {
|
||||
const result = await gradeWithRubric(ex.input, rubric, options);
|
||||
return {
|
||||
example: ex,
|
||||
actualScore: result.score,
|
||||
difference: result.score - ex.expectedScore,
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
const withinThreshold = results.filter(r =>
|
||||
Math.abs(r.difference) < AGREEMENT_THRESHOLD
|
||||
);
|
||||
|
||||
const agreement = withinThreshold.length / results.length;
|
||||
const drift = results.map(r => r.difference);
|
||||
|
||||
return {
|
||||
agreement,
|
||||
drift,
|
||||
needsAdjustment: agreement < MIN_AGREEMENT_RATE,
|
||||
details: results,
|
||||
};
|
||||
}
|
||||
|
||||
export function analyzeCalibration(result: CalibrationResult): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
lines.push(`Calibration Results`);
|
||||
lines.push(`==================`);
|
||||
lines.push(`Agreement Rate: ${(result.agreement * 100).toFixed(1)}%`);
|
||||
lines.push(`Status: ${result.needsAdjustment ? '⚠️ Needs Adjustment' : '✅ Calibrated'}`);
|
||||
lines.push('');
|
||||
|
||||
if (result.drift.length > 0) {
|
||||
const avgDrift = result.drift.reduce((a, b) => a + b, 0) / result.drift.length;
|
||||
const maxDrift = Math.max(...result.drift.map(Math.abs));
|
||||
|
||||
lines.push(`Average Drift: ${avgDrift > 0 ? '+' : ''}${avgDrift.toFixed(3)}`);
|
||||
lines.push(`Max Absolute Drift: ${maxDrift.toFixed(3)}`);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
lines.push(`Individual Results:`);
|
||||
for (const detail of result.details) {
|
||||
const status = Math.abs(detail.difference) < AGREEMENT_THRESHOLD ? '✓' : '✗';
|
||||
lines.push(` ${status} Expected: ${detail.example.expectedScore.toFixed(2)}, Actual: ${detail.actualScore.toFixed(2)}, Diff: ${detail.difference > 0 ? '+' : ''}${detail.difference.toFixed(3)}`);
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
98
src/graders/llm/grader.ts
Normal file
98
src/graders/llm/grader.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import type { Rubric, RubricGradingResult, GraderOptions, CriterionScore } from '../types.js';
|
||||
import { loadRubric } from './rubric-loader.js';
|
||||
import { buildGraderSystemPrompt, buildGraderUserPrompt } from './prompt-builder.js';
|
||||
|
||||
const DEFAULT_MODEL = 'claude-sonnet-4-20250514';
|
||||
const DEFAULT_MAX_TOKENS = 1024;
|
||||
|
||||
interface GradingResponse {
|
||||
scores: Record<string, { score: number; feedback: string }>;
|
||||
overall: number;
|
||||
summary: string;
|
||||
}
|
||||
|
||||
export class LLMGrader {
|
||||
private client: Anthropic;
|
||||
private options: Required<GraderOptions>;
|
||||
|
||||
constructor(options: GraderOptions = {}) {
|
||||
this.client = new Anthropic();
|
||||
this.options = {
|
||||
model: options.model || DEFAULT_MODEL,
|
||||
maxTokens: options.maxTokens || DEFAULT_MAX_TOKENS,
|
||||
rubricsDir: options.rubricsDir || 'rubrics',
|
||||
};
|
||||
}
|
||||
|
||||
async grade(output: string, rubricNameOrDef: string | Rubric): Promise<RubricGradingResult> {
|
||||
const rubric = typeof rubricNameOrDef === 'string'
|
||||
? loadRubric(rubricNameOrDef, this.options.rubricsDir)
|
||||
: rubricNameOrDef;
|
||||
|
||||
const systemPrompt = buildGraderSystemPrompt();
|
||||
const userPrompt = buildGraderUserPrompt(output, rubric);
|
||||
|
||||
const response = await this.client.messages.create({
|
||||
model: this.options.model,
|
||||
max_tokens: this.options.maxTokens,
|
||||
system: systemPrompt,
|
||||
messages: [{ role: 'user', content: userPrompt }],
|
||||
});
|
||||
|
||||
const responseText = response.content[0].type === 'text'
|
||||
? response.content[0].text
|
||||
: '';
|
||||
|
||||
const parsed = this.parseResponse(responseText);
|
||||
|
||||
return this.buildResult(parsed, rubric);
|
||||
}
|
||||
|
||||
private parseResponse(text: string): GradingResponse {
|
||||
const jsonMatch = text.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) {
|
||||
throw new Error('Failed to extract JSON from grader response');
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(jsonMatch[0]) as GradingResponse;
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to parse grader response as JSON: ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
private buildResult(parsed: GradingResponse, rubric: Rubric): RubricGradingResult {
|
||||
const criterionScores: CriterionScore[] = rubric.criteria.map(c => {
|
||||
const score = parsed.scores[c.name];
|
||||
return {
|
||||
name: c.name,
|
||||
score: score?.score ?? 0,
|
||||
feedback: score?.feedback ?? 'No feedback provided',
|
||||
};
|
||||
});
|
||||
|
||||
const weightedScore = rubric.criteria.reduce((sum, c) => {
|
||||
const criterionScore = parsed.scores[c.name]?.score ?? 0;
|
||||
return sum + criterionScore * c.weight;
|
||||
}, 0);
|
||||
|
||||
const finalScore = parsed.overall ?? weightedScore;
|
||||
|
||||
return {
|
||||
pass: finalScore >= rubric.passingThreshold,
|
||||
score: finalScore,
|
||||
reason: parsed.summary || 'No summary provided',
|
||||
criterionScores,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function gradeWithRubric(
|
||||
output: string,
|
||||
rubricNameOrDef: string | Rubric,
|
||||
options?: GraderOptions
|
||||
): Promise<RubricGradingResult> {
|
||||
const grader = new LLMGrader(options);
|
||||
return grader.grade(output, rubricNameOrDef);
|
||||
}
|
||||
4
src/graders/llm/index.ts
Normal file
4
src/graders/llm/index.ts
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
export { LLMGrader, gradeWithRubric } from './grader.js';
|
||||
export { loadRubric, loadAllRubrics, clearRubricCache, formatRubricForPrompt } from './rubric-loader.js';
|
||||
export { calibrate, analyzeCalibration } from './calibrator.js';
|
||||
export { buildGraderSystemPrompt, buildGraderUserPrompt, clearPromptCache } from './prompt-builder.js';
|
||||
50
src/graders/llm/prompt-builder.ts
Normal file
50
src/graders/llm/prompt-builder.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { readFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import type { Rubric } from '../types.js';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PROMPTS_DIR = join(__dirname, '../../../prompts');
|
||||
|
||||
let promptCache: Map<string, string> = new Map();
|
||||
|
||||
function loadPrompt(name: string): string {
|
||||
if (promptCache.has(name)) {
|
||||
return promptCache.get(name)!;
|
||||
}
|
||||
|
||||
const filePath = join(PROMPTS_DIR, `${name}.md`);
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
promptCache.set(name, content);
|
||||
return content;
|
||||
}
|
||||
|
||||
export function buildGraderSystemPrompt(): string {
|
||||
return loadPrompt('grader-system');
|
||||
}
|
||||
|
||||
export function buildGraderUserPrompt(output: string, rubric: Rubric): string {
|
||||
const template = loadPrompt('grader-user');
|
||||
|
||||
const criteriaList = rubric.criteria.map(c => {
|
||||
let entry = `#### ${c.name} (weight: ${(c.weight * 100).toFixed(0)}%)\n\n${c.description}`;
|
||||
|
||||
if (c.examples) {
|
||||
entry += `\n\n**Good example:** ${c.examples.good}`;
|
||||
entry += `\n**Bad example:** ${c.examples.bad}`;
|
||||
}
|
||||
|
||||
return entry;
|
||||
}).join('\n\n');
|
||||
|
||||
return template
|
||||
.replace('{{RUBRIC_NAME}}', rubric.name)
|
||||
.replace('{{RUBRIC_DESCRIPTION}}', rubric.description)
|
||||
.replace('{{PASSING_THRESHOLD}}', String(Math.round(rubric.passingThreshold * 100)))
|
||||
.replace('{{CRITERIA_LIST}}', criteriaList)
|
||||
.replace('{{OUTPUT}}', output);
|
||||
}
|
||||
|
||||
export function clearPromptCache(): void {
|
||||
promptCache.clear();
|
||||
}
|
||||
127
src/graders/llm/rubric-loader.ts
Normal file
127
src/graders/llm/rubric-loader.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import { readFileSync, existsSync, readdirSync } from 'fs';
|
||||
import { join, basename } from 'path';
|
||||
import yaml from 'js-yaml';
|
||||
import type { Rubric } from '../types.js';
|
||||
|
||||
const DEFAULT_RUBRICS_DIR = 'rubrics';
|
||||
|
||||
let rubricCache: Map<string, Rubric> = new Map();
|
||||
|
||||
export function loadRubric(nameOrPath: string, rubricsDir: string = DEFAULT_RUBRICS_DIR): Rubric {
|
||||
if (rubricCache.has(nameOrPath)) {
|
||||
return rubricCache.get(nameOrPath)!;
|
||||
}
|
||||
|
||||
let rubricPath: string;
|
||||
|
||||
if (existsSync(nameOrPath)) {
|
||||
rubricPath = nameOrPath;
|
||||
} else {
|
||||
rubricPath = join(rubricsDir, `${nameOrPath}.yaml`);
|
||||
if (!existsSync(rubricPath)) {
|
||||
rubricPath = join(rubricsDir, `${nameOrPath}.yml`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!existsSync(rubricPath)) {
|
||||
throw new Error(`Rubric not found: ${nameOrPath} (searched in ${rubricsDir})`);
|
||||
}
|
||||
|
||||
const content = readFileSync(rubricPath, 'utf-8');
|
||||
const rubric = yaml.load(content) as Rubric;
|
||||
|
||||
validateRubric(rubric);
|
||||
rubricCache.set(nameOrPath, rubric);
|
||||
|
||||
return rubric;
|
||||
}
|
||||
|
||||
export function loadAllRubrics(rubricsDir: string = DEFAULT_RUBRICS_DIR): Map<string, Rubric> {
|
||||
if (!existsSync(rubricsDir)) {
|
||||
return new Map();
|
||||
}
|
||||
|
||||
const files = readdirSync(rubricsDir).filter(f => f.endsWith('.yaml') || f.endsWith('.yml'));
|
||||
const rubrics = new Map<string, Rubric>();
|
||||
|
||||
for (const file of files) {
|
||||
const name = basename(file).replace(/\.(yaml|yml)$/, '');
|
||||
try {
|
||||
const rubric = loadRubric(join(rubricsDir, file));
|
||||
rubrics.set(name, rubric);
|
||||
} catch (e) {
|
||||
console.warn(`Failed to load rubric ${file}:`, e);
|
||||
}
|
||||
}
|
||||
|
||||
return rubrics;
|
||||
}
|
||||
|
||||
function validateRubric(rubric: unknown): asserts rubric is Rubric {
|
||||
if (!rubric || typeof rubric !== 'object') {
|
||||
throw new Error('Rubric must be an object');
|
||||
}
|
||||
|
||||
const r = rubric as Record<string, unknown>;
|
||||
|
||||
if (typeof r.name !== 'string') {
|
||||
throw new Error('Rubric must have a name (string)');
|
||||
}
|
||||
if (typeof r.description !== 'string') {
|
||||
throw new Error('Rubric must have a description (string)');
|
||||
}
|
||||
if (typeof r.passingThreshold !== 'number' || r.passingThreshold < 0 || r.passingThreshold > 1) {
|
||||
throw new Error('Rubric must have a passingThreshold between 0 and 1');
|
||||
}
|
||||
if (!Array.isArray(r.criteria) || r.criteria.length === 0) {
|
||||
throw new Error('Rubric must have at least one criterion');
|
||||
}
|
||||
|
||||
for (const criterion of r.criteria) {
|
||||
validateCriterion(criterion);
|
||||
}
|
||||
|
||||
const totalWeight = (r.criteria as Array<{ weight: number }>).reduce((sum, c) => sum + c.weight, 0);
|
||||
if (Math.abs(totalWeight - 1) > 0.01) {
|
||||
console.warn(`Rubric '${r.name}' weights sum to ${totalWeight}, not 1.0`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateCriterion(criterion: unknown): void {
|
||||
if (!criterion || typeof criterion !== 'object') {
|
||||
throw new Error('Criterion must be an object');
|
||||
}
|
||||
|
||||
const c = criterion as Record<string, unknown>;
|
||||
|
||||
if (typeof c.name !== 'string') {
|
||||
throw new Error('Criterion must have a name');
|
||||
}
|
||||
if (typeof c.description !== 'string') {
|
||||
throw new Error('Criterion must have a description');
|
||||
}
|
||||
if (typeof c.weight !== 'number' || c.weight < 0 || c.weight > 1) {
|
||||
throw new Error('Criterion must have a weight between 0 and 1');
|
||||
}
|
||||
}
|
||||
|
||||
export function clearRubricCache(): void {
|
||||
rubricCache.clear();
|
||||
}
|
||||
|
||||
export function formatRubricForPrompt(rubric: Rubric): string {
|
||||
let prompt = `# ${rubric.name}\n\n${rubric.description}\n\nPassing threshold: ${rubric.passingThreshold * 100}%\n\n## Criteria\n\n`;
|
||||
|
||||
for (const criterion of rubric.criteria) {
|
||||
prompt += `### ${criterion.name} (weight: ${criterion.weight * 100}%)\n`;
|
||||
prompt += `${criterion.description}\n`;
|
||||
|
||||
if (criterion.examples) {
|
||||
prompt += `\n**Good example:** ${criterion.examples.good}\n`;
|
||||
prompt += `**Bad example:** ${criterion.examples.bad}\n`;
|
||||
}
|
||||
prompt += '\n';
|
||||
}
|
||||
|
||||
return prompt;
|
||||
}
|
||||
63
src/graders/types.ts
Normal file
63
src/graders/types.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
export interface Rubric {
|
||||
name: string;
|
||||
description: string;
|
||||
criteria: RubricCriterion[];
|
||||
passingThreshold: number;
|
||||
}
|
||||
|
||||
export interface RubricCriterion {
|
||||
name: string;
|
||||
description: string;
|
||||
weight: number;
|
||||
examples?: {
|
||||
good: string;
|
||||
bad: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface RubricGradingResult {
|
||||
pass: boolean;
|
||||
score: number;
|
||||
reason: string;
|
||||
criterionScores: CriterionScore[];
|
||||
}
|
||||
|
||||
export interface CriterionScore {
|
||||
name: string;
|
||||
score: number;
|
||||
feedback: string;
|
||||
}
|
||||
|
||||
export interface GradeRequest {
|
||||
output: string;
|
||||
rubric: string | Rubric;
|
||||
context?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface GraderOptions {
|
||||
model?: string;
|
||||
maxTokens?: number;
|
||||
rubricsDir?: string;
|
||||
}
|
||||
|
||||
export interface CalibrationExample {
|
||||
input: string;
|
||||
expectedScore: number;
|
||||
expectedFeedback?: string[];
|
||||
}
|
||||
|
||||
export interface CalibrationSet {
|
||||
rubric: string;
|
||||
examples: CalibrationExample[];
|
||||
}
|
||||
|
||||
export interface CalibrationResult {
|
||||
agreement: number;
|
||||
drift: number[];
|
||||
needsAdjustment: boolean;
|
||||
details: {
|
||||
example: CalibrationExample;
|
||||
actualScore: number;
|
||||
difference: number;
|
||||
}[];
|
||||
}
|
||||
|
|
@ -1,2 +1,4 @@
|
|||
export * from './introspector/index.js';
|
||||
export * from './analyzer/index.js';
|
||||
export * from './renderers/index.js';
|
||||
export * from './graders/index.js';
|
||||
|
|
|
|||
96
src/renderers/base.ts
Normal file
96
src/renderers/base.ts
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import type { EvalSpec, EvalScenario } from '../analyzer/types.js';
|
||||
import type { RenderOptions, RenderResult, GeneratedFile, ScenarioGroup, RenderStats } from './types.js';
|
||||
|
||||
export abstract class BaseRenderer {
|
||||
protected options: RenderOptions;
|
||||
|
||||
constructor(options: RenderOptions) {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
abstract get language(): 'python' | 'typescript';
|
||||
abstract get fileExtension(): string;
|
||||
|
||||
async render(spec: EvalSpec): Promise<RenderResult> {
|
||||
const groups = this.groupByModule(spec.scenarios);
|
||||
const files: GeneratedFile[] = [];
|
||||
let assertionCount = 0;
|
||||
let skippedCount = 0;
|
||||
|
||||
for (const group of groups) {
|
||||
const validScenarios = group.scenarios.filter(s => this.canRender(s));
|
||||
skippedCount += group.scenarios.length - validScenarios.length;
|
||||
|
||||
if (validScenarios.length === 0) continue;
|
||||
|
||||
const content = this.renderTestFile(group.module, validScenarios);
|
||||
const path = this.getOutputPath(group.module);
|
||||
|
||||
assertionCount += validScenarios.reduce((sum, s) => sum + s.assertions.length, 0);
|
||||
|
||||
files.push({
|
||||
path,
|
||||
content,
|
||||
scenarios: validScenarios.map(s => s.id),
|
||||
language: this.language,
|
||||
});
|
||||
}
|
||||
|
||||
const stats: RenderStats = {
|
||||
scenarioCount: spec.scenarios.length - skippedCount,
|
||||
fileCount: files.length,
|
||||
assertionCount,
|
||||
skippedCount,
|
||||
};
|
||||
|
||||
return { files, stats };
|
||||
}
|
||||
|
||||
protected groupByModule(scenarios: EvalScenario[]): ScenarioGroup[] {
|
||||
const groups = new Map<string, EvalScenario[]>();
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
const module = scenario.target.module;
|
||||
if (!groups.has(module)) {
|
||||
groups.set(module, []);
|
||||
}
|
||||
groups.get(module)!.push(scenario);
|
||||
}
|
||||
|
||||
return Array.from(groups.entries()).map(([module, scenarios]) => ({
|
||||
module,
|
||||
scenarios,
|
||||
}));
|
||||
}
|
||||
|
||||
protected canRender(scenario: EvalScenario): boolean {
|
||||
return scenario.assertions.every(a => a.type !== 'llm-rubric');
|
||||
}
|
||||
|
||||
protected getOutputPath(modulePath: string): string {
|
||||
const baseName = modulePath
|
||||
.replace(/\.(py|ts|tsx|js|jsx)$/, '')
|
||||
.replace(/\//g, '_');
|
||||
return `${this.options.outputDir}/test_${baseName}${this.fileExtension}`;
|
||||
}
|
||||
|
||||
protected abstract renderTestFile(module: string, scenarios: EvalScenario[]): string;
|
||||
|
||||
protected abstract renderAssertion(assertion: EvalScenario['assertions'][0], resultVar: string): string;
|
||||
|
||||
protected toSnakeCase(str: string): string {
|
||||
return str.replace(/-/g, '_').replace(/([A-Z])/g, '_$1').toLowerCase();
|
||||
}
|
||||
|
||||
protected toCamelCase(str: string): string {
|
||||
return str.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
|
||||
}
|
||||
|
||||
protected formatValue(value: unknown): string {
|
||||
if (value === null) return 'null';
|
||||
if (value === undefined) return 'undefined';
|
||||
if (typeof value === 'string') return JSON.stringify(value);
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
}
|
||||
67
src/renderers/index.ts
Normal file
67
src/renderers/index.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import type { EvalSpec } from '../analyzer/types.js';
|
||||
import type { RenderOptions, RenderResult, Framework, GeneratedFile } from './types.js';
|
||||
import { BaseRenderer } from './base.js';
|
||||
import { PytestRenderer } from './python/pytest-renderer.js';
|
||||
import { VitestRenderer, JestRenderer } from './typescript/vitest-renderer.js';
|
||||
import { writeFileSync, mkdirSync } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
|
||||
export type { RenderOptions, RenderResult, GeneratedFile, Framework } from './types.js';
|
||||
export { BaseRenderer } from './base.js';
|
||||
export { PytestRenderer } from './python/pytest-renderer.js';
|
||||
export { VitestRenderer, JestRenderer } from './typescript/vitest-renderer.js';
|
||||
|
||||
const rendererRegistry: Record<Framework, new (options: RenderOptions) => BaseRenderer> = {
|
||||
pytest: PytestRenderer,
|
||||
vitest: VitestRenderer,
|
||||
jest: JestRenderer,
|
||||
};
|
||||
|
||||
export function createRenderer(options: RenderOptions): BaseRenderer {
|
||||
const RendererClass = rendererRegistry[options.framework];
|
||||
if (!RendererClass) {
|
||||
throw new Error(`Unknown framework: ${options.framework}`);
|
||||
}
|
||||
return new RendererClass(options);
|
||||
}
|
||||
|
||||
export async function renderSpec(spec: EvalSpec, options: RenderOptions): Promise<RenderResult> {
|
||||
const renderer = createRenderer(options);
|
||||
const result = await renderer.render(spec);
|
||||
|
||||
if (!options.dryRun) {
|
||||
for (const file of result.files) {
|
||||
mkdirSync(dirname(file.path), { recursive: true });
|
||||
writeFileSync(file.path, file.content, 'utf-8');
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function renderIncremental(
|
||||
spec: EvalSpec,
|
||||
options: RenderOptions,
|
||||
changedFiles: string[]
|
||||
): Promise<RenderResult> {
|
||||
const filteredSpec: EvalSpec = {
|
||||
...spec,
|
||||
scenarios: spec.scenarios.filter(s =>
|
||||
changedFiles.some(f => s.target.module.includes(f))
|
||||
),
|
||||
};
|
||||
return renderSpec(filteredSpec, options);
|
||||
}
|
||||
|
||||
export function detectFramework(spec: EvalSpec): Framework {
|
||||
const languages = spec.repo.languages;
|
||||
|
||||
if (languages.includes('python')) {
|
||||
return 'pytest';
|
||||
}
|
||||
if (languages.includes('typescript') || languages.includes('javascript')) {
|
||||
return 'vitest';
|
||||
}
|
||||
|
||||
return 'vitest';
|
||||
}
|
||||
104
src/renderers/python/assertions.ts
Normal file
104
src/renderers/python/assertions.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import type { Assertion } from '../../analyzer/types.js';
|
||||
|
||||
export function renderPytestAssertion(assertion: Assertion, resultVar: string): string {
|
||||
const target = getTargetExpression(assertion, resultVar);
|
||||
|
||||
switch (assertion.type) {
|
||||
case 'equals':
|
||||
return `assert ${target} == ${formatPythonValue(assertion.expected)}`;
|
||||
|
||||
case 'contains':
|
||||
return `assert ${formatPythonValue(assertion.value)} in ${target}`;
|
||||
|
||||
case 'typeof':
|
||||
return `assert isinstance(${target}, ${mapPythonType(assertion.expected)})`;
|
||||
|
||||
case 'matches':
|
||||
return `assert re.match(${formatPythonValue(assertion.pattern)}, ${target})`;
|
||||
|
||||
case 'throws':
|
||||
return ''; // Handled specially in test structure
|
||||
|
||||
case 'truthy':
|
||||
return `assert ${target}`;
|
||||
|
||||
case 'falsy':
|
||||
return `assert not ${target}`;
|
||||
|
||||
case 'custom':
|
||||
return `assert ${assertion.check} # ${assertion.description}`;
|
||||
|
||||
case 'llm-rubric':
|
||||
return `# LLM Rubric: ${assertion.rubric} - skipped (requires grader)`;
|
||||
|
||||
default:
|
||||
return `# Unknown assertion type: ${(assertion as Assertion).type}`;
|
||||
}
|
||||
}
|
||||
|
||||
function getTargetExpression(assertion: Assertion, resultVar: string): string {
|
||||
if ('path' in assertion && assertion.path) {
|
||||
const path = assertion.path;
|
||||
if (path.startsWith('[')) {
|
||||
return `${resultVar}${path}`;
|
||||
}
|
||||
return `${resultVar}["${path}"]`;
|
||||
}
|
||||
return resultVar;
|
||||
}
|
||||
|
||||
function formatPythonValue(value: unknown): string {
|
||||
if (value === null) return 'None';
|
||||
if (value === undefined) return 'None';
|
||||
if (typeof value === 'boolean') return value ? 'True' : 'False';
|
||||
if (typeof value === 'string') return JSON.stringify(value);
|
||||
if (typeof value === 'number') return String(value);
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map(formatPythonValue).join(', ')}]`;
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
const entries = Object.entries(value)
|
||||
.map(([k, v]) => `"${k}": ${formatPythonValue(v)}`)
|
||||
.join(', ');
|
||||
return `{${entries}}`;
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function mapPythonType(tsType: string): string {
|
||||
const typeMap: Record<string, string> = {
|
||||
'string': 'str',
|
||||
'number': '(int, float)',
|
||||
'boolean': 'bool',
|
||||
'object': 'dict',
|
||||
'array': 'list',
|
||||
'null': 'type(None)',
|
||||
'undefined': 'type(None)',
|
||||
};
|
||||
return typeMap[tsType] || tsType;
|
||||
}
|
||||
|
||||
export function renderThrowsContext(assertion: Assertion): { contextManager: string; exceptionType: string } | null {
|
||||
if (assertion.type !== 'throws') return null;
|
||||
|
||||
const exceptionType = assertion.errorType || 'Exception';
|
||||
let contextManager = `pytest.raises(${exceptionType})`;
|
||||
|
||||
if (assertion.messageContains) {
|
||||
contextManager = `pytest.raises(${exceptionType}, match=${formatPythonValue(assertion.messageContains)})`;
|
||||
}
|
||||
|
||||
return { contextManager, exceptionType };
|
||||
}
|
||||
|
||||
export function formatPythonArgs(args: Record<string, unknown>): string {
|
||||
return Object.entries(args)
|
||||
.map(([_, value]) => formatPythonValue(value))
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
export function formatPythonKwargs(kwargs: Record<string, unknown>): string {
|
||||
return Object.entries(kwargs)
|
||||
.map(([key, value]) => `${key}=${formatPythonValue(value)}`)
|
||||
.join(', ');
|
||||
}
|
||||
160
src/renderers/python/pytest-renderer.ts
Normal file
160
src/renderers/python/pytest-renderer.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import type { EvalScenario, Assertion } from '../../analyzer/types.js';
|
||||
import { BaseRenderer } from '../base.js';
|
||||
import { renderPytestAssertion, renderThrowsContext, formatPythonArgs, formatPythonKwargs } from './assertions.js';
|
||||
|
||||
export class PytestRenderer extends BaseRenderer {
|
||||
get language(): 'python' {
|
||||
return 'python';
|
||||
}
|
||||
|
||||
get fileExtension(): string {
|
||||
return '.py';
|
||||
}
|
||||
|
||||
protected renderTestFile(module: string, scenarios: EvalScenario[]): string {
|
||||
const imports = this.generateImports(module, scenarios);
|
||||
const fixtures = this.options.includeFixtures ? this.generateFixtures(scenarios) : '';
|
||||
const tests = scenarios.map(s => this.renderTest(s)).join('\n\n');
|
||||
|
||||
return `${imports}\n\n${fixtures}${tests}\n`;
|
||||
}
|
||||
|
||||
private generateImports(module: string, scenarios: EvalScenario[]): string {
|
||||
const imports: string[] = [
|
||||
'import pytest',
|
||||
];
|
||||
|
||||
const hasRegex = scenarios.some(s =>
|
||||
s.assertions.some(a => a.type === 'matches')
|
||||
);
|
||||
if (hasRegex) {
|
||||
imports.push('import re');
|
||||
}
|
||||
|
||||
const hasMocks = scenarios.some(s => s.setup?.mocks?.length);
|
||||
if (hasMocks || this.options.generateMocks) {
|
||||
imports.push('from unittest.mock import patch, MagicMock');
|
||||
}
|
||||
|
||||
const functions = [...new Set(scenarios.map(s => s.target.function))];
|
||||
const modulePath = module.replace(/\.(py|ts|tsx|js|jsx)$/, '').replace(/\//g, '.');
|
||||
imports.push(`from ${modulePath} import ${functions.join(', ')}`);
|
||||
|
||||
return imports.join('\n');
|
||||
}
|
||||
|
||||
private generateFixtures(scenarios: EvalScenario[]): string {
|
||||
const fixtureNames = new Set<string>();
|
||||
for (const scenario of scenarios) {
|
||||
if (scenario.setup?.fixtures) {
|
||||
for (const fixture of scenario.setup.fixtures) {
|
||||
fixtureNames.add(fixture);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fixtureNames.size === 0) return '';
|
||||
|
||||
const fixtures = Array.from(fixtureNames).map(name => `
|
||||
@pytest.fixture
|
||||
def ${name}():
|
||||
# TODO: Implement fixture
|
||||
pass
|
||||
`).join('\n');
|
||||
|
||||
return fixtures + '\n';
|
||||
}
|
||||
|
||||
private renderTest(scenario: EvalScenario): string {
|
||||
const testName = `test_${this.toSnakeCase(scenario.id)}`;
|
||||
const docstring = scenario.description ? ` """${scenario.description}"""\n` : '';
|
||||
|
||||
const throwsAssertion = scenario.assertions.find(a => a.type === 'throws');
|
||||
const regularAssertions = scenario.assertions.filter(a => a.type !== 'throws');
|
||||
|
||||
const fixtureParams = scenario.setup?.fixtures?.join(', ') || '';
|
||||
const funcParams = fixtureParams ? `(${fixtureParams})` : '()';
|
||||
|
||||
let body: string;
|
||||
if (throwsAssertion) {
|
||||
body = this.renderThrowsTest(scenario, throwsAssertion);
|
||||
} else {
|
||||
body = this.renderRegularTest(scenario, regularAssertions);
|
||||
}
|
||||
|
||||
const mocks = this.renderMocks(scenario);
|
||||
if (mocks) {
|
||||
body = mocks.decorators + `def ${testName}${funcParams}:\n${docstring}${mocks.setup}${body}`;
|
||||
} else {
|
||||
body = `def ${testName}${funcParams}:\n${docstring}${body}`;
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
private renderRegularTest(scenario: EvalScenario, assertions: Assertion[]): string {
|
||||
const funcCall = this.renderFunctionCall(scenario);
|
||||
const assertionLines = assertions
|
||||
.map(a => this.renderAssertion(a, 'result'))
|
||||
.filter(Boolean)
|
||||
.map(a => ` ${a}`)
|
||||
.join('\n');
|
||||
|
||||
return ` result = ${funcCall}\n${assertionLines}`;
|
||||
}
|
||||
|
||||
private renderThrowsTest(scenario: EvalScenario, throwsAssertion: Assertion): string {
|
||||
const ctx = renderThrowsContext(throwsAssertion);
|
||||
if (!ctx) return this.renderRegularTest(scenario, scenario.assertions);
|
||||
|
||||
const funcCall = this.renderFunctionCall(scenario);
|
||||
return ` with ${ctx.contextManager}:\n ${funcCall}`;
|
||||
}
|
||||
|
||||
private renderFunctionCall(scenario: EvalScenario): string {
|
||||
const func = scenario.target.function;
|
||||
const args = formatPythonArgs(scenario.input.args);
|
||||
const kwargs = scenario.input.kwargs ? formatPythonKwargs(scenario.input.kwargs) : '';
|
||||
|
||||
const allArgs = [args, kwargs].filter(Boolean).join(', ');
|
||||
return `${func}(${allArgs})`;
|
||||
}
|
||||
|
||||
private renderMocks(scenario: EvalScenario): { decorators: string; setup: string } | null {
|
||||
if (!scenario.setup?.mocks?.length && !this.options.generateMocks) return null;
|
||||
|
||||
const mocks = scenario.setup?.mocks || [];
|
||||
if (mocks.length === 0) return null;
|
||||
|
||||
const decorators = mocks
|
||||
.map((m, i) => `@patch("${m.target}")\n`)
|
||||
.join('');
|
||||
|
||||
const setup = mocks
|
||||
.map((m, i) => {
|
||||
const mockName = `mock_${i}`;
|
||||
if (m.returnValue !== undefined) {
|
||||
return ` ${mockName}.return_value = ${this.formatValue(m.returnValue)}\n`;
|
||||
}
|
||||
if (m.sideEffect) {
|
||||
return ` ${mockName}.side_effect = ${m.sideEffect}\n`;
|
||||
}
|
||||
return '';
|
||||
})
|
||||
.join('');
|
||||
|
||||
return { decorators, setup };
|
||||
}
|
||||
|
||||
protected renderAssertion(assertion: Assertion, resultVar: string): string {
|
||||
return renderPytestAssertion(assertion, resultVar);
|
||||
}
|
||||
|
||||
protected formatValue(value: unknown): string {
|
||||
if (value === null) return 'None';
|
||||
if (value === undefined) return 'None';
|
||||
if (typeof value === 'boolean') return value ? 'True' : 'False';
|
||||
if (typeof value === 'string') return JSON.stringify(value);
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
40
src/renderers/types.ts
Normal file
40
src/renderers/types.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import type { EvalSpec, EvalScenario } from '../analyzer/types.js';
|
||||
|
||||
export type Framework = 'pytest' | 'vitest' | 'jest';
|
||||
|
||||
export interface RenderOptions {
|
||||
outputDir: string;
|
||||
framework: Framework;
|
||||
includeFixtures: boolean;
|
||||
generateMocks: boolean;
|
||||
dryRun?: boolean;
|
||||
}
|
||||
|
||||
export interface RenderResult {
|
||||
files: GeneratedFile[];
|
||||
stats: RenderStats;
|
||||
}
|
||||
|
||||
export interface RenderStats {
|
||||
scenarioCount: number;
|
||||
fileCount: number;
|
||||
assertionCount: number;
|
||||
skippedCount: number;
|
||||
}
|
||||
|
||||
export interface GeneratedFile {
|
||||
path: string;
|
||||
content: string;
|
||||
scenarios: string[];
|
||||
language: 'python' | 'typescript';
|
||||
}
|
||||
|
||||
export interface RendererContext {
|
||||
spec: EvalSpec;
|
||||
options: RenderOptions;
|
||||
}
|
||||
|
||||
export interface ScenarioGroup {
|
||||
module: string;
|
||||
scenarios: EvalScenario[];
|
||||
}
|
||||
114
src/renderers/typescript/assertions.ts
Normal file
114
src/renderers/typescript/assertions.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
import type { Assertion } from '../../analyzer/types.js';
|
||||
|
||||
export type TSFramework = 'vitest' | 'jest';
|
||||
|
||||
export function renderTSAssertion(assertion: Assertion, resultVar: string, framework: TSFramework): string {
|
||||
const target = getTargetExpression(assertion, resultVar);
|
||||
|
||||
switch (assertion.type) {
|
||||
case 'equals':
|
||||
if (typeof assertion.expected === 'object' && assertion.expected !== null) {
|
||||
return `expect(${target}).toEqual(${formatTSValue(assertion.expected)});`;
|
||||
}
|
||||
return `expect(${target}).toBe(${formatTSValue(assertion.expected)});`;
|
||||
|
||||
case 'contains':
|
||||
return `expect(${target}).toContain(${formatTSValue(assertion.value)});`;
|
||||
|
||||
case 'typeof':
|
||||
return renderTypeofAssertion(target, assertion.expected, framework);
|
||||
|
||||
case 'matches':
|
||||
return `expect(${target}).toMatch(${formatTSValue(assertion.pattern)});`;
|
||||
|
||||
case 'throws':
|
||||
return ''; // Handled specially in test structure
|
||||
|
||||
case 'truthy':
|
||||
return `expect(${target}).toBeTruthy();`;
|
||||
|
||||
case 'falsy':
|
||||
return `expect(${target}).toBeFalsy();`;
|
||||
|
||||
case 'custom':
|
||||
return `expect(${assertion.check}).toBe(true); // ${assertion.description}`;
|
||||
|
||||
case 'llm-rubric':
|
||||
return `// LLM Rubric: ${assertion.rubric} - skipped (requires grader)`;
|
||||
|
||||
default:
|
||||
return `// Unknown assertion type: ${(assertion as Assertion).type}`;
|
||||
}
|
||||
}
|
||||
|
||||
function renderTypeofAssertion(target: string, expected: string, _framework: TSFramework): string {
|
||||
switch (expected) {
|
||||
case 'array':
|
||||
return `expect(Array.isArray(${target})).toBe(true);`;
|
||||
case 'null':
|
||||
return `expect(${target}).toBeNull();`;
|
||||
case 'undefined':
|
||||
return `expect(${target}).toBeUndefined();`;
|
||||
case 'object':
|
||||
return `expect(typeof ${target}).toBe('object');`;
|
||||
default:
|
||||
return `expect(typeof ${target}).toBe('${expected}');`;
|
||||
}
|
||||
}
|
||||
|
||||
function getTargetExpression(assertion: Assertion, resultVar: string): string {
|
||||
if ('path' in assertion && assertion.path) {
|
||||
const path = assertion.path;
|
||||
if (path.startsWith('[')) {
|
||||
return `${resultVar}${path}`;
|
||||
}
|
||||
if (path.includes('.')) {
|
||||
return `${resultVar}.${path}`;
|
||||
}
|
||||
return `${resultVar}['${path}']`;
|
||||
}
|
||||
return resultVar;
|
||||
}
|
||||
|
||||
function formatTSValue(value: unknown): string {
|
||||
if (value === null) return 'null';
|
||||
if (value === undefined) return 'undefined';
|
||||
if (typeof value === 'string') return JSON.stringify(value);
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map(formatTSValue).join(', ')}]`;
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
const entries = Object.entries(value)
|
||||
.map(([k, v]) => `${k}: ${formatTSValue(v)}`)
|
||||
.join(', ');
|
||||
return `{ ${entries} }`;
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
export function renderThrowsExpectation(
|
||||
funcCall: string,
|
||||
assertion: Assertion,
|
||||
isAsync: boolean
|
||||
): string {
|
||||
if (assertion.type !== 'throws') return '';
|
||||
|
||||
const expectFn = isAsync
|
||||
? `await expect(async () => ${funcCall})`
|
||||
: `expect(() => ${funcCall})`;
|
||||
|
||||
const throwMatcher = isAsync ? 'rejects.toThrow' : 'toThrow';
|
||||
|
||||
if (assertion.errorType) {
|
||||
return `${expectFn}.${throwMatcher}(${assertion.errorType});`;
|
||||
}
|
||||
if (assertion.messageContains) {
|
||||
return `${expectFn}.${throwMatcher}(${formatTSValue(assertion.messageContains)});`;
|
||||
}
|
||||
return `${expectFn}.${throwMatcher}();`;
|
||||
}
|
||||
|
||||
export function formatTSArgs(args: Record<string, unknown>): string {
|
||||
return Object.values(args).map(formatTSValue).join(', ');
|
||||
}
|
||||
152
src/renderers/typescript/vitest-renderer.ts
Normal file
152
src/renderers/typescript/vitest-renderer.ts
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import type { EvalScenario, Assertion } from '../../analyzer/types.js';
|
||||
import { BaseRenderer } from '../base.js';
|
||||
import { renderTSAssertion, renderThrowsExpectation, formatTSArgs } from './assertions.js';
|
||||
|
||||
export class VitestRenderer extends BaseRenderer {
|
||||
get language(): 'typescript' {
|
||||
return 'typescript';
|
||||
}
|
||||
|
||||
get fileExtension(): string {
|
||||
return '.test.ts';
|
||||
}
|
||||
|
||||
protected renderTestFile(module: string, scenarios: EvalScenario[]): string {
|
||||
const imports = this.generateImports(module, scenarios);
|
||||
const describes = this.generateDescribe(module, scenarios);
|
||||
return `${imports}\n\n${describes}\n`;
|
||||
}
|
||||
|
||||
protected generateImports(module: string, scenarios: EvalScenario[]): string {
|
||||
const imports: string[] = [
|
||||
`import { describe, it, expect } from 'vitest';`,
|
||||
];
|
||||
|
||||
const hasMocks = scenarios.some(s => s.setup?.mocks?.length);
|
||||
if (hasMocks || this.options.generateMocks) {
|
||||
imports.push(`import { vi } from 'vitest';`);
|
||||
}
|
||||
|
||||
const functions = [...new Set(scenarios.map(s => s.target.function))];
|
||||
const modulePath = module.replace(/\.(ts|tsx|js|jsx)$/, '');
|
||||
imports.push(`import { ${functions.join(', ')} } from '${modulePath}';`);
|
||||
|
||||
return imports.join('\n');
|
||||
}
|
||||
|
||||
protected generateDescribe(module: string, scenarios: EvalScenario[]): string {
|
||||
const moduleName = module.split('/').pop()?.replace(/\.(ts|tsx|js|jsx)$/, '') || module;
|
||||
const tests = scenarios.map(s => this.renderTest(s)).join('\n\n');
|
||||
return `describe('${moduleName}', () => {\n${tests}\n});`;
|
||||
}
|
||||
|
||||
protected renderTest(scenario: EvalScenario): string {
|
||||
const testName = scenario.name || scenario.id;
|
||||
const isAsync = this.hasAsyncTarget(scenario);
|
||||
const asyncPrefix = isAsync ? 'async ' : '';
|
||||
|
||||
const throwsAssertion = scenario.assertions.find(a => a.type === 'throws');
|
||||
const regularAssertions = scenario.assertions.filter(a => a.type !== 'throws');
|
||||
|
||||
let body: string;
|
||||
if (throwsAssertion) {
|
||||
body = this.renderThrowsTest(scenario, throwsAssertion, isAsync);
|
||||
} else {
|
||||
body = this.renderRegularTest(scenario, regularAssertions, isAsync);
|
||||
}
|
||||
|
||||
const mocks = this.renderMocks(scenario);
|
||||
const mockSetup = mocks ? `\n${mocks}` : '';
|
||||
|
||||
return ` it('${testName}', ${asyncPrefix}() => {${mockSetup}
|
||||
${body}
|
||||
});`;
|
||||
}
|
||||
|
||||
protected hasAsyncTarget(scenario: EvalScenario): boolean {
|
||||
return scenario.target.type === 'function' &&
|
||||
(scenario.target.function.startsWith('async') ||
|
||||
scenario.tags?.includes('async'));
|
||||
}
|
||||
|
||||
protected renderRegularTest(scenario: EvalScenario, assertions: Assertion[], isAsync: boolean): string {
|
||||
const funcCall = this.renderFunctionCall(scenario);
|
||||
const awaitPrefix = isAsync ? 'await ' : '';
|
||||
|
||||
const assertionLines = assertions
|
||||
.map(a => this.renderAssertion(a, 'result'))
|
||||
.filter(Boolean)
|
||||
.map(a => ` ${a}`)
|
||||
.join('\n');
|
||||
|
||||
return ` const result = ${awaitPrefix}${funcCall};\n${assertionLines}`;
|
||||
}
|
||||
|
||||
protected renderThrowsTest(scenario: EvalScenario, throwsAssertion: Assertion, isAsync: boolean): string {
|
||||
const funcCall = this.renderFunctionCall(scenario);
|
||||
return ` ${renderThrowsExpectation(funcCall, throwsAssertion, isAsync)}`;
|
||||
}
|
||||
|
||||
protected renderFunctionCall(scenario: EvalScenario): string {
|
||||
const func = scenario.target.function;
|
||||
const args = formatTSArgs(scenario.input.args);
|
||||
return `${func}(${args})`;
|
||||
}
|
||||
|
||||
protected renderMocks(scenario: EvalScenario): string | null {
|
||||
if (!scenario.setup?.mocks?.length) return null;
|
||||
|
||||
return scenario.setup.mocks
|
||||
.map(m => {
|
||||
if (m.returnValue !== undefined) {
|
||||
return ` vi.mock('${m.target}', () => (${JSON.stringify(m.returnValue)}));`;
|
||||
}
|
||||
if (m.sideEffect) {
|
||||
return ` vi.mock('${m.target}', () => { throw new Error('${m.sideEffect}'); });`;
|
||||
}
|
||||
return ` vi.mock('${m.target}');`;
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
protected renderAssertion(assertion: Assertion, resultVar: string): string {
|
||||
return renderTSAssertion(assertion, resultVar, 'vitest');
|
||||
}
|
||||
}
|
||||
|
||||
export class JestRenderer extends VitestRenderer {
|
||||
protected generateImports(module: string, scenarios: EvalScenario[]): string {
|
||||
const imports: string[] = [];
|
||||
|
||||
const hasMocks = scenarios.some(s => s.setup?.mocks?.length);
|
||||
if (hasMocks || this.options.generateMocks) {
|
||||
imports.push(`import { jest } from '@jest/globals';`);
|
||||
}
|
||||
|
||||
const functions = [...new Set(scenarios.map(s => s.target.function))];
|
||||
const modulePath = module.replace(/\.(ts|tsx|js|jsx)$/, '');
|
||||
imports.push(`import { ${functions.join(', ')} } from '${modulePath}';`);
|
||||
|
||||
return imports.join('\n');
|
||||
}
|
||||
|
||||
protected renderMocks(scenario: EvalScenario): string | null {
|
||||
if (!scenario.setup?.mocks?.length) return null;
|
||||
|
||||
return scenario.setup.mocks
|
||||
.map(m => {
|
||||
if (m.returnValue !== undefined) {
|
||||
return ` jest.mock('${m.target}', () => (${JSON.stringify(m.returnValue)}));`;
|
||||
}
|
||||
if (m.sideEffect) {
|
||||
return ` jest.mock('${m.target}', () => { throw new Error('${m.sideEffect}'); });`;
|
||||
}
|
||||
return ` jest.mock('${m.target}');`;
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
protected renderAssertion(assertion: Assertion, resultVar: string): string {
|
||||
return renderTSAssertion(assertion, resultVar, 'jest');
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue