95 lines
2.6 KiB
JavaScript
95 lines
2.6 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
|
|
const [jobsDir] = process.argv.slice(2);
|
|
|
|
if (!jobsDir) {
|
|
console.error("Usage: verify-required-skill.mjs <jobs-dir>");
|
|
process.exit(2);
|
|
}
|
|
|
|
function collectTrajectories(directory) {
|
|
const trajectories = [];
|
|
for (const entry of fs.readdirSync(directory, {withFileTypes: true})) {
|
|
const entryPath = path.join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
trajectories.push(...collectTrajectories(entryPath));
|
|
} else if (entry.isFile() && entry.name === "acp_trajectory.jsonl") {
|
|
trajectories.push(entryPath);
|
|
}
|
|
}
|
|
return trajectories;
|
|
}
|
|
|
|
function contentText(event) {
|
|
return (event.content ?? [])
|
|
.map((item) => item?.content?.text)
|
|
.filter((text) => typeof text === "string")
|
|
.join("\n");
|
|
}
|
|
|
|
const invokedSkills = new Set();
|
|
const attemptedSkillCalls = [];
|
|
const parseErrors = [];
|
|
const trajectories = collectTrajectories(jobsDir);
|
|
|
|
for (const trajectory of trajectories) {
|
|
const lines = fs.readFileSync(trajectory, "utf8").split(/\r?\n/);
|
|
for (let index = 0; index < lines.length; index += 1) {
|
|
if (!lines[index].trim()) continue;
|
|
let event;
|
|
try {
|
|
event = JSON.parse(lines[index]);
|
|
} catch (error) {
|
|
parseErrors.push(`${trajectory}:${index + 1}: ${error.message}`);
|
|
continue;
|
|
}
|
|
if (event.type !== "tool_call") continue;
|
|
const text = contentText(event);
|
|
if (event.title === "skill") {
|
|
attemptedSkillCalls.push({
|
|
status: event.status ?? "unknown",
|
|
message: text.slice(0, 500),
|
|
trajectory,
|
|
line: index + 1,
|
|
});
|
|
}
|
|
if (event.status !== "completed") continue;
|
|
for (const match of text.matchAll(/<skill_content\s+name="([^"]+)"/g)) {
|
|
invokedSkills.add(match[1]);
|
|
}
|
|
}
|
|
}
|
|
|
|
const invoked = attemptedSkillCalls.some((attempt) => attempt.status === "completed");
|
|
const report = {
|
|
invoked,
|
|
invoked_skills: [...invokedSkills].sort(),
|
|
attempted_skill_calls: attemptedSkillCalls,
|
|
trajectory_files: trajectories.length,
|
|
parse_errors: parseErrors,
|
|
};
|
|
|
|
fs.writeFileSync(
|
|
path.join(jobsDir, "required-skill.json"),
|
|
`${JSON.stringify(report, null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
|
|
if (!invoked) {
|
|
const failedAttempts = attemptedSkillCalls.filter(
|
|
(attempt) => attempt.status !== "completed",
|
|
);
|
|
const failureDetail = failedAttempts.length
|
|
? `; ${failedAttempts.length} Skill call(s) failed: ${failedAttempts.map((attempt) => attempt.message || attempt.status).join(" | ")}`
|
|
: "";
|
|
console.error(
|
|
`No successful Skill invocation was observed${failureDetail}`,
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log("Required Skill validation passed.");
|