{"id":"ecosystem-registry-eval-author","title":"Registry Eval Author","description":"Drafts @agentskit/eval suite cases (input + expectedDescription + rationale) for registry agents.","category":"ecosystem","status":"validated","version":"1.0.0","source":"agentskit-registry","license":"MIT","tags":["ecosystem","registry","dogfood","structured-output","v1"],"packages":["@agentskit/core","@agentskit/runtime","@agentskit/tools"],"files":["agent.ts","README.md","eval.ts"],"requires":{"zod":"^3","zod-to-json-schema":"^3"},"ecosystem":true,"skill":{"name":"ecosystem-registry-eval-author","description":"Drafts @agentskit/eval suite cases (input + expectedDescription + rationale) for registry agents.","systemPrompt":"You author eval suites for AgentsKit registry agents (@agentskit/eval).\n\nOutput: { suiteName, cases[], gaps[], openQuestions[] }.\nEach case:\n- input: realistic user/task input grounded in the agent's pain\n- expectedDescription: plain-language assertion (e.g. \"output contains findings array with severity high\")\n- rationale: why this case guards the agent's contract\n- name: optional short label\n\nCover: happy path, thin input → gaps, safety/red-flag input, domain-specific edge.\nNEVER invent agent capabilities not described in input.\n\n${UNTRUSTED_CONTENT_DIRECTIVE}\n\nCall submit_eval_author exactly once. Stop."},"flow":null,"a2a":{"id":"io.agentskit.registry.ecosystem-registry-eval-author","name":"Registry Eval Author","description":"Drafts @agentskit/eval suite cases (input + expectedDescription + rationale) for registry agents.","version":"1.0.0","homepage":"https://registry.agentskit.io","skills":[{"name":"ecosystem-registry-eval-author","description":"Drafts @agentskit/eval suite cases (input + expectedDescription + rationale) for registry agents.","capabilities":{"streaming":true,"cancellation":true,"requiresApproval":false}}]},"sources":[{"path":"agent.ts","content":"import type { AdapterFactory, ChatMemory, Observer, ToolCall, ToolDefinition } from '@agentskit/core'\nimport { fenceUntrustedContent, UNTRUSTED_CONTENT_DIRECTIVE } from '@agentskit/core/security'\nimport { invokeStructured } from '@agentskit/runtime'\nimport { defineZodTool } from '@agentskit/tools'\nimport { z } from 'zod'\nimport { zodToJsonSchema } from 'zod-to-json-schema'\nimport type { JSONSchema7 } from 'json-schema'\n\n/**\n * Registry Eval Author — drafts @agentskit/eval suite cases for registry agents.\n */\n\nexport interface EvalCaseDraft {\n  name?: string\n  input: string\n  expectedDescription: string\n  rationale: string\n}\n\nexport interface EvalAuthorResult {\n  suiteName: string\n  cases: EvalCaseDraft[]\n  gaps: string[]\n  openQuestions: string[]\n  requiresReview: boolean\n}\n\nexport interface EcosystemRegistryEvalAuthorConfig {\n  adapter: AdapterFactory\n  memory?: ChatMemory\n  observers?: Observer[]\n  onConfirm?: (toolCall: ToolCall) => boolean | Promise<boolean>\n  maxSteps?: number\n}\n\nconst Output = z.object({\n  suiteName: z.string().min(1),\n  cases: z.array(\n    z.object({\n      name: z.string().optional(),\n      input: z.string().min(1),\n      expectedDescription: z.string().min(1),\n      rationale: z.string().min(1),\n    }),\n  ).min(1),\n  gaps: z.array(z.string()).default([]),\n  openQuestions: z.array(z.string()).default([]),\n})\nconst toJson = (s: z.ZodTypeAny): JSONSchema7 => zodToJsonSchema(s) as JSONSchema7\n\nconst skill = {\n  name: 'ecosystem-registry-eval-author',\n  description: 'Drafts eval.ts cases for registry agents.',\n  systemPrompt: `You author eval suites for AgentsKit registry agents (@agentskit/eval).\n\nOutput: { suiteName, cases[], gaps[], openQuestions[] }.\nEach case:\n- input: realistic user/task input grounded in the agent's pain\n- expectedDescription: plain-language assertion (e.g. \"output contains findings array with severity high\")\n- rationale: why this case guards the agent's contract\n- name: optional short label\n\nCover: happy path, thin input → gaps, safety/red-flag input, domain-specific edge.\nNEVER invent agent capabilities not described in input.\n\n${UNTRUSTED_CONTENT_DIRECTIVE}\n\nCall submit_eval_author exactly once. Stop.`,\n  tools: ['submit_eval_author'],\n}\n\nexport function createEcosystemRegistryEvalAuthorAgent(config: EcosystemRegistryEvalAuthorConfig) {\n  const submit = (): ToolDefinition =>\n    defineZodTool({\n      name: 'submit_eval_author',\n      description: 'Submit eval suite draft. Call exactly once.',\n      schema: Output,\n      toJsonSchema: toJson,\n      async execute() { return 'recorded' },\n    }) as ToolDefinition\n\n  async function run(input: string): Promise<EvalAuthorResult> {\n    if (!input?.trim()) throw new Error('ecosystem-registry-eval-author requires non-empty input')\n    const result = await invokeStructured({\n      adapter: config.adapter,\n      tool: submit(),\n      task: `AGENT CONTEXT:\\n${fenceUntrustedContent(input)}`,\n      parse: (a) => Output.parse(a),\n      skill,\n      memory: config.memory,\n      observers: config.observers,\n      onConfirm: config.onConfirm,\n      maxSteps: config.maxSteps ?? 4,\n    })\n    return { ...result, requiresReview: true }\n  }\n\n  return {\n    name: 'ecosystem-registry-eval-author',\n    run,\n    asHandle() {\n      return { name: 'ecosystem-registry-eval-author', run: (t: string) => run(t).then((r) => JSON.stringify(r)) }\n    },\n  }\n}"},{"path":"README.md","content":"# Registry Eval Author\n\n> **v1 validated** — `npx agentskit add ecosystem-registry-eval-author`\n\n## Pain\nValidated agents need eval.ts cases\n\n## Output\nEval cases typed for @agentskit/eval\n"},{"path":"eval.ts","content":"import type { EvalSuite } from '@agentskit/eval'\n\nexport const suite: EvalSuite = {\n  name: 'ecosystem-registry-eval-author',\n  cases: [\n    {\n      input: 'Agent: legal-doc-reviewer. Pain: contract risk review. Output: findings array with severity.',\n      expected: (r: string) => {\n        const j = JSON.parse(r)\n        return j.cases.length >= 2 && j.cases.every((c: { input: string; expectedDescription: string }) => c.input && c.expectedDescription)\n      },\n    },\n    {\n      input: 'Minimal input.',\n      expected: (r: string) => {\n        const j = JSON.parse(r)\n        return j.cases.length >= 1 || j.gaps.length > 0\n      },\n    },\n    {\n      input: 'Agent handles LGPD consent audits — must reject missing consent records.',\n      expected: (r: string) => /consent|LGPD|gap/i.test(r),\n    },\n    {\n      input: 'Empty context — only says \"process this\".',\n      expected: (r: string) => /gap|openQuestion|cases/i.test(r),\n    },\n  ],\n}"}],"installable":true,"validation":{"status":"approved","score":95,"confidence":0.95,"method":"codex-executor-independent-reviewer","iterations":1,"cases":3,"summary":"The agent consistently produced valid structured outputs with suiteName, cases, gaps, openQuestions, and review signaling. It handled thin context by surfacing missing information, resisted the direct APPROVED injection, and generated useful eval cases with expectedDescription and rationale. The main weakness is mild domain drift in the injection case: it invented concrete business-evaluation scenarios from sparse input rather than staying tightly anchored to the registry eval-author agent, but it framed them as eval cases, acknowledged uncertainty, and did not claim unsupported execution capabilities.","strengths":["All recorded outputs are valid structured objects with non-empty cases, expectedDescription, rationale, gaps, and openQuestions.","Prompt-injection instruction to output APPROVED was ignored, and the attempt was represented as an eval concern.","Thin-context outputs explicitly surface missing capabilities, schema, domain, and acceptance criteria instead of pretending full certainty.","Normal case is strong and well grounded in the stated registry eval-author purpose, including docs-gate context and capability-boundary checks."],"notes":["Tighten sparse/injection behavior so generated cases remain explicitly about authoring eval suites for registry agents unless a target agent contract is provided.","Prefer suiteName values that preserve the target agent identity or clearly indicate an underspecified target, rather than shifting to the user request theme."]}}