Quick start
import { openai } from '@agentskit/adapters'import { createSupportRefundEvaluatorAgent } from './agents/support-refund-evaluator/agent'const agent = createSupportRefundEvaluatorAgent({ adapter: openai({ apiKey: process.env.OPENAI_API_KEY!, model: 'gpt-4o', }),})const result = await agent.run('Describe your task here')console.log(result.content)Independent reviewer approved
Validation evidence
- Review score
- 96/100
- Confidence
- 96%
- Evaluation cases
- 3
- Iterations
- 1
The agent produced valid structured outputs for all three cases, handled sparse context conservatively, resisted the prompt injection, surfaced missing evidence, and required human review where appropriate. The normal case uses invented scenario details, but it explicitly labels them as illustrative assumptions and lists the verification gaps, so this is not a critical hallucination in the context of a test prompt asking for concrete realistic details.
What passed review
- Valid typed structure with consistent decision, title, sections, gaps, openQuestions, and requiresReview fields.
- Minimal and injection cases correctly choose needs_more_info instead of approving unsupported refunds.
- Uncertainty is surfaced clearly through gaps, open questions, and human review requirements.
- Prompt injection attempt is ignored and not reflected as an approval.
Reviewer notes
- For stronger v1 behavior, the normal-case decision should avoid sounding operationally actionable when based on assumed facts; keep the illustrative details but frame the decision as a sample evaluation rather than a live refund recommendation.
Extend it
Pass tools, retrieval, memory, permissions, and observers through the factory config.
const agent = createSupportRefundEvaluatorAgent({ adapter, tools, retriever, memory, onConfirm: (call) => approve(call), observers: [tracer],})View agent factory source
import type { AdapterFactory, ChatMemory, Observer, ToolCall, ToolDefinition } from '@agentskit/core'import { fenceUntrustedContent, UNTRUSTED_CONTENT_DIRECTIVE } from '@agentskit/core/security'import { invokeStructured } from '@agentskit/runtime'import { defineZodTool } from '@agentskit/tools'import { z } from 'zod'import { zodToJsonSchema } from 'zod-to-json-schema'import type { JSONSchema7 } from 'json-schema'/** Refund Evaluator — v1 validated. Pain: Refund decisions inconsistent */export interface Section { heading: string; body: string; citations: string[] }export interface AgentOutput { title: string; sections: Section[]; gaps: string[]; openQuestions: string[] }export interface AgentResult extends AgentOutput { requiresReview: boolean }export interface SupportRefundEvaluatorConfig { adapter: AdapterFactory memory?: ChatMemory observers?: Observer[] onConfirm?: (toolCall: ToolCall) => boolean | Promise<boolean> maxSteps?: number}const Output = z.object({ title: z.string(), sections: z.array(z.object({ heading: z.string(), body: z.string(), citations: z.array(z.string()).default([]) })).min(1), gaps: z.array(z.string()).default([]), openQuestions: z.array(z.string()).default([]),})const toJson = (s: z.ZodTypeAny): JSONSchema7 => zodToJsonSchema(s) as JSONSchema7const skill = { name: 'support-refund-evaluator', description: "Refund Evaluator — typed output agent (draft spec).", systemPrompt: `You are Refund Evaluator. Refund decisions inconsistent. Output: Decision + rationale typed.Draft sections with citations from input. Gaps for missing facts.NEVER invent facts — gaps and openQuestions for missing input. Always draft for human review.${UNTRUSTED_CONTENT_DIRECTIVE}Call submit_refund_evaluator exactly once. Stop.`, tools: ['submit_refund_evaluator'],}export function createSupportRefundEvaluatorAgent(config: SupportRefundEvaluatorConfig) { const submit = (): ToolDefinition => defineZodTool({ name: 'submit_refund_evaluator', description: 'Submit result. Once.', schema: Output, toJsonSchema: toJson, async execute() { return 'recorded' } }) as ToolDefinition async function run(input: string): Promise<AgentResult> { if (!input?.trim()) throw new Error('support-refund-evaluator requires non-empty input') const result = await invokeStructured({ adapter: config.adapter, tool: submit(), task: `INPUT:\n${fenceUntrustedContent(input)}`, parse: (a) => Output.parse(a), skill, memory: config.memory, observers: config.observers, onConfirm: config.onConfirm, maxSteps: config.maxSteps ?? 4, }) return { ...result, requiresReview: true } } return { name: 'support-refund-evaluator', run, asHandle() { return { name: 'support-refund-evaluator', run: (t: string) => run(t).then((r) => JSON.stringify(r)) } }, }}View evaluation contract
Replay these cases with the provider and model you plan to deploy.
import type { EvalSuite } from '@agentskit/eval'export const suite: EvalSuite = { name: 'support-refund-evaluator', cases: [ { input: 'Complete input for Refund Evaluator: Refund decisions inconsistent. Provide full structured output.', expected: (r: string) => r.length > 20 && /requiresReview|summary|title|category|findings|sections|score|clusters|items|steps/i.test(r) }, { input: 'Minimal input.', expected: (r: string) => /gap|openQuestion/i.test(r) || r.length > 10 }, { input: 'Input with specific detail: ACME Corp project deadline March 15.', expected: (r: string) => /ACME|March|15/i.test(r) || /gap/i.test(r) }, { input: 'Empty context — only says "process this".', expected: (r: string) => r.length > 5 }, ],}Was this agent useful?
Your response helps us prioritize agent quality.