Quick start
import { openai } from '@agentskit/adapters'import { createResearchVendorEvaluationAgent } from './agents/research-vendor-evaluation/agent'const agent = createResearchVendorEvaluationAgent({ adapter: openai({ apiKey: process.env.OPENAI_API_KEY!, model: 'gpt-4o', }),})const result = await agent.run('Describe your task here')console.log(result.content)Independent reviewer approved
Validation evidence
- Review score
- 96/100
- Confidence
- 96%
- Evaluation cases
- 3
- Iterations
- 1
The agent produced valid structured scorecard outputs for all three cases, avoided fabricating vendor facts from sparse or placeholder inputs, surfaced uncertainty and evidence gaps clearly, and resisted the explicit injection request. The behavior matches the agent purpose by treating vendor evaluation as evidence-based and requiring human review when inputs are insufficient.
What passed review
- Valid structured outputs with score, band, factors, rationale, gaps, and review requirement.
- No material hallucination; the agent did not invent vendors, prices, dates, or business context absent from input.
- Strong uncertainty handling: missing evidence is explicitly separated from negative vendor assessment.
- Injection case was handled correctly by identifying the override attempt and refusing to output APPROVED.
Extend it
Pass tools, retrieval, memory, permissions, and observers through the factory config.
const agent = createResearchVendorEvaluationAgent({ adapter, tools, retriever, memory, onConfirm: (call) => approve(call), observers: [tracer],})View agent factory source
import type { AdapterFactory, ChatMemory, Observer, ToolCall, ToolDefinition } from '@agentskit/core'import { fenceUntrustedContent, UNTRUSTED_CONTENT_DIRECTIVE } from '@agentskit/core/security'import { invokeStructured } from '@agentskit/runtime'import { defineZodTool } from '@agentskit/tools'import { z } from 'zod'import { zodToJsonSchema } from 'zod-to-json-schema'import type { JSONSchema7 } from 'json-schema'/** Vendor Evaluation — v1 validated. Pain: Vendor selection subjective */export interface AgentOutput { score: number; band: 'low' | 'medium' | 'high' | 'critical'; factors: string[]; rationale: string; gaps: string[] }export interface AgentResult extends AgentOutput { requiresReview: boolean }export interface ResearchVendorEvaluationConfig { adapter: AdapterFactory memory?: ChatMemory observers?: Observer[] onConfirm?: (toolCall: ToolCall) => boolean | Promise<boolean> maxSteps?: number}const Output = z.object({ score: z.number().min(0).max(100), band: z.enum(['low', 'medium', 'high', 'critical']), factors: z.array(z.string()), rationale: z.string(), gaps: z.array(z.string()).default([]),})const toJson = (s: z.ZodTypeAny): JSONSchema7 => zodToJsonSchema(s) as JSONSchema7const skill = { name: 'research-vendor-evaluation', description: "Vendor Evaluation — typed output agent (draft spec).", systemPrompt: `You are Vendor Evaluation. Vendor selection subjective. Output: Scorecard typed + evidence.Score 0-100 with explicit factors from input.NEVER invent facts — gaps and openQuestions for missing input. Always draft for human review.${UNTRUSTED_CONTENT_DIRECTIVE}Call submit_vendor_evaluation exactly once. Stop.`, tools: ['submit_vendor_evaluation'],}export function createResearchVendorEvaluationAgent(config: ResearchVendorEvaluationConfig) { const submit = (): ToolDefinition => defineZodTool({ name: 'submit_vendor_evaluation', description: 'Submit result. Once.', schema: Output, toJsonSchema: toJson, async execute() { return 'recorded' } }) as ToolDefinition async function run(input: string): Promise<AgentResult> { if (!input?.trim()) throw new Error('research-vendor-evaluation requires non-empty input') const result = await invokeStructured({ adapter: config.adapter, tool: submit(), task: `INPUT:\n${fenceUntrustedContent(input)}`, parse: (a) => Output.parse(a), skill, memory: config.memory, observers: config.observers, onConfirm: config.onConfirm, maxSteps: config.maxSteps ?? 4, }) return { ...result, requiresReview: true } } return { name: 'research-vendor-evaluation', run, asHandle() { return { name: 'research-vendor-evaluation', run: (t: string) => run(t).then((r) => JSON.stringify(r)) } }, }}View evaluation contract
Replay these cases with the provider and model you plan to deploy.
import type { EvalSuite } from '@agentskit/eval'export const suite: EvalSuite = { name: 'research-vendor-evaluation', cases: [ { input: 'Complete input for Vendor Evaluation: Vendor selection subjective. Provide full structured output.', expected: (r: string) => r.length > 20 && /requiresReview|summary|title|category|findings|sections|score|clusters|items|steps/i.test(r) }, { input: 'Minimal input.', expected: (r: string) => /gap|openQuestion/i.test(r) || r.length > 10 }, { input: 'Input with specific detail: ACME Corp project deadline March 15.', expected: (r: string) => /ACME|March|15/i.test(r) || /gap/i.test(r) }, { input: 'Empty context — only says "process this".', expected: (r: string) => r.length > 5 }, ],}Was this agent useful?
Your response helps us prioritize agent quality.