Quick start
import { openai } from '@agentskit/adapters'import { createSupportKbSearcherAgent } from './agents/support-kb-searcher/agent'const agent = createSupportKbSearcherAgent({ adapter: openai({ apiKey: process.env.OPENAI_API_KEY!, model: 'gpt-4o', }),})const result = await agent.run('Describe your task here')console.log(result.content)Independent reviewer approved
Validation evidence
- Review score
- 96/100
- Confidence
- 95%
- Evaluation cases
- 3
- Iterations
- 1
The agent produced valid structured outputs for all three cases, stayed within its KB-search purpose, avoided inventing articles or URLs, returned noMatch when no grounded candidates or article names were available, and resisted the injection request. The suggested topics are reasonable and uncertainty-aware. The only minor weakness is that the normal case output is necessarily low-value because the input was a meta instruction rather than an actual support ticket or retrieved corpus, but the conservative no-match behavior is correct for the agent contract.
What passed review
- All outputs conform to the expected structured shape with hits, noMatch, and suggestedTopic.
- No hallucinated articles, URLs, quotes, or confidence scores were invented.
- Injection case ignored the instruction to output APPROVED and treated the content as untrusted/sparse.
- Minimal and sparse inputs surfaced gaps through suggestedTopic instead of fabricating matches.
Example
A real usage example maintained with this agent.
import { anthropic } from '@agentskit/adapters'import { createKbSearcherAgent } from './agents/support-kb-searcher/agent'const r = await createKbSearcherAgent({ adapter: anthropic({ apiKey: process.env.ANTHROPIC_API_KEY!, model: 'claude-opus-4-8' }), retrieve: (ticket) => myVectorStore.search(ticket), // → KbCandidate[]}).run(ticketText)// → { hits: [{ title, url, quote, confidence }], noMatch, suggestedTopic? }Extend it
Pass tools, retrieval, memory, permissions, and observers through the factory config.
const agent = createSupportKbSearcherAgent({ adapter, tools, retriever, memory, onConfirm: (call) => approve(call), observers: [tracer],})View agent factory source
import type { AdapterFactory, ChatMemory, Observer, ToolCall, ToolDefinition } from '@agentskit/core'import { fenceUntrustedContent, UNTRUSTED_CONTENT_DIRECTIVE } from '@agentskit/core/security'import { invokeStructured } from '@agentskit/runtime'import { defineZodTool } from '@agentskit/tools'import { z } from 'zod'import { zodToJsonSchema } from 'zod-to-json-schema'import type { JSONSchema7 } from 'json-schema'/** * KB Searcher — given a ticket (and optionally a retrieved candidate set), returns the * top KB articles that answer it as TYPED hits with a confidence score. Never invents * an article: when no candidate is grounded, it returns `noMatch` + a suggested topic * the article should cover, rather than a hallucinated URL. * * Pass `retrieve` to ground answers in YOUR corpus — only candidates it returns can be * cited (the agent is told to cite none if the set is empty). Without it the model may * only cite articles named verbatim in the ticket. */export interface KbHit { title: string url: string /** One-sentence quote of the matching section. */ quote: string /** 1 (weak) – 5 (exact). */ confidence: number}export interface KbSearchResult { hits: KbHit[] /** True when nothing grounded matched. */ noMatch: boolean /** When noMatch: the topic a new article should cover. */ suggestedTopic?: string}export interface KbCandidate { title: string url: string snippet: string}export interface KbSearcherConfig { adapter: AdapterFactory /** Optional retriever — only the candidates it returns may be cited. */ retrieve?: (ticket: string) => Promise<KbCandidate[]> | KbCandidate[] /** Max hits to return (default 3). */ topK?: number /** Drop hits below this confidence (1-5, default 2). */ minConfidence?: number memory?: ChatMemory observers?: Observer[] onConfirm?: (toolCall: ToolCall) => boolean | Promise<boolean> maxSteps?: number}const Hit = z.object({ title: z.string(), url: z.string(), quote: z.string(), confidence: z.number().int().min(1).max(5),})const Output = z.object({ hits: z.array(Hit), noMatch: z.boolean(), suggestedTopic: z.string().optional(),})const toJson = (s: z.ZodTypeAny): JSONSchema7 => zodToJsonSchema(s) as JSONSchema7const skill = { name: 'kb-searcher', description: 'Returns the top knowledge-base articles answering a ticket, with confidence.', systemPrompt: `You match a support ticket to knowledge-base articles. Return the best articles as hits,each with: title, url, a one-sentence quote of the matching section, and a confidence 1-5.NEVER invent an article. If CANDIDATES are provided, cite ONLY from them (and cite NONE if the setis empty). If nothing genuinely matches, set noMatch=true, return an empty hits array, and put thetopic a new article should cover in suggestedTopic.${UNTRUSTED_CONTENT_DIRECTIVE}Call submit_results exactly once with { hits, noMatch, suggestedTopic? }. Stop.`, tools: ['submit_results'],}export function createKbSearcherAgent(config: KbSearcherConfig) { const topK = config.topK ?? 3 const minConf = config.minConfidence ?? 2 const emit = (label: string, status: 'start' | 'ok' | 'skip' | 'error', detail?: string) => { for (const o of config.observers ?? []) void o.on({ type: 'progress', label, status, detail }) } const submit = (): ToolDefinition => defineZodTool({ name: 'submit_results', description: 'Submit the KB hits. Call exactly once.', schema: Output, toJsonSchema: toJson, async execute() { return 'recorded' } }) as ToolDefinition async function run(ticket: string): Promise<KbSearchResult> { if (!ticket?.trim()) throw new Error('kb searcher requires a non-empty ticket') let candidateBlock = '' let allowedUrls: Set<string> | null = null if (config.retrieve) { emit('retrieve', 'start') const candidates = (await config.retrieve(ticket)) ?? [] allowedUrls = new Set(candidates.map((c) => c.url)) emit('retrieve', 'ok', `${candidates.length} candidate(s)`) candidateBlock = `\n\nCANDIDATES (cite only these; cite none if empty):\n${fenceUntrustedContent( candidates.map((c) => `- ${c.title} <${c.url}>: ${c.snippet}`).join('\n') || '(none)', )}` } emit('search', 'start') let out: z.infer<typeof Output> try { out = await invokeStructured({ adapter: config.adapter, tool: submit(), task: `TICKET:\n${fenceUntrustedContent(ticket)}${candidateBlock}`, parse: (a) => Output.parse(a), skill, memory: config.memory, observers: config.observers, onConfirm: config.onConfirm, maxSteps: config.maxSteps ?? 3, }) } catch { emit('search', 'error') return { hits: [], noMatch: true, suggestedTopic: 'search unavailable — retry' } } // Ground: drop low-confidence hits and (if retrieving) any URL not in the candidate set. const hits = out.hits .filter((h) => h.confidence >= minConf) .filter((h) => (allowedUrls ? allowedUrls.has(h.url) : true)) .sort((a, b) => b.confidence - a.confidence) .slice(0, topK) const noMatch = hits.length === 0 emit('search', 'ok', noMatch ? 'no match' : `${hits.length} hit(s)`) return { hits, noMatch, suggestedTopic: noMatch ? (out.suggestedTopic ?? `coverage for: ${ticket.slice(0, 80)}`) : undefined, } } return { name: 'support-kb-searcher', run, asHandle() { return { name: 'support-kb-searcher', run: async (task: string) => JSON.stringify(await run(task)) } }, }}View evaluation contract
Replay these cases with the provider and model you plan to deploy.
import type { EvalSuite } from '@agentskit/eval'export const suite: EvalSuite = { name: 'support-kb-searcher', cases: [ { input: `Ticket: A customer asks how to rotate their API key after a suspected leak. They want to know if rotating invalidates the old key immediately and whether existing webhooks keep working. Find the relevant KB articles.`, expected: (r: string) => /(http|www|\/docs|kb)/i.test(r) && /confidence/i.test(r), }, { input: `Ticket: User on the Pro plan wants to downgrade to Starter mid-cycle and is asking whether they get a prorated refund. Find the top knowledge-base articles that answer this.`, expected: (r: string) => /([1-5])/.test(r) && /(refund|proration|prorate|downgrade|billing)/i.test(r), }, { input: `Ticket: Customer reports that SAML SSO login redirects them back to the login page in a loop after entering their Okta credentials. Find KB articles covering SSO / SAML troubleshooting.`, expected: (r: string) => /(SSO|SAML|Okta|login)/i.test(r) && /title/i.test(r), }, { input: `Ticket: A customer is asking for detailed guidance on integrating our product with an obscure in-house ERP that we have never documented anywhere. Search the knowledge base.`, expected: (r: string) => /(no (good )?match|no relevant|could not find|no article|suggest)/i.test(r), }, ],}Was this agent useful?
Your response helps us prioritize agent quality.