Output Dev Evaluator Function

src/workflows/{workflow-name}/
├── workflow.ts
├── steps.ts
├── evaluators.ts    # All evaluators in one file
├── types.ts
└── ...

src/workflows/{workflow-name}/
├── workflow.ts
├── steps.ts
├── evaluators/      # Evaluators split into individual files
│   ├── quality.ts
│   ├── accuracy.ts
│   └── completeness.ts
├── types.ts
└── ...

// WRONG - evaluators cannot import other evaluators
import { otherEvaluator } from '../../shared/evaluators/other.js'; // ✗
import { anotherEvaluator } from './other_evaluators.js'; // ✗

// CORRECT - Import from @outputai/core
import {
  evaluator,
  z,
  EvaluationBooleanResult,
  EvaluationNumberResult,
  EvaluationStringResult,
  EvaluationFeedback
} from '@outputai/core';

// WRONG - Never import z from zod
import { z } from 'zod';

// CORRECT - Use @outputai/llm wrapper
import { generateText, aiSdk } from '@outputai/llm';

// WRONG - Never call LLM providers directly
import OpenAI from 'openai';

// CORRECT
import { BlogContent } from './types.js';

// WRONG - Missing .js extension
import { BlogContent } from './types';

import { evaluator, z, EvaluationBooleanResult } from '@outputai/core';

export const myEvaluator = evaluator( {
  name: 'my_evaluator',
  description: 'Description of what this evaluator assesses',
  inputSchema: z.object( { /* input schema */ } ),
  fn: async input => {
    // Evaluation logic
    return new EvaluationBooleanResult( {
      value: true,
      confidence: 0.95
    } );
  }
} );

name: 'evaluate_content_quality'

description: 'Evaluate the quality and completeness of generated content'

inputSchema: z.object( {
  content: z.string(),
  expectedLength: z.number()
} )

fn: async input => {
  const isValid = input.content.length >= input.expectedLength;
  return new EvaluationBooleanResult( {
    value: isValid,
    confidence: 0.95
  } );
}

import { EvaluationBooleanResult } from '@outputai/core';

return new EvaluationBooleanResult( {
  value: true,           // boolean result
  confidence: 0.95,      // 0.0 to 1.0
  reasoning: 'Optional explanation of the evaluation'
} );

import { EvaluationNumberResult } from '@outputai/core';

return new EvaluationNumberResult( {
  value: 85,             // numeric result (e.g., 0-100 score)
  confidence: 0.85,      // 0.0 to 1.0
  reasoning: 'Optional explanation of the score'
} );

import { EvaluationStringResult } from '@outputai/core';

return new EvaluationStringResult( {
  value: 'positive',     // string result (e.g., category, sentiment, label)
  confidence: 0.9,       // 0.0 to 1.0
  reasoning: 'Optional explanation of the classification'
} );

import { evaluator, z, EvaluationBooleanResult } from '@outputai/core';

export const evaluateCompleteness = evaluator( {
  name: 'evaluate_completeness',
  description: 'Check if content meets minimum length requirements',
  inputSchema: z.object( {
    content: z.string(),
    minLength: z.number().default( 100 )
  } ),
  fn: async ( { content, minLength } ) => {
    const isComplete = content.length >= minLength;

    return new EvaluationBooleanResult( {
      value: isComplete,
      confidence: 1.0,
      reasoning: isComplete ?
        `Content has ${content.length} characters, meets minimum of ${minLength}` :
        `Content has ${content.length} characters, below minimum of ${minLength}`
    } );
  }
} );

import { evaluator, z, EvaluationBooleanResult } from '@outputai/core';

export const evaluateGibberish = evaluator( {
  name: 'evaluate_gibberish',
  description: 'Check if a given string is gibberish',
  inputSchema: z.string(),
  fn: async content => {
    const gibberishPatterns = [ 'foo', 'bar', 'lorem', 'ipsum' ];
    const isGibberish = gibberishPatterns.some( p => content.toLowerCase().includes( p ) );

    return new EvaluationBooleanResult( {
      value: !isGibberish,
      confidence: 0.95
    } );
  }
} );

import { evaluator, z, EvaluationNumberResult } from '@outputai/core';

export const evaluateReadability = evaluator( {
  name: 'evaluate_readability',
  description: 'Calculate readability score based on sentence structure',
  inputSchema: z.object( {
    content: z.string()
  } ),
  fn: async ( { content } ) => {
    const sentences = content.split( /[.!?]+/ ).filter( s => s.trim() );
    const words = content.split( /\s+/ ).filter( w => w.trim() );
    const avgWordsPerSentence = words.length / Math.max( sentences.length, 1 );

    // Simple readability score (lower avg words = more readable)
    const score = Math.max( 0, Math.min( 100, 100 - ( avgWordsPerSentence - 15 ) * 5 ) );

    return new EvaluationNumberResult( {
      value: Math.round( score ),
      confidence: 0.8,
      reasoning: `Average ${avgWordsPerSentence.toFixed( 1 )} words per sentence`
    } );
  }
} );

import { evaluator, z, EvaluationStringResult } from '@outputai/core';

export const evaluateSentiment = evaluator( {
  name: 'evaluate_sentiment',
  description: 'Classify the sentiment of content',
  inputSchema: z.object( {
    content: z.string()
  } ),
  fn: async ( { content } ) => {
    const positiveWords = [ 'great', 'excellent', 'amazing', 'good', 'love' ];
    const negativeWords = [ 'bad', 'terrible', 'awful', 'hate', 'poor' ];

    const lowerContent = content.toLowerCase();
    const positiveCount = positiveWords.filter( w => lowerContent.includes( w ) ).length;
    const negativeCount = negativeWords.filter( w => lowerContent.includes( w ) ).length;

    const { sentiment, confidence } = positiveCount > negativeCount ?
      { sentiment: 'positive', confidence: Math.min( 0.95, 0.6 + positiveCount * 0.1 ) } :
      negativeCount > positiveCount ?
        { sentiment: 'negative', confidence: Math.min( 0.95, 0.6 + negativeCount * 0.1 ) } :
        { sentiment: 'neutral', confidence: 0.7 };

    return new EvaluationStringResult( {
      value: sentiment,
      confidence,
      reasoning: `Found ${positiveCount} positive and ${negativeCount} negative indicators`
    } );
  }
} );

import { evaluator, z, EvaluationNumberResult } from '@outputai/core';
import { generateText, aiSdk } from '@outputai/llm';

export const evaluateSignalToNoise = evaluator( {
  name: 'evaluate_signal_to_noise',
  description: 'Evaluate the signal-to-noise ratio of content',
  inputSchema: z.object( {
    title: z.string(),
    content: z.string()
  } ),
  fn: async ( { title, content } ) => {
    const { output } = await generateText( {
      prompt: 'signal_noise@v1',  // References prompts/signal_noise@v1.prompt
      variables: {
        title,
        content
      },
      output: aiSdk.Output.object( {
        schema: z.object( {
          score: z.number().describe( 'Signal-to-noise score 0-100' )
        } )
      } )
    } );

    return new EvaluationNumberResult( {
      value: output.score,
      confidence: 0.85
    } );
  }
} );

import { evaluator, z, EvaluationBooleanResult } from '@outputai/core';
import { generateText, aiSdk } from '@outputai/llm';

export const evaluateFactualAccuracy = evaluator( {
  name: 'evaluate_factual_accuracy',
  description: 'Check if content contains factual claims that can be verified',
  inputSchema: z.object( {
    content: z.string(),
    topic: z.string()
  } ),
  fn: async ( { content, topic } ) => {
    const { output } = await generateText( {
      prompt: 'factual_check@v1',
      variables: { content, topic },
      output: aiSdk.Output.object( {
        schema: z.object( {
          isFactual: z.boolean().describe( 'Whether content appears factually accurate' ),
          confidence: z.number().describe( 'Confidence in assessment 0-1' ),
          issues: z.array( z.string() ).optional().describe( 'Any factual issues found' )
        } )
      } )
    } );

    return new EvaluationBooleanResult( {
      value: output.isFactual,
      confidence: output.confidence,
      reasoning: output.issues?.length ?
        `Issues found: ${output.issues.join( ', ' )}` :
        'No factual issues detected'
    } );
  }
} );

import { evaluator, z, EvaluationStringResult } from '@outputai/core';
import { generateText, aiSdk } from '@outputai/llm';

export const evaluateContentCategory = evaluator( {
  name: 'evaluate_content_category',
  description: 'Classify content into a category',
  inputSchema: z.object( {
    content: z.string(),
    categories: z.array( z.string() )
  } ),
  fn: async ( { content, categories } ) => {
    const { output } = await generateText( {
      prompt: 'categorize_content@v1',
      variables: {
        content,
        categories: categories.join( ', ' )
      },
      output: aiSdk.Output.object( {
        schema: z.object( {
          category: z.string().describe( 'The best matching category' ),
          confidence: z.number().describe( 'Confidence in classification 0-1' ),
          explanation: z.string().describe( 'Why this category was chosen' )
        } )
      } )
    } );

    return new EvaluationStringResult( {
      value: output.category,
      confidence: output.confidence,
      reasoning: output.explanation
    } );
  }
} );