Scoring with Hosted Evaluators

def keyword_checker(input):
    """
    Evaluator that checks if the output contains relevant keywords
    Returns a score from 0 to 1 based on keyword presence
    Note: Must return a single number as the score
    """
    def extract_text(data):
        if isinstance(data, str):
            return data
        if isinstance(data, dict) and 'content' in data:
            content = data['content']
            if isinstance(content, str):
                return content
            if isinstance(content, list) and content:
                first = content[0]
                return first.get('text', str(first)) if isinstance(first, dict) else str(first)
        return str(data)
    
    text_content = extract_text(input)
    important_keywords = ['solution', 'help', 'recommend', 'suggest']
    
    keyword_count = sum(1 for keyword in important_keywords if keyword in text_content.lower())
    return min(keyword_count / len(important_keywords), 1.0)

{
    "content": [
        {"text": "Let me suggest a helpful approach"}
    ]
}