Multimodal Voice and Text Agents

Voice-Only vs Multimodal Contexts

Prompting for Voice-Only Responses

VOICE_ONLY_SYSTEM_PROMPT = (
    'You are a voice-only assistant. The user cannot see any screen.\n\n'
    'Requirements:\n'
    '- Never reference visual elements ("tap here", "see the chart", "the blue button")\n'
    '- Never use numbered or bulleted lists — use spoken sequences instead:\n'
    '  BAD: "1. First do X 2. Then do Y"\n'
    '  GOOD: "Start by doing X. When that is done, do Y."\n'
    '- Limit responses to what can be comfortably spoken in 30 seconds\n'
    '- Offer to give more detail rather than overwhelming the user\n'
    '- Use verbal signposts: "First", "Next", "Finally"\n'
    '- Read out all important data: codes, dates, amounts as full words'
)
print(VOICE_ONLY_SYSTEM_PROMPT)

Coordinating Spoken and On-Screen Text

MULTIMODAL_SYSTEM_PROMPT = (
    'You are a multimodal assistant with both a voice and a screen.\n\n'
    'When responding, consider what each modality does best:\n\n'
    'SPEAK (voice):\n'
    '- Conversational summary, emotional tone, key highlights\n'
    '- Guide the user to look at the screen when needed:\n'
    '  "I have shown the details on screen. The key number to notice is..."\n\n'
    'SHOW (screen):\n'
    '- Detailed data, tables, long lists, code, maps, images\n\n'
    'When your response includes structured data, respond in this format:\n'
    'SPOKEN: <what to say aloud>\n'
    'VISUAL: <what to display on screen in markdown>'
)

# Example LLM output for multimodal response:
EXAMPLE_MULTIMODAL_OUTPUT = (
    'SPOKEN: Your top three expenses this month are food, transport, and entertainment. '
    'Food was the biggest, almost double your budget. Check the screen for the full breakdown.\n\n'
    'VISUAL: | Category | Budget | Actual | Difference |\n'
    '|---|---|---|---|\n'
    '| Food | $400 | $780 | -$380 |\n'
    '| Transport | $150 | $162 | -$12 |\n'
    '| Entertainment | $100 | $145 | -$45 |'
)
print(EXAMPLE_MULTIMODAL_OUTPUT)

Structuring LLM Output for Voice Agents

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

VOICE_AGENT_SYSTEM = (
    'You are a financial voice assistant. For each response, return JSON with:\n'
    '{\n'
    '  "spoken": "Short spoken response (max 2 sentences)",\n'
    '  "visual_title": "Header for the on-screen card (optional)",\n'
    '  "visual_content": "Detailed content for screen (markdown, optional)",\n'
    '  "action_label": "Button label if action needed (optional)",\n'
    '  "action_type": "one of: none, confirm, navigate, call"\n'
    '}\n'
    'Return only the JSON object.'
)

def voice_agent_query(user_message):
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=500,
        system=VOICE_AGENT_SYSTEM,
        messages=[{'role': 'user', 'content': user_message}]
    )
    try:
        response_data = json.loads(r.content[0].text)
        return response_data
    except json.JSONDecodeError:
        return {'spoken': r.content[0].text, 'visual_content': None}

result = voice_agent_query('What is my account balance?')
print('SPEAK:', result.get('spoken'))
print('SHOW:', result.get('visual_content', 'Nothing to display'))

Transcript Formatting for Voice Agents

import datetime
import json

class VoiceTranscript:
    def __init__(self, session_id):
        self.session_id = session_id
        self.turns = []

    def add_user_turn(self, text, audio_duration_ms=None):
        self.turns.append({
            'speaker': 'user',
            'timestamp': datetime.datetime.utcnow().isoformat(),
            'text': text,
            'audio_duration_ms': audio_duration_ms,
        })

    def add_agent_turn(self, spoken_text, visual_content=None, action=None):
        self.turns.append({
            'speaker': 'agent',
            'timestamp': datetime.datetime.utcnow().isoformat(),
            'spoken': spoken_text,
            'visual': visual_content,
            'action': action,
        })

    def save(self, filepath):
        with open(filepath, 'w') as f:
            json.dump({'session_id': self.session_id, 'turns': self.turns}, f, indent=2)
        print(f'Transcript saved: {filepath}')

# Usage
transcript = VoiceTranscript('session_001')
transcript.add_user_turn('What is my balance?', audio_duration_ms=1200)
transcript.add_agent_turn('Your balance is four hundred dollars.', visual_content='Balance: $400')
transcript.save('/tmp/session_001_transcript.json')

Handling Voice Input Errors

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

AMBIGUITY_HANDLING_SYSTEM = (
    'You are a voice assistant. User input comes from speech recognition '
    'and may contain transcription errors.\n\n'
    'When input seems unclear or ambiguous:\n'
    '1. State what you think the user might have meant.\n'
    '2. Ask a single clarifying yes/no question to confirm.\n'
    '3. Never ask more than one question at a time.\n'
    '4. Offer the most likely interpretation as the default.\n\n'
    'Example:\n'
    'Input: "transfer five hundred to john or gene" (ambiguous name)\n'
    'Response: "It sounds like you want to transfer five hundred dollars. '
    'Did you mean John Smith or Gene Lee?"'
)

def handle_voice_input(user_speech):
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        system=AMBIGUITY_HANDLING_SYSTEM,
        messages=[{'role': 'user', 'content': user_speech}]
    )
    return r.content[0].text

print(handle_voice_input('pay the electric company bill thing'))

Turn-Taking in Voice Conversations

TURN_TAKING_SYSTEM = (
    'You are a voice assistant. Responses must be designed for spoken conversation:\n\n'
    'End each response with exactly ONE of:\n'
    '- A direct question inviting the user to respond\n'
    '- A clear statement that the task is complete (e.g., "That is done.")\n'
    '- An explicit offer to continue (e.g., "Is there anything else?")\n\n'
    'Never end mid-thought. Never trail off. '
    'Avoid open-ended statements that leave the user unsure if they should speak.\n\n'
    'GOOD endings:\n'
    '- "The transfer is complete. Would you like a confirmation number?"\n'
    '- "That is all I have. Is there anything else?"\n'
    'BAD endings:\n'
    '- "You might also want to consider..." (open, unclear)\n'
    '- "The balance is..." (incomplete)'
)
print(TURN_TAKING_SYSTEM[:300])

Interruption Handling

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

INTERRUPTION_SYSTEM = (
    'You are a voice assistant. Users may interrupt mid-conversation.\n\n'
    'If the user changes topic abruptly, smoothly acknowledge the change:\n'
    '"Of course. Let us switch to that." Then answer the new question.\n\n'
    'If the user says something like "wait", "stop", "hold on":\n'
    'Pause and say "Sure, take your time" and wait for them to continue.\n\n'
    'If the user repeats a question, they likely did not hear the answer:\n'
    'Say "Let me repeat that." and say it again more slowly.\n\n'
    'Never express frustration at interruptions or repetition.'
)

def handle_conversation(turns):
    """Handle multi-turn voice conversation with interruptions."""
    messages = []
    for speaker, text in turns:
        messages.append({'role': speaker, 'content': text})

    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        system=INTERRUPTION_SYSTEM,
        messages=messages
    )
    return r.content[0].text

# Simulate an interruption scenario
conversation = [
    ('user', 'What is my balance?'),
    ('assistant', 'Your checking account balance is four hundred dollars and—'),
    ('user', 'Actually wait, can you tell me my savings instead?'),
]
print(handle_conversation(conversation))

On-Screen Companion to Voice

def render_multimodal_response(agent_output):
    """
    Render a voice agent response to both TTS and screen components.
    agent_output: dict with 'spoken', 'visual_content', 'action_label'
    """
    # Route to TTS
    spoken = agent_output.get('spoken', '')
    if spoken:
        send_to_tts(spoken)  # Your TTS function
        print(f'[AUDIO] {spoken}')

    # Route to screen
    visual = agent_output.get('visual_content')
    if visual:
        render_card_on_screen(visual)  # Your UI function
        print(f'[SCREEN] {visual[:100]}')

    # Optional action button
    action_label = agent_output.get('action_label')
    if action_label:
        show_action_button(action_label)  # Your UI function
        print(f'[BUTTON] {action_label}')

def send_to_tts(text):
    print(f'TTS: {text}')

def render_card_on_screen(content):
    print(f'Screen card: {content[:50]}')

def show_action_button(label):
    print(f'Button: {label}')

# Test it
render_multimodal_response({
    'spoken': 'I found three flights to New York.',
    'visual_content': '| Flight | Departs | Price |\n|---|---|---|\n| AA101 | 08:00 | $299 |',
    'action_label': 'Book cheapest'
})

Accessibility Considerations

ACCESSIBILITY_VOICE_SYSTEM = (
    'This voice assistant serves users who may be using voice as their '
    'primary access method due to disability or preference.\n\n'
    'Guidelines:\n'
    '- Never require the user to see a screen to complete a task.\n'
    '- Read out all information that matters, including confirmation codes, '
    'totals, and status messages.\n'
    '- Offer to repeat any information: '
    '"I can repeat that if you would like."\n'
    '- Describe any actions you took: '
    '"I have sent the confirmation to your email."\n'
    '- Accept multiple phrasings for the same command — users phrase '
    'voice commands inconsistently.\n'
    '- Confirm all destructive or financial actions before executing:\n'
    '  "Just to confirm: you want to transfer $500 to John. Is that right?"'
)
print(ACCESSIBILITY_VOICE_SYSTEM[:300])

Testing Voice Agent Responses

Knowledge Check: Voice-Only Constraint

Recap: Multimodal Voice and Text Agents