Bild- und Textagenten mit Claude Vision und GPT-4V

Multimodale Agenten: Bilder + Text

Bilder mit der OpenAI API senden

from openai import OpenAI

client = OpenAI(api_key='YOUR_OPENAI_API_KEY')

# Using a public URL
response = client.chat.completions.create(
    model='gpt-4o',
    messages=[
        {
            'role': 'user',
            'content': [
                {
                    'type': 'image_url',
                    'image_url': {
                        'url': 'https://example.com/chart.png'
                    }
                },
                {
                    'type': 'text',
                    'text': 'What trend does this chart show?'
                }
            ]
        }
    ],
    max_tokens=512
)
print(response.choices[0].message.content)

Base64-Bildkodierung

import base64
from pathlib import Path

Path('screenshot.png').write_bytes(b'\x89PNG\r\n\x1a\n' + bytes(range(40)))

def encode_image_to_base64(image_path: str) -> str:
    with open(image_path, 'rb') as f:
        return base64.b64encode(f.read()).decode('utf-8')

def build_image_message(image_path: str, question: str, mime: str = 'jpeg') -> dict:
    b64 = encode_image_to_base64(image_path)
    data_url = f'data:image/{mime};base64,{b64}'
    return {
        'role': 'user',
        'content': [
            {'type': 'image_url', 'image_url': {'url': data_url}},
            {'type': 'text', 'text': question}
        ]
    }

message = build_image_message('screenshot.png', 'What error is shown?', mime='png')
print('Message built, image size:', len(message['content'][0]['image_url']['url']))

Claude Vision API

import anthropic
import base64

client = anthropic.Anthropic(api_key='YOUR_ANTHROPIC_API_KEY')

with open('diagram.png', 'rb') as f:
    image_data = base64.standard_b64encode(f.read()).decode('utf-8')

response = client.messages.create(
    model='claude-opus-4-5',
    max_tokens=512,
    messages=[
        {
            'role': 'user',
            'content': [
                {
                    'type': 'image',
                    'source': {
                        'type': 'base64',
                        'media_type': 'image/png',
                        'data': image_data
                    }
                },
                {
                    'type': 'text',
                    'text': 'Describe the architecture shown in this diagram.'
                }
            ]
        }
    ]
)
print(response.content[0].text)

MIME-Typ anhand der Dateierweiterung erkennen

import os
import base64

with open('photo.jpg', 'wb') as f:
    f.write(b'\xff\xd8\xff' + bytes(range(30)))

MIME_MAP = {
    '.jpg': 'image/jpeg',
    '.jpeg': 'image/jpeg',
    '.png': 'image/png',
    '.gif': 'image/gif',
    '.webp': 'image/webp'
}

def get_mime_type(image_path: str) -> str:
    ext = os.path.splitext(image_path)[1].lower()
    mime = MIME_MAP.get(ext)
    if not mime:
        raise ValueError(f'Unsupported image format: {ext}')
    return mime

def load_image_as_base64(image_path: str) -> tuple:
    """Returns (base64_string, mime_type)"""
    mime = get_mime_type(image_path)
    with open(image_path, 'rb') as f:
        b64 = base64.b64encode(f.read()).decode('utf-8')
    return b64, mime

b64, mime = load_image_as_base64('photo.jpg')
print(f'MIME: {mime}, Size: {len(b64)} bytes base64')

Bildanalyse in einer Agentenschleife

def analyze_image_tool(image_path: str, query: str) -> dict:
    """
    Agent tool: analyse an image and return structured findings.
    """
    import anthropic, base64
    client = anthropic.Anthropic(api_key='YOUR_API_KEY')
    b64, mime = load_image_as_base64(image_path)

    structured_query = (
        query + '\n\nRespond with JSON only: '
        '{"description": str, "objects": [str], "text_found": str, "confidence": float}'
    )
    response = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=512,
        messages=[{
            'role': 'user',
            'content': [
                {'type': 'image', 'source': {'type': 'base64',
                  'media_type': mime, 'data': b64}},
                {'type': 'text', 'text': structured_query}
            ]
        }]
    )
    import json
    return json.loads(response.content[0].text)

Detailgrad der Bildanalyse steuern (OpenAI)

def query_image_openai(
    image_path: str,
    question: str,
    detail: str = 'auto'  # 'low', 'high', or 'auto'
) -> str:
    import base64
    from openai import OpenAI
    client = OpenAI(api_key='YOUR_OPENAI_API_KEY')
    b64, mime = load_image_as_base64(image_path)
    data_url = f'data:{mime};base64,{b64}'

    response = client.chat.completions.create(
        model='gpt-4o',
        max_tokens=512,
        messages=[{
            'role': 'user',
            'content': [
                {
                    'type': 'image_url',
                    'image_url': {'url': data_url, 'detail': detail}
                },
                {'type': 'text', 'text': question}
            ]
        }]
    )
    return response.choices[0].message.content

Konversationen mit mehreren Bildern

def compare_two_images(
    image_path_1: str,
    image_path_2: str,
    comparison_question: str
) -> str:
    import anthropic, base64
    client = anthropic.Anthropic(api_key='YOUR_API_KEY')
    b64_1, mime_1 = load_image_as_base64(image_path_1)
    b64_2, mime_2 = load_image_as_base64(image_path_2)

    response = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=512,
        messages=[{
            'role': 'user',
            'content': [
                {'type': 'image', 'source': {'type': 'base64',
                  'media_type': mime_1, 'data': b64_1}},
                {'type': 'image', 'source': {'type': 'base64',
                  'media_type': mime_2, 'data': b64_2}},
                {'type': 'text', 'text': comparison_question}
            ]
        }]
    )
    return response.content[0].text

Bildagent: Screenshots auslesen

SCREENSHOT_READER_PROMPT = (
    'You are a UI analyser. Given this application screenshot, extract:\n'
    '1. The current page/screen name\n'
    '2. Any error messages\n'
    '3. Key UI elements visible (buttons, form fields, text)\n'
    '4. The overall app state\n\n'
    'Return JSON: {"screen": str, "errors": [str], '
    '"elements": [str], "state": str}'
)

def read_screenshot(screenshot_path: str) -> dict:
    import anthropic, base64, json
    client = anthropic.Anthropic(api_key='YOUR_API_KEY')
    b64, mime = load_image_as_base64(screenshot_path)
    response = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=512,
        messages=[{
            'role': 'user',
            'content': [
                {'type': 'image', 'source': {'type': 'base64',
                  'media_type': mime, 'data': b64}},
                {'type': 'text', 'text': SCREENSHOT_READER_PROMPT}
            ]
        }]
    )
    return json.loads(response.content[0].text)

Kostenoptimierung für Vision-Aufrufe

from PIL import Image
import io, base64

def resize_and_encode(
    image_path: str,
    max_dim: int = 1024
) -> tuple:
    img = Image.open(image_path)
    # Resize maintaining aspect ratio
    ratio = min(max_dim / img.width, max_dim / img.height)
    if ratio < 1.0:
        new_w = int(img.width * ratio)
        new_h = int(img.height * ratio)
        img = img.resize((new_w, new_h), Image.LANCZOS)
        print(f'Resized to {new_w}x{new_h} (from {img.width}x{img.height})')

    # Convert to JPEG for smaller size
    buf = io.BytesIO()
    img.save(buf, format='JPEG', quality=85)
    b64 = base64.b64encode(buf.getvalue()).decode('utf-8')
    return b64, 'image/jpeg'

Einschränkungen von Vision-Modellen und Fallbacks

def analyze_with_confidence(
    image_path: str,
    question: str,
    confidence_threshold: float = 0.7
) -> dict:
    import anthropic, json
    client = anthropic.Anthropic(api_key='YOUR_API_KEY')
    b64, mime = load_image_as_base64(image_path)

    prompt = (
        question + '\n\nAlso rate your confidence (0.0-1.0) in the answer.\n'
        'Return JSON: {"answer": str, "confidence": float, '
        '"uncertainty_reason": str}'
    )
    response = client.messages.create(
        model='claude-opus-4-5', max_tokens=256,
        messages=[{'role': 'user', 'content': [
            {'type': 'image', 'source': {'type': 'base64',
              'media_type': mime, 'data': b64}},
            {'type': 'text', 'text': prompt}
        ]}]
    )
    result = json.loads(response.content[0].text)
    if result['confidence'] < confidence_threshold:
        result['action'] = 'human_review_needed'
    return result

Wissensüberprüfung

Zusammenfassung: Bild-Text-Agenten