信頼性の高いフォーム入力エージェントの構築

現実的なプロジェクト

アーキテクチャ

Step 1: Define the Task

TASK = {
    'url': 'https://example.com/apply',
    'login': {'user': 'alice', 'pass': '...'},
    'fields': {
        'first_name': 'Alice',
        'last_name': 'Smith',
        'email': 'alice@example.com',
        'role': 'Senior Engineer'
    }
}

Step 2: Login

page.goto(TASK['url'])
if page.locator('input[name="username"]').is_visible():
    page.fill('input[name="username"]', TASK['login']['user'])
    page.fill('input[name="password"]', TASK['login']['pass'])
    page.click('button[type="submit"]')
    page.wait_for_load_state('networkidle')

Step 3: Field Filler

def fill_field(page, field_name, value):
    for selector in [
        f'input[name="{field_name}"]',
        f'input[aria-label="{field_name}"]',
        f'input[placeholder*="{field_name}"]',
        f'label:has-text("{field_name}") + input'
    ]:
        if page.locator(selector).count() > 0:
            page.fill(selector, value)
            return True
    return False

# --- demo ---
class _FakeLocator:
    def __init__(self, found):
        self._found = found
    def count(self):
        return 1 if self._found else 0

class _FakePage:
    """Only the 3rd selector (placeholder) matches, to show the fallback loop."""
    def locator(self, selector):
        return _FakeLocator(found='placeholder*=' in selector)
    def fill(self, selector, value):
        print(f'Filled via selector: {selector} -> {value!r}')

page = _FakePage()
ok = fill_field(page, 'email', 'alice@example.com')
print(f'fill_field succeeded: {ok}')

ステップ4：LLM支援ロケーター

def ask_llm_for_selector(html, field_name):
    prompt = f'Find the input element for "{field_name}" in this HTML and return a CSS selector. HTML:\n{html[:5000]}'
    return llm.invoke(prompt).content.strip()

ステップ5：ビジョンフォールバック

if not filled:
    screenshot = page.screenshot()
    coordinate = vision_llm_locate(screenshot, field_name)
    page.mouse.click(coordinate.x, coordinate.y)
    page.keyboard.type(value)

ステップ6：各ステップを検証する

actual = page.input_value(selector)
assert actual == expected_value, f'Expected {expected_value}, got {actual}'

Step 7: Submit and Verify

page.click('button[type="submit"]')
page.wait_for_load_state('networkidle')
if page.locator('text=Thank you').count() == 0:
    raise SubmissionFailed('Success page not detected')

ステップ8：再試行ロジック

@retry(stop=stop_after_attempt(3), wait=wait_fixed(2))
def fill_and_submit(page, task):
    ...

ステップ9：CAPTCHAへの対応

ステップ10：ログ記録と再生

context.tracing.stop(path=f'trace-{task_id}.zip')
# View with: playwright show-trace trace-abc.zip

同時実行に関する考慮事項

対象サイトへの配慮

堅牢性戦略

要点の振り返り