Tavily dan SerpAPI untuk Pencarian Agen

Mengapa Agen Memerlukan Pencarian Web

Tavily: Dibuat Khusus untuk Agen AI

from tavily import TavilyClient
import os

client = TavilyClient(api_key=os.getenv('TAVILY_API_KEY'))

results = client.search(
    query='latest OpenAI GPT-5 release date',
    max_results=5
)

for r in results['results']:
    print(r['title'])
    print(r['url'])
    print(r['content'][:200])
    print('---')

Parameter Pencarian Tavily

# Basic search
results = client.search(
    query='Python async best practices 2024',
    max_results=5,
    search_depth='basic'  # 'basic' (fast) or 'advanced' (slower, deeper)
)

# Domain-filtered search
results = client.search(
    query='machine learning papers',
    max_results=5,
    include_domains=['arxiv.org', 'papers.nips.cc', 'openreview.net']
)

# Exclude low-quality domains
results = client.search(
    query='stock market today',
    max_results=5,
    exclude_domains=['pinterest.com', 'quora.com']
)

Struktur Hasil Tavily

results = client.search('Claude AI pricing 2024', max_results=3)

for r in results['results']:
    print(f"Title:   {r['title']}")
    print(f"URL:     {r['url']}")
    print(f"Score:   {r.get('score', 'N/A')}")
    print(f"Date:    {r.get('published_date', 'unknown')}")
    print(f"Content: {r['content'][:300]}")
    print()

# Tavily also returns 'answer' — a direct answer synthesized from results
if results.get('answer'):
    print('Direct answer:', results['answer'])

SerpApi: Akses Terprogram ke Google

from serpapi import GoogleSearch
import os

params = {
    'q': 'best Python web frameworks 2024',
    'api_key': os.getenv('SERPAPI_KEY'),
    'num': 5,              # number of results
    'hl': 'en',            # language
    'gl': 'us',            # country
    'safe': 'active'       # safe search
}

search = GoogleSearch(params)
results = search.get_dict()

for r in results.get('organic_results', []):
    print(r['title'])
    print(r['link'])
    print(r.get('snippet', ''))
    print()

Struktur Hasil SerpApi

def extract_serp_results(serp_data, max_results=5):
    results = []

    # Featured snippet / answer box (highest priority)
    if 'answer_box' in serp_data:
        box = serp_data['answer_box']
        results.append({
            'title': box.get('title', 'Featured Snippet'),
            'url':   box.get('link', ''),
            'snippet': box.get('answer') or box.get('snippet', ''),
            'is_featured': True
        })

    # Organic results
    for r in serp_data.get('organic_results', [])[:max_results]:
        results.append({
            'title':   r.get('title', ''),
            'url':     r.get('link', ''),
            'snippet': r.get('snippet', ''),
            'position': r.get('position', 0),
            'is_featured': False
        })

    return results

if __name__ == '__main__':
    demo_serp = {
        'answer_box': {'title': 'Python version', 'link': 'https://python.org', 'answer': '3.13'},
        'organic_results': [
            {'title': 'Python Docs', 'link': 'https://docs.python.org', 'snippet': 'Official docs', 'position': 1},
        ],
    }
    for r in extract_serp_results(demo_serp):
        print(f"{'[FEATURED] ' if r['is_featured'] else ''}{r['title']} - {r['url']}")

Opsi Kesegaran dan Pemfilteran Tanggal

# Tavily: results from last 7 days
recent_results = client.search(
    query='AI news',
    max_results=5,
    days=7  # only results published in last 7 days
)

# SerpApi: results from last month
params = {
    'q': 'AI news',
    'api_key': os.getenv('SERPAPI_KEY'),
    'tbs': 'qdr:m',   # qdr:d=day, qdr:w=week, qdr:m=month, qdr:y=year
    'num': 5
}
search = GoogleSearch(params)
results = search.get_dict()

Membangun Alat Pencarian untuk LangChain

from langchain.tools import Tool
from tavily import TavilyClient
import os

tavily = TavilyClient(api_key=os.getenv('TAVILY_API_KEY'))

def tavily_search(query: str) -> str:
    results = tavily.search(query=query, max_results=3, search_depth='basic')
    output = []
    for r in results['results']:
        output.append(f"Title: {r['title']}\nURL: {r['url']}\nContent: {r['content'][:500]}")
    return '\n\n'.join(output)

web_search_tool = Tool(
    name='web_search',
    func=tavily_search,
    description='Search the web for current information. Input: a search query string. Returns top results with title, URL, and content.'
)

Pemfilteran Domain untuk Pengendalian Kualitas

TRUSTED_DOMAINS = {
    'medical':  ['nih.gov', 'who.int', 'mayoclinic.org', 'pubmed.ncbi.nlm.nih.gov'],
    'legal':    ['law.cornell.edu', 'supreme.justia.com', 'congress.gov'],
    'tech':     ['docs.python.org', 'developer.mozilla.org', 'stackoverflow.com'],
    'finance':  ['sec.gov', 'federalreserve.gov', 'bloomberg.com']
}

SPAM_DOMAINS = ['pinterest.com', 'quora.com', 'answers.yahoo.com', 'wikihow.com']

def domain_aware_search(query, domain_category=None):
    kwargs = {'query': query, 'max_results': 5, 'exclude_domains': SPAM_DOMAINS}
    if domain_category and domain_category in TRUSTED_DOMAINS:
        kwargs['include_domains'] = TRUSTED_DOMAINS[domain_category]
    return client.search(**kwargs)

Menangani Kesalahan Pencarian dan Batas Laju

import time

def robust_search(query, max_retries=3):
    for attempt in range(max_retries):
        try:
            results = client.search(query=query, max_results=5)
            if results and results.get('results'):
                return results['results']
            print(f'No results for: {query}')
            return []
        except Exception as e:
            if '429' in str(e) or 'rate' in str(e).lower():
                wait = 2 ** attempt  # 1s, 2s, 4s
                print(f'Rate limited. Waiting {wait}s...')
                time.sleep(wait)
            else:
                print(f'Search error: {e}')
                return []
    return []

Normalisasi Panjang Cuplikan

MAX_SNIPPET_CHARS = 600
MIN_SNIPPET_CHARS = 100

def normalize_snippets(results):
    normalized = []
    for r in results:
        content = r.get('content') or r.get('snippet', '')
        # Truncate long snippets at sentence boundary
        if len(content) > MAX_SNIPPET_CHARS:
            # Find last sentence end before limit
            cutoff = content.rfind('. ', 0, MAX_SNIPPET_CHARS)
            content = content[:cutoff + 1] if cutoff > 0 else content[:MAX_SNIPPET_CHARS]
        # Skip very short snippets
        if len(content) < MIN_SNIPPET_CHARS:
            continue
        normalized.append({**r, 'content': content})
    return normalized

if __name__ == '__main__':
    demo_results = [
        {'content': 'Short.'},
        {'content': 'A' * 200 + '. ' + 'B' * 500},
    ]
    for r in normalize_snippets(demo_results):
        print(f"Length {len(r['content'])}: {r['content'][:60]}...")

Uji Pemahaman

Ringkasan: Tavily dan SerpApi untuk Pencarian Agen