ドキュメントの読み込みとテキスト抽出

from pypdf import PdfReader

def load_pdf(file_path):
    reader = PdfReader(file_path)
    pages = []
    for page_num, page in enumerate(reader.pages, start=1):
        text = page.extract_text()
        if text and text.strip():  # skip blank pages
            pages.append({
                'text': text,
                'metadata': {
                    'source': file_path,
                    'page': page_num,
                    'doc_type': 'pdf'
                }
            })
    return pages

docs = load_pdf('annual_report.pdf')
print(f'Loaded {len(docs)} non-empty pages')

from docx import Document

def load_docx(file_path):
    doc = Document(file_path)
    sections = []
    current_section = {'heading': '', 'text': ''}

    for para in doc.paragraphs:
        if para.style.name.startswith('Heading'):
            if current_section['text'].strip():
                sections.append(current_section.copy())
            current_section = {'heading': para.text, 'text': ''}
        else:
            current_section['text'] += para.text + '\n'

    if current_section['text'].strip():
        sections.append(current_section)

    return [{'text': f"{s['heading']}\n{s['text']}",
             'metadata': {'source': file_path, 'section': s['heading']}}
            for s in sections]

import requests
from bs4 import BeautifulSoup

def load_url(url):
    response = requests.get(url, timeout=10)
    response.raise_for_status()

    soup = BeautifulSoup(response.text, 'html.parser')

    # Remove noise elements
    for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']):
        tag.decompose()

    # Try to find main content
    main = soup.find('article') or soup.find('main') or soup.find('body')
    text = main.get_text(separator='\n', strip=True)

    return [{'text': text, 'metadata': {'source': url, 'doc_type': 'html'}}]

import pytesseract
from pdf2image import convert_from_path

def load_scanned_pdf(file_path):
    # Convert PDF pages to PIL images
    pages_as_images = convert_from_path(file_path, dpi=300)
    results = []
    for page_num, img in enumerate(pages_as_images, start=1):
        text = pytesseract.image_to_string(img, lang='eng')
        results.append({
            'text': text,
            'metadata': {
                'source': file_path,
                'page': page_num,
                'ocr': True  # flag for quality review
            }
        })
    return results

from unstructured.partition.auto import partition

def load_any_document(file_path):
    elements = partition(filename=file_path)
    docs = []
    for element in elements:
        docs.append({
            'text': str(element),
            'metadata': {
                'source': file_path,
                'element_type': type(element).__name__,
                'category': element.category
            }
        })
    return docs

# Works on .pdf, .docx, .pptx, .html, .eml, .xlsx
docs = load_any_document('presentation.pptx')

import re

def clean_text(text):
    # Rejoin hyphenated line breaks (PDF column artifacts)
    text = re.sub(r'-(\n)([a-z])', r'\2', text)
    # Normalize whitespace
    text = re.sub(r' +', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    # Remove page numbers standing alone on a line
    text = re.sub(r'^\d+$', '', text, flags=re.MULTILINE)
    # Strip leading/trailing whitespace from each line
    lines = [line.strip() for line in text.split('\n')]
    return '\n'.join(lines).strip()

import psycopg2

def load_from_database(conn_string, query):
    conn = psycopg2.connect(conn_string)
    cursor = conn.cursor()
    cursor.execute(query)
    columns = [desc[0] for desc in cursor.description]
    docs = []
    for row in cursor.fetchall():
        # Convert row to natural language text
        parts = [f'{col}: {val}' for col, val in zip(columns, row) if val]
        text = '\n'.join(parts)
        docs.append({'text': text, 'metadata': {'source': 'database'}})
    conn.close()
    return docs

import os
from datetime import datetime

def load_pdf_with_provenance(file_path):
    from pypdf import PdfReader
    reader = PdfReader(file_path)
    stat = os.stat(file_path)
    base_metadata = {
        'source': file_path,
        'title': reader.metadata.get('/Title', os.path.basename(file_path)),
        'author': reader.metadata.get('/Author', 'Unknown'),
        'doc_type': 'pdf',
        'file_size_kb': stat.st_size // 1024,
        'loaded_at': datetime.utcnow().isoformat()
    }
    pages = []
    for i, page in enumerate(reader.pages, 1):
        text = page.extract_text()
        if text and text.strip():
            pages.append({'text': text, 'metadata': {**base_metadata, 'page': i}})
    return pages

from concurrent.futures import ThreadPoolExecutor, as_completed

def batch_load_documents(file_paths, max_workers=8):
    all_docs = []
    errors = []
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        future_to_path = {
            executor.submit(load_pdf_with_provenance, p): p
            for p in file_paths
        }
        for future in as_completed(future_to_path):
            path = future_to_path[future]
            try:
                docs = future.result()
                all_docs.extend(docs)
                print(f'Loaded {len(docs)} pages from {path}')
            except Exception as e:
                errors.append({'path': path, 'error': str(e)})
    return all_docs, errors

def validate_documents(docs, min_length=100):
    valid = []
    skipped = 0
    for doc in docs:
        text = doc['text']
        if len(text) < min_length:
            skipped += 1
            continue
        # Check for high non-ASCII ratio (garbled OCR)
        non_ascii = sum(1 for c in text if ord(c) > 127)
        if non_ascii / max(len(text), 1) > 0.3:
            skipped += 1
            continue
        valid.append(doc)
    print(f'Valid: {len(valid)}, Skipped: {skipped}')
    return valid