Dokument-Loader und Parser

LlamaIndex im Vergleich zu LangChain

Install

# pip install llama-index
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader

Easiest Path: SimpleDirectoryReader

documents = SimpleDirectoryReader('./docs').load_data()
print(f'{len(documents)} documents')
print(documents[0].text[:200])

LlamaHub

# pip install llama-index-readers-notion
from llama_index.readers.notion import NotionPageReader
reader = NotionPageReader(integration_token='...')
docs = reader.load_data(database_ids=['db-id'])

LlamaParse für PDFs

# pip install llama-parse
from llama_parse import LlamaParse
parser = LlamaParse(result_type='markdown', api_key='...')
docs = parser.load_data('handbook.pdf')

Tabellen sind schwierig

Web Loaders

from llama_index.readers.web import SimpleWebPageReader
docs = SimpleWebPageReader().load_data(['https://example.com'])

Database Loaders

from llama_index.readers.database import DatabaseReader
reader = DatabaseReader(sql_database=SQLDatabase(engine))
docs = reader.load_data(query='SELECT * FROM articles')

Dokumentenmetadaten

doc = documents[0]
print(doc.text)
print(doc.metadata)
# {'file_name': 'handbook.pdf', 'page_label': '1'}

Benutzerdefinierte Metadaten hinzufügen

for doc in documents:
    doc.metadata['department'] = 'engineering'
    doc.metadata['last_reviewed'] = '2024-08'

Metadaten aus dem LLM ausschließen

doc.excluded_llm_metadata_keys = ['internal_id', 'file_path']
# LLM still sees the text, but not those keys.

Asynchrones Laden

docs = await reader.aload_data(['url1', 'url2', 'url3'])

Die Stärke von LlamaParse

Zusammenfassung