문서 로더 및 파서

LlamaIndex와 LangChain 비교

Install

# pip install llama-index
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader

Easiest Path: SimpleDirectoryReader

documents = SimpleDirectoryReader('./docs').load_data()
print(f'{len(documents)} documents')
print(documents[0].text[:200])

LlamaHub

# pip install llama-index-readers-notion
from llama_index.readers.notion import NotionPageReader
reader = NotionPageReader(integration_token='...')
docs = reader.load_data(database_ids=['db-id'])

PDF용 LlamaParse

# pip install llama-parse
from llama_parse import LlamaParse
parser = LlamaParse(result_type='markdown', api_key='...')
docs = parser.load_data('handbook.pdf')

표는 어렵습니다

Web Loaders

from llama_index.readers.web import SimpleWebPageReader
docs = SimpleWebPageReader().load_data(['https://example.com'])

Database Loaders

from llama_index.readers.database import DatabaseReader
reader = DatabaseReader(sql_database=SQLDatabase(engine))
docs = reader.load_data(query='SELECT * FROM articles')

문서 메타데이터

doc = documents[0]
print(doc.text)
print(doc.metadata)
# {'file_name': 'handbook.pdf', 'page_label': '1'}

사용자 지정 메타데이터 추가

for doc in documents:
    doc.metadata['department'] = 'engineering'
    doc.metadata['last_reviewed'] = '2024-08'

LLM에서 메타데이터 제외하기

doc.excluded_llm_metadata_keys = ['internal_id', 'file_path']
# LLM still sees the text, but not those keys.

비동기 로딩

docs = await reader.aload_data(['url1', 'url2', 'url3'])

LlamaParse의 강점

복습