로더, 분할기 및 벡터 저장소

LangChain의 RAG 3가지 핵심 요소

문서 로더

from langchain_community.document_loaders import PyPDFLoader, WebBaseLoader, TextLoader

pdf_docs = PyPDFLoader('handbook.pdf').load()
web_docs = WebBaseLoader('https://example.com').load()
md_docs = TextLoader('README.md').load()

문서 객체

doc = pdf_docs[0]
print(doc.page_content)   # the text
print(doc.metadata)       # {'source': 'handbook.pdf', 'page': 1}

일반적인 로더

텍스트 분할기

from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=800,
    chunk_overlap=100,
    separators=['\n\n', '\n', '. ', ' ']
)
chunks = splitter.split_documents(pdf_docs)

특수 분할기

토큰 인식 분할

splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
    encoding_name='cl100k_base',
    chunk_size=400,
    chunk_overlap=50
)

벡터 저장소

from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

embeddings = OpenAIEmbeddings(model='text-embedding-3-small')
store = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory='./db'
)

Retrieval

retriever = store.as_retriever(search_kwargs={'k': 4})
results = retriever.invoke('What is our return policy?')
for d in results:
    print(d.page_content[:200])

영속성

store = Chroma(persist_directory='./db', embedding_function=embeddings)

기타 벡터 저장소

from langchain_community.vectorstores import FAISS, Pinecone, Qdrant, Weaviate

store = FAISS.from_documents(chunks, embeddings)
store.save_local('./faiss_db')

MultiVectorRetriever

from langchain.retrievers import MultiVectorRetriever
# Use the parent-child pattern automatically.

셀프 쿼리 검색기

from langchain.retrievers.self_query.base import SelfQueryRetriever
# 'What did Alice say in 2023?' -> filter={author: 'Alice', year: 2023}

검색기 출력

복습