ローダー、スプリッター、ベクトルストア

LangChainにおけるRAGの3本柱

ドキュメントローダー

from langchain_community.document_loaders import PyPDFLoader, WebBaseLoader, TextLoader

pdf_docs = PyPDFLoader('handbook.pdf').load()
web_docs = WebBaseLoader('https://example.com').load()
md_docs = TextLoader('README.md').load()

Documentオブジェクト

doc = pdf_docs[0]
print(doc.page_content)   # the text
print(doc.metadata)       # {'source': 'handbook.pdf', 'page': 1}

一般的なローダー

テキストスプリッター

from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=800,
    chunk_overlap=100,
    separators=['\n\n', '\n', '. ', ' ']
)
chunks = splitter.split_documents(pdf_docs)

特化型スプリッター

トークンを考慮した分割

splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
    encoding_name='cl100k_base',
    chunk_size=400,
    chunk_overlap=50
)

ベクトルストア

from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

embeddings = OpenAIEmbeddings(model='text-embedding-3-small')
store = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory='./db'
)

Retrieval

retriever = store.as_retriever(search_kwargs={'k': 4})
results = retriever.invoke('What is our return policy?')
for d in results:
    print(d.page_content[:200])

永続化

store = Chroma(persist_directory='./db', embedding_function=embeddings)

その他のベクトルストア

from langchain_community.vectorstores import FAISS, Pinecone, Qdrant, Weaviate

store = FAISS.from_documents(chunks, embeddings)
store.save_local('./faiss_db')

MultiVectorRetriever

from langchain.retrievers import MultiVectorRetriever
# Use the parent-child pattern automatically.

Self-Query Retriever

from langchain.retrievers.self_query.base import SelfQueryRetriever
# 'What did Alice say in 2023?' -> filter={author: 'Alice', year: 2023}

リトリーバーの出力

まとめ