LangChain RAG: Retrieval Augmented Generation — Tutorial Lengkap

# Instalasi LangChain dan dependencies
pip install langchain langchain-openai langchain-community
pip install chromadb faiss-cpu tiktoken
pip install pypdf docx2txt unstructured

# Untuk OpenAI API
pip install openai

# Untuk environment variables
pip install python-dotenv

# =============================================
# Document Loading dengan LangChain
# =============================================
from langchain_community.document_loaders import (
    PyPDFLoader,
    TextLoader,
    CSVLoader,
    DirectoryLoader,
    WebBaseLoader,
    UnstructuredMarkdownLoader
)

# ----- 1. Load PDF -----
loader = PyPDFLoader("dokumen/produk.pdf")
pages = loader.load()
print(f"Jumlah halaman: {len(pages)}")
print(f"Contoh halaman 1: {pages[0].page_content[:200]}")
print(f"Metadata: {pages[0].metadata}")
# {'source': 'dokumen/produk.pdf', 'page': 0}

# ----- 2. Load Text File -----
loader = TextLoader("dokumen/catatan.txt", encoding="utf-8")
docs = loader.load()

# ----- 3. Load CSV -----
loader = CSVLoader(
    file_path="dokumen/data.csv",
    csv_args={"delimiter": ",", "quotechar": '"'},
    source_column="url"
)
docs = loader.load()

# ----- 4. Load dari Web -----
loader = WebBaseLoader([
    "https://example.com/artikel-1",
    "https://example.com/artikel-2"
])
web_docs = loader.load()

# ----- 5. Load seluruh direktori -----
loader = DirectoryLoader(
    "./dokumen/",
    glob="**/*.pdf",
    loader_cls=PyPDFLoader,
    show_progress=True
)
all_docs = loader.load()
print(f"Total dokumen: {len(all_docs)}")

# ----- Metadata manipulation -----
for doc in all_docs:
    doc.metadata["category"] = "produk"
    doc.metadata["company"] = "BeebaneLabs"

# =============================================
# Text Splitting dengan LangChain
# =============================================
from langchain.text_splitter import (
    RecursiveCharacterTextSplitter,
    CharacterTextSplitter,
    TokenTextSplitter,
    MarkdownHeaderTextSplitter
)

# ----- 1. RecursiveCharacterTextSplitter (Paling Umum) -----
# Memecah berdasarkan separator secara rekursif
splitter = RecursiveCharacterTextSplitter(
    chunk_size=500,
    chunk_overlap=100,
    length_function=len,
    separators=["\n\n", "\n", ". ", " ", ""]
)

chunks = splitter.split_documents(docs)
print(f"Dokumen: {len(docs)} → Chunks: {len(chunks)}")
print(f"Contoh chunk: {chunks[0].page_content[:100]}")

# ----- 2. TokenTextSplitter (Berdasarkan Token) -----
# Lebih akurat untuk LLM karena LLM bekerja dengan token
splitter = TokenTextSplitter(
    chunk_size=200,    # 200 token per chunk
    chunk_overlap=20,  # 20 token overlap
    encoding_name="cl100k_base"  # GPT-4 encoding
)
token_chunks = splitter.split_documents(docs)

# ----- 3. MarkdownHeaderTextSplitter -----
# Memecah berdasarkan header markdown, mempertahankan struktur
headers_to_split = [
    ("#", "Header 1"),
    ("##", "Header 2"),
    ("###", "Header 3"),
]
splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split)

# Untuk markdown text (bukan Document object)
md_text = """
# Judul Utama
Ini adalah konten bagian 1.

## Sub Judul
Ini adalah konten bagian 2.

### Sub Sub
Detail lebih lanjut.
"""
md_splits = splitter.split_text(md_text)
for split in md_splits:
    print(f"Content: {split.page_content[:50]}")
    print(f"Metadata: {split.metadata}")
    print("---")

# ----- 4. Semantic Chunking (Advanced) -----
# Chunk berdasarkan kemiripan semantik (butuh embedding)
from langchain_experimental.text_splitter import SemanticChunker
from langchain_openai import OpenAIEmbeddings

semantic_splitter = SemanticChunker(
    OpenAIEmbeddings(),
    breakpoint_threshold_type="percentile",
    breakpoint_threshold_amount=95
)
semantic_chunks = semantic_splitter.split_documents(docs)

# =============================================
# Embedding Models di LangChain
# =============================================
import os
from dotenv import load_dotenv
load_dotenv()

# ----- 1. OpenAI Embeddings -----
from langchain_openai import OpenAIEmbeddings

embeddings_openai = OpenAIEmbeddings(
    model="text-embedding-3-small",  # 1536 dimensi
    # model="text-embedding-3-large",  # 3072 dimensi (lebih akurat)
)

# Embed satu teks
vector = embeddings_openai.embed_query("Apa itu RAG?")
print(f"Dimensi: {len(vector)}")  # 1536
print(f"5 elemen pertama: {vector[:5]}")

# Embed banyak teks sekaligus (batch)
texts = ["Apa itu RAG?", "Cara kerja LangChain", "Vector database"]
vectors = embeddings_openai.embed_documents(texts)
print(f"Jumlah vektor: {len(vectors)}")  # 3

# ----- 2. HuggingFace Embeddings (Gratis, Lokal) -----
from langchain_huggingface import HuggingFaceEmbeddings

embeddings_hf = HuggingFaceEmbeddings(
    model_name="sentence-transformers/all-MiniLM-L6-v2",
    model_kwargs={"device": "cpu"},  # "cuda" jika punya GPU
    encode_kwargs={"normalize_embeddings": True}
)

vector = embeddings_hf.embed_query("Apa itu RAG?")
print(f"Dimensi: {len(vector)}")  # 384

# ----- 3. Multilingual Embeddings (Untuk Bahasa Indonesia) -----
embeddings_multi = HuggingFaceEmbeddings(
    model_name="intfloat/multilingual-e5-large",
    model_kwargs={"device": "cpu"},
    encode_kwargs={"normalize_embeddings": True}
)
# Dimensi: 1024, mendukung bahasa Indonesia dengan baik

# =============================================
# Vector Store dengan ChromaDB
# =============================================
from langchain_community.vectorstores import Chroma
from langchain_openai import OpenAIEmbeddings
from langchain_community.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

# 1. Load & Split
loader = PyPDFLoader("dokumen/panduan.pdf")
docs = loader.load()

splitter = RecursiveCharacterTextSplitter(
    chunk_size=500, chunk_overlap=100
)
chunks = splitter.split_documents(docs)

# 2. Buat Vector Store (ChromaDB - persisted local)
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory="./chroma_db",  # Simpan ke disk
    collection_name="dokumen_produk"
)
print(f"Jumlah dokumen di store: {vectorstore._collection.count()}")

# 3. Similarity Search
query = "Apa fitur utama produk ini?"
results = vectorstore.similarity_search(query, k=3)

for i, doc in enumerate(results):
    print(f"\n--- Hasil {i+1} ---")
    print(f"Content: {doc.page_content[:150]}")
    print(f"Source: {doc.metadata.get('source', 'N/A')}")
    print(f"Page: {doc.metadata.get('page', 'N/A')}")

# 4. Similarity Search dengan Score
results_with_score = vectorstore.similarity_search_with_score(query, k=3)
for doc, score in results_with_score:
    print(f"Score: {score:.4f} | Content: {doc.page_content[:100]}")

# 5. Load Vector Store yang sudah ada (persisted)
vectorstore_loaded = Chroma(
    persist_directory="./chroma_db",
    embedding_function=embeddings,
    collection_name="dokumen_produk"
)

# =============================================
# Vector Store dengan FAISS (Facebook AI Similarity Search)
# =============================================
from langchain_community.vectorstores import FAISS

# Buat FAISS vector store
faiss_store = FAISS.from_documents(chunks, embeddings)

# Similarity search
results = faiss_store.similarity_search("apa itu machine learning?", k=5)

# Save & Load
faiss_store.save_local("./faiss_index")
faiss_store_loaded = FAISS.load_local(
    "./faiss_index",
    embeddings,
    allow_dangerous_deserialization=True
)

# Merge dua vector store
faiss_store_2 = FAISS.from_documents(additional_chunks, embeddings)
faiss_store.merge_from(faiss_store_2)

# =============================================
# RAG Chain dengan LangChain LCEL
# =============================================
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough

# Setup
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
vectorstore = Chroma(
    persist_directory="./chroma_db",
    embedding_function=embeddings
)
retriever = vectorstore.as_retriever(
    search_type="similarity",
    search_kwargs={"k": 4}
)

# Prompt Template
template = """Anda adalah asisten AI yang membantu menjawab pertanyaan 
berdasarkan konteks yang diberikan. Jika jawaban tidak ada di konteks, 
katakan "Maaf, saya tidak menemukan informasi tersebut di dokumen."

Konteks:
{context}

Pertanyaan: {question}

Jawaban:"""

prompt = ChatPromptTemplate.from_template(template)

# LLM
llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)

# Helper function untuk format docs
def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

# RAG Chain menggunakan LCEL (LangChain Expression Language)
rag_chain = (
    {"context": retriever | format_docs, "question": RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)

# Invoke
answer = rag_chain.invoke("Apa saja fitur utama produk ini?")
print(answer)

# Streaming response
for chunk in rag_chain.stream("Bagaimana cara menggunakan fitur X?"):
    print(chunk, end="", flush=True)

# =============================================
# Conversational RAG dengan History
# =============================================
from langchain.chains import create_history_aware_retriever
from langchain.chains import create_retrieval_chain
from langchain.chains.combine_documents import create_stuff_documents_chain
from langchain_core.messages import HumanMessage, AIMessage
from langchain_core.prompts import MessagesPlaceholder

# Contextualize question prompt (rewrite question with history)
contextualize_prompt = ChatPromptTemplate.from_messages([
    ("system", "Given a chat history and the latest user question, "
     "reformulate the question to be standalone."),
    MessagesPlaceholder("chat_history"),
    ("human", "{input}"),
])

history_aware_retriever = create_history_aware_retriever(
    llm, retriever, contextualize_prompt
)

# Answer prompt
answer_prompt = ChatPromptTemplate.from_messages([
    ("system", "Answer based on context:\n\n{context}"),
    MessagesPlaceholder("chat_history"),
    ("human", "{input}"),
])

document_chain = create_stuff_documents_chain(llm, answer_prompt)
rag_chain = create_retrieval_chain(history_aware_retriever, document_chain)

# Conversation dengan history
chat_history = []

# Turn 1
response = rag_chain.invoke({
    "chat_history": chat_history,
    "input": "Apa itu produk X?"
})
print(response["answer"])
chat_history.append(HumanMessage(content="Apa itu produk X?"))
chat_history.append(AIMessage(content=response["answer"]))

# Turn 2 (follow-up question)
response2 = rag_chain.invoke({
    "chat_history": chat_history,
    "input": "Berapa harganya?"  # Context-aware!
})
print(response2["answer"])

# =============================================
# Multi-Query Retriever
# =============================================
from langchain.retrievers import MultiQueryRetriever

# Otomatis generate multiple query dari satu pertanyaan
multi_retriever = MultiQueryRetriever.from_llm(
    retriever=vectorstore.as_retriever(),
    llm=llm
)

# User: "Cara install aplikasi?"
# Generated queries:
# 1. "Langkah instalasi aplikasi"
# 2. "Panduan setup aplikasi"
# 3. "Tutorial deployment aplikasi"
# → Gabungkan hasil dari semua query!

results = multi_retriever.invoke("Cara install aplikasi?")

# =============================================
# Contextual Compression (Re-ranking)
# =============================================
from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import LLMChainExtractor

compressor = LLMChainExtractor.from_llm(llm)
compression_retriever = ContextualCompressionRetriever(
    base_compressor=compressor,
    base_retriever=vectorstore.as_retriever(search_kwargs={"k": 10})
)
# Ambil 10 docs, lalu kompres hanya bagian yang relevan

# =============================================
# Self-Query: LLM generate filter dari pertanyaan natural
# =============================================
from langchain.chains.query_constructor.base import AttributeInfo
from langchain.retrievers.self_query.base import SelfQueryRetriever

metadata_field_info = [
    AttributeInfo(name="category", description="Kategori dokumen", type="string"),
    AttributeInfo(name="year", description="Tahun publikasi", type="integer"),
    AttributeInfo(name="source", description="Sumber file", type="string"),
]

self_query_retriever = SelfQueryRetriever.from_llm(
    llm=llm,
    vectorstore=vectorstore,
    document_contents="Dokumen produk",
    metadata_field_info=metadata_field_info,
)

# User: "Dokumen tentang pricing dari tahun 2025"
# → Auto generate filter: category="pricing" AND year=2025
results = self_query_retriever.invoke(
    "Dokumen tentang pricing dari tahun 2025"
)

# =============================================
# RAG API dengan FastAPI
# =============================================
# pip install fastapi uvicorn

from fastapi import FastAPI
from pydantic import BaseModel

app = FastAPI()

# Pre-load vectorstore & chain (di startup)
vectorstore = Chroma(
    persist_directory="./chroma_db",
    embedding_function=OpenAIEmbeddings()
)
retriever = vectorstore.as_retriever(search_kwargs={"k": 4})

class QueryRequest(BaseModel):
    question: str
    top_k: int = 4

class QueryResponse(BaseModel):
    answer: str
    sources: list

@app.post("/ask", response_model=QueryResponse)
async def ask_question(req: QueryRequest):
    # Retrieve relevant docs
    docs = retriever.invoke(req.question)
    
    # Generate answer
    context = "\n\n".join(d.page_content for d in docs)
    answer = rag_chain.invoke(req.question)
    
    sources = [
        {"source": d.metadata.get("source", ""), 
         "page": d.metadata.get("page", 0)}
        for d in docs
    ]
    
    return QueryResponse(answer=answer, sources=sources)

# Run: uvicorn main:app --reload --port 8000