Qdrant Vector Database: Semantic Search & AI

# Docker (recommended)
docker run -d --name qdrant \
    -p 6333:6333 \
    -p 6334:6334 \
    -v ~/qdrant-data:/qdrant/storage \
    qdrant/qdrant:v1.9.0

# REST API: port 6333
# gRPC API: port 6334
# Dashboard: http://localhost:6333/dashboard

# Atau install binary
wget https://github.com/qdrant/qdrant/releases/download/v1.9.0/qdrant-x86_64-unknown-linux-gnu.tar.gz
tar xzf qdrant-x86_64-unknown-linux-gnu.tar.gz
./qdrant

# Python client
pip install qdrant-client openai

# JavaScript/TypeScript client
npm install @qdrant/js-client-rest

# Verifikasi
curl http://localhost:6333/health
# Output: {"title":"qdrant - vector search engine","version":"1.9.0"}

from qdrant_client import QdrantClient
from qdrant_client.models import (
    Distance, VectorParams, PointStruct,
    Filter, FieldCondition, MatchValue,
    PayloadSchemaType, CreateAliasOperation
)

# Koneksi ke Qdrant
client = QdrantClient(url="http://localhost:6333")

# Membuat collection
client.create_collection(
    collection_name="documents",
    vectors_config=VectorParams(
        size=384,          # Dimensi embedding
        distance=Distance.COSINE  # COSINE, EUCLID, DOT, MANHATTAN
    ),
)

# Collection dengan multiple named vectors
client.create_collection(
    collection_name="products",
    vectors_config={
        "text": VectorParams(size=384, distance=Distance.COSINE),
        "image": VectorParams(size=512, distance=Distance.COSINE),
    }
)

# Cek collection info
collection_info = client.get_collection("documents")
print(f"Vectors: {collection_info.vectors_count}")
print(f"Status: {collection_info.status}")

import uuid
from qdrant_client.models import PointStruct

# Insert points dengan payload
points = [
    PointStruct(
        id=str(uuid.uuid4()),
        vector=[0.12, 0.45, 0.78, ...],  # 384-dim vector
        payload={
            "title": "Tutorial Python untuk Pemula",
            "category": "programming",
            "language": "id",
            "difficulty": "beginner",
            "tags": ["python", "tutorial", "pemula"],
            "views": 1500,
            "rating": 4.8
        }
    ),
    PointStruct(
        id=str(uuid.uuid4()),
        vector=[0.34, 0.67, 0.23, ...],
        payload={
            "title": "Machine Learning dengan TensorFlow",
            "category": "ai",
            "language": "id",
            "difficulty": "intermediate",
            "tags": ["ml", "tensorflow", "deep-learning"],
            "views": 2300,
            "rating": 4.9
        }
    ),
]

client.upsert(
    collection_name="documents",
    points=points
)

# Search — cari vector paling mirip
results = client.search(
    collection_name="documents",
    query_vector=[0.15, 0.42, 0.80, ...],  # Query vector
    limit=5,
)

for result in results:
    print(f"ID: {result.id}")
    print(f"Score: {result.score:.4f}")
    print(f"Title: {result.payload['title']}")
    print(f"Category: {result.payload['category']}")
    print("---")

# Search dengan filter
results = client.search(
    collection_name="documents",
    query_vector=[0.15, 0.42, 0.80, ...],
    query_filter=Filter(
        must=[
            FieldCondition(
                key="category",
                match=MatchValue(value="programming")
            ),
            FieldCondition(
                key="difficulty",
                match=MatchValue(value="beginner")
            )
        ]
    ),
    limit=5,
)

# Batch insert (untuk data besar)
import random
batch_points = [
    PointStruct(
        id=i,
        vector=[random.random() for _ in range(384)],
        payload={"index": i, "batch": True}
    )
    for i in range(10000)
]

client.upsert(
    collection_name="documents",
    points=batch_points,
    batch_size=100  # Insert per batch
)

from qdrant_client.models import (
    Filter, FieldCondition, MatchValue,
    MatchAny, Range, IsNullCondition,
    NestedCondition
)

# Filter equality
Filter(must=[
    FieldCondition(key="language", match=MatchValue(value="id"))
])

# Filter IN (match any)
Filter(must=[
    FieldCondition(key="category", match=MatchAny(value=["programming", "ai"]))
])

# Filter range
Filter(must=[
    FieldCondition(
        key="rating",
        range=Range(gte=4.5, lte=5.0)
    ),
    FieldCondition(
        key="views",
        range=Range(gte=1000)
    )
])

# Filter array contains
Filter(must=[
    FieldCondition(key="tags", match=MatchValue(value="python"))
])

# Filter negation (must_not)
Filter(
    must=[
        FieldCondition(key="language", match=MatchValue(value="id"))
    ],
    must_not=[
        FieldCondition(key="difficulty", match=MatchValue(value="beginner"))
    ]
)

# OR logic (should)
Filter(
    should=[
        FieldCondition(key="category", match=MatchValue(value="programming")),
        FieldCondition(key="category", match=MatchValue(value="ai")),
    ]
)

# Complex filter: (programming OR ai) AND rating >= 4.5 AND NOT beginner
client.search(
    collection_name="documents",
    query_vector=[0.15, 0.42, 0.80, ...],
    query_filter=Filter(
        should=[
            FieldCondition(key="category", match=MatchValue(value="programming")),
            FieldCondition(key="category", match=MatchValue(value="ai")),
        ],
        must=[
            FieldCondition(key="rating", range=Range(gte=4.5)),
        ],
        must_not=[
            FieldCondition(key="difficulty", match=MatchValue(value="beginner")),
        ]
    ),
    limit=10
)

from qdrant_client.models import PayloadSchemaType

# Index kolom yang sering di-filter untuk performa lebih baik
client.create_payload_index(
    collection_name="documents",
    field_name="category",
    field_schema=PayloadSchemaType.KEYWORD
)

client.create_payload_index(
    collection_name="documents",
    field_name="rating",
    field_schema=PayloadSchemaType.FLOAT
)

client.create_payload_index(
    collection_name="documents",
    field_name="views",
    field_schema=PayloadSchemaType.INTEGER
)

client.create_payload_index(
    collection_name="documents",
    field_name="title",
    field_schema=PayloadSchemaType.TEXT  # Full-text index
)

from qdrant_client.models import HnswConfig, OptimizersConfig

# Membuat collection dengan HNSW config
client.create_collection(
    collection_name="high_perf_docs",
    vectors_config=VectorParams(
        size=384,
        distance=Distance.COSINE
    ),
    hnsw_config=HnswConfig(
        m=16,                # Jumlah koneksi per node (default: 16)
        ef_construct=100,    # Ukuran ef saat membangun index (default: 100)
        full_scan_threshold=20000,  # Jika dataset kecil, gunakan brute force
        max_indexing_threads=0,     # 0 = auto (gunakan semua core)
    ),
    optimizers_config=OptimizersConfig(
        deleted_threshold=0.2,
        vacuum_min_vector_number=1000,
        default_segment_number=0,
        max_segment_size=None,
        memmap_threshold=20000,
        indexing_threshold=20000,
        flush_interval_sec=5,
        max_optimization_threads=1
    )
)

# Mengubah ef saat search (trade-off akurasi vs speed)
results = client.search(
    collection_name="high_perf_docs",
    query_vector=[0.15, 0.42, ...],
    limit=10,
    search_params={
        "hnsw_ef": 128,  # Lebih tinggi = lebih akurat, lebih lambat
        "exact": False     # True = brute force (lebih akurat untuk small dataset)
    }
)

5.2 Parameter HNSW Penting

from qdrant_client.models import ScalarQuantization, ScalarType

# Scalar quantization: mengurangi float32 ke uint8 (4x hemat memory)
client.create_collection(
    collection_name="quantized_docs",
    vectors_config=VectorParams(
        size=384,
        distance=Distance.COSINE
    ),
    quantization_config=ScalarQuantization(
        scalar=ScalarType.INT8,
        quantile=0.99,      # Menggunakan 99th percentile untuk range
        always_ram=True      # Simpan quantized vectors di RAM
    )
)

# Search dengan rescoring (akurasi tetap tinggi)
results = client.search(
    collection_name="quantized_docs",
    query_vector=[0.15, 0.42, ...],
    limit=10,
    search_params={
        "quantization": {
            "ignore": False,
            "rescore": True,      # Rescore dengan original vectors
            "oversampling": 2.0   # Ambil 2x lebih banyak candidate
        }
    }
)

6. RAG Implementation

Retrieval Augmented Generation (RAG) adalah pola arsitektur AI yang menggabungkan vector search dengan LLM. Data relevan diambil dari vector database berdasarkan query, lalu dikirimkan sebagai konteks ke LLM untuk menghasilkan jawaban yang lebih akurat dan grounded.

import openai
from qdrant_client import QdrantClient
from qdrant_client.models import PointStruct

openai_client = openai.OpenAI()
qdrant_client = QdrantClient(url="http://localhost:6333")

# Step 1: Index dokumen
def index_document(text, metadata):
    """Generate embedding dan simpan ke Qdrant."""
    response = openai_client.embeddings.create(
        input=text,
        model="text-embedding-3-small"
    )
    vector = response.data[0].embedding

    qdrant_client.upsert(
        collection_name="knowledge_base",
        points=[PointStruct(
            id=metadata["id"],
            vector=vector,
            payload={
                "text": text,
                "source": metadata.get("source", ""),
                "page": metadata.get("page", 0)
            }
        )]
    )

# Step 2: Retrieval
def retrieve_context(query, top_k=5):
    """Cari dokumen relevan berdasarkan query."""
    response = openai_client.embeddings.create(
        input=query,
        model="text-embedding-3-small"
    )
    query_vector = response.data[0].embedding

    results = qdrant_client.search(
        collection_name="knowledge_base",
        query_vector=query_vector,
        limit=top_k,
        score_threshold=0.7
    )

    return [hit.payload["text"] for hit in results]

# Step 3: Generate answer dengan konteks
def ask_question(question):
    """RAG: retrieve + generate."""
    context_docs = retrieve_context(question)
    context = "\n\n---\n\n".join(context_docs)

    prompt = f"""Berdasarkan konteks berikut, jawab pertanyaan user.
Jika jawaban tidak ada di konteks, katakan "Saya tidak tahu."

Konteks:
{context}

Pertanyaan: {question}

Jawaban:"""

    response = openai_client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "Anda adalah asisten yang akurat dan helpful."},
            {"role": "user", "content": prompt}
        ],
        temperature=0.3
    )

    return {
        "answer": response.choices[0].message.content,
        "sources": context_docs
    }

# Contoh penggunaan
# index_document("CockroachDB adalah distributed SQL database...", {"id": 1, "source": "tutorial"})
result = ask_question("Apa itu CockroachDB?")
print(result["answer"])

7. Clustering & Production

7.1 Qdrant Cluster

# Docker Compose: 3-node Qdrant cluster
# docker-compose.yml
version: '3.8'
services:
  qdrant-1:
    image: qdrant/qdrant:v1.9.0
    ports:
      - "6333:6333"
      - "6334:6334"
    volumes:
      - ./node1:/qdrant/storage
    environment:
      - QDRANT__CLUSTER__ENABLED=true
      - QDRANT__CLUSTER__P2P__PORT=6335
      - QDRANT__CLUSTER__P2P__HOSTS=qdrant-1:6335,qdrant-2:6335,qdrant-3:6335

  qdrant-2:
    image: qdrant/qdrant:v1.9.0
    volumes:
      - ./node2:/qdrant/storage
    environment:
      - QDRANT__CLUSTER__ENABLED=true
      - QDRANT__CLUSTER__P2P__PORT=6335
      - QDRANT__CLUSTER__P2P__HOSTS=qdrant-1:6335,qdrant-2:6335,qdrant-3:6335

  qdrant-3:
    image: qdrant/qdrant:v1.9.0
    volumes:
      - ./node3:/qdrant/storage
    environment:
      - QDRANT__CLUSTER__ENABLED=true
      - QDRANT__CLUSTER__P2P__PORT=6335
      - QDRANT__CLUSTER__P2P__HOSTS=qdrant-1:6335,qdrant-2:6335,qdrant-3:6335

# docker-compose up -d

# Collection dengan replication
client.create_collection(
    collection_name="production_docs",
    vectors_config=VectorParams(size=384, distance=Distance.COSINE),
    replication_factor=2,           # 2 replika per shard
    write_consistency_factor=2,     # Minimal 2 acknowledgement untuk write
    shard_number=3                  # 3 shards
)

8. Integrasi dengan AI Framework

8.1 LangChain + Qdrant

from langchain_qdrant import QdrantVectorStore
from langchain_openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import TextLoader

# Load dan split dokumen
loader = TextLoader("tutorial.txt")
documents = loader.load()
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
docs = splitter.split_documents(documents)

# Buat vector store
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
vector_store = QdrantVectorStore.from_documents(
    docs,
    embeddings,
    url="http://localhost:6333",
    collection_name="langchain_docs",
)

# Retrieval
retriever = vector_store.as_retriever(search_kwargs={"k": 5})
results = retriever.invoke("Bagaimana cara setup CockroachDB?")

# Atau langsung search
results = vector_store.similarity_search_with_score(
    "distributed database",
    k=5
)

8.2 Monitor & Maintain

# Collection statistics
info = client.get_collection("documents")
print(f"Points: {info.points_count}")
print(f"Vectors: {info.vectors_count}")
print(f"Segments: {info.segments_count}")
print(f"Status: {info.status}")
print(f"Indexed vectors: {info.indexed_vectors_count}")

# Cluster info
cluster_info = client.get_cluster_info()
print(f"Peer ID: {cluster_info.peer_id}")
print(f"Peers: {cluster_info.peers}")

# Optimasi (force segment merge)
client.update_collection(
    collection_name="documents",
    optimizer_config=OptimizersConfig(
        deleted_threshold=0.2,
        vacuum_min_vector_number=1000
    )
)

# Snapshot untuk backup
client.create_snapshot(collection_name="documents")

9. Quiz: Uji Pemahamanmu!

Setelah membaca tutorial di atas, jawablah 5 pertanyaan berikut: