Vector Embeddings untuk Semantic Search — Models, Similarity & Dimensionality Reduction

# =============================================
# Text Embeddings
# =============================================

# ----- 1. OpenAI Embeddings -----
from openai import OpenAI
client = OpenAI()

response = client.embeddings.create(
    model="text-embedding-3-small",
    input="Belajar vector embeddings untuk semantic search"
)
vector = response.data[0].embedding
print(f"Dimensi: {len(vector)}")  # 1536
print(f"5 elemen pertama: {vector[:5]}")

# Batch embedding
texts = [
    "Machine learning adalah subset AI",
    "Deep learning menggunakan neural network",
    "Resep nasi goreng enak dan mudah",
    "Jadwal pertandingan sepak bola"
]
response = client.embeddings.create(
    model="text-embedding-3-small",
    input=texts
)
embeddings = [item.embedding for item in response.data]

# ----- 2. Hugging Face (Gratis, Lokal) -----
from sentence_transformers import SentenceTransformer

model = SentenceTransformer("all-MiniLM-L6-v2")  # 384 dimensi
embeddings = model.encode(texts)
print(f"Shape: {embeddings.shape}")  # (4, 384)

# Multilingual untuk bahasa Indonesia
model_multi = SentenceTransformer("intfloat/multilingual-e5-large")
embeddings_id = model_multi.encode([
    "Apa itu machine learning?",
    "Machine learning adalah cabang AI",
    "Resep masakan Indonesia"
])
print(f"Shape: {embeddings_id.shape}")  # (3, 1024)

# =============================================
# Efficient Batch Similarity Search
# =============================================
import numpy as np

def batch_cosine_search(query_emb, doc_embeddings, top_k=5):
    """Vectorized cosine similarity — sangat cepat."""
    # Normalize
    query_norm = query_emb / np.linalg.norm(query_emb)
    doc_norms = doc_embeddings / np.linalg.norm(doc_embeddings, axis=1, keepdims=True)
    
    # Dot product (sama dengan cosine karena sudah normalized)
    scores = np.dot(doc_norms, query_norm)
    
    # Top-K
    top_indices = np.argsort(scores)[::-1][:top_k]
    return [(i, scores[i]) for i in top_indices]

# Contoh dengan 1 juta dokumen (simulasi)
import time
n_docs = 1_000_000
dim = 384

# Simulate embeddings
doc_embeddings = np.random.randn(n_docs, dim).astype(np.float32)
query = np.random.randn(dim).astype(np.float32)

start = time.time()
results = batch_cosine_search(query, doc_embeddings, top_k=10)
elapsed = time.time() - start
print(f"Search 1M docs: {elapsed:.3f} seconds")
print(f"Top result: index={results[0][0]}, score={results[0][1]:.4f}")

# =============================================
# Similarity Metrics
# =============================================
import numpy as np
from numpy.linalg import norm

def cosine_similarity(a, b):
    return np.dot(a, b) / (norm(a) * norm(b))

def euclidean_distance(a, b):
    return norm(a - b)

def dot_product(a, b):
    return np.dot(a, b)

# Contoh penggunaan
texts = [
    "Saya suka belajar AI",          # 0
    "AI sangat menarik dipelajari",   # 1
    "Resep nasi goreng spesial",      # 2
]

embeddings = model.encode(texts)

# Hitung similarity
for i in range(len(texts)):
    for j in range(i+1, len(texts)):
        sim = cosine_similarity(embeddings[i], embeddings[j])
        print(f"'{texts[i][:30]}' vs '{texts[j][:30]}': {sim:.3f}")

# Output:
# 'Saya suka belajar AI' vs 'AI sangat menarik dipelajari': 0.821
# 'Saya suka belajar AI' vs 'Resep nasi goreng spesial': 0.142
# 'AI sangat menarik dipelajari' vs 'Resep nasi goreng spesial': 0.098

# =============================================
# Semantic Search Engine
# =============================================
import numpy as np
from sentence_transformers import SentenceTransformer

class SemanticSearchEngine:
    def __init__(self, model_name="all-MiniLM-L6-v2"):
        self.model = SentenceTransformer(model_name)
        self.documents = []
        self.embeddings = None
    
    def add_documents(self, docs):
        self.documents.extend(docs)
        new_embeddings = self.model.encode(docs, show_progress_bar=True)
        if self.embeddings is None:
            self.embeddings = new_embeddings
        else:
            self.embeddings = np.vstack([self.embeddings, new_embeddings])
        print(f"Added {len(docs)} docs. Total: {len(self.documents)}")
    
    def search(self, query, top_k=5, threshold=0.3):
        query_embedding = self.model.encode([query])[0]
        
        # Cosine similarity
        similarities = np.dot(self.embeddings, query_embedding) / (
            np.linalg.norm(self.embeddings, axis=1) * np.linalg.norm(query_embedding)
        )
        
        # Sort by similarity
        top_indices = np.argsort(similarities)[::-1][:top_k]
        
        results = []
        for idx in top_indices:
            score = similarities[idx]
            if score >= threshold:
                results.append({
                    "document": self.documents[idx],
                    "score": float(score),
                    "index": int(idx)
                })
        return results

# Usage
engine = SemanticSearchEngine()
engine.add_documents([
    "Python adalah bahasa pemrograman populer untuk data science",
    "Machine learning menggunakan data untuk membuat prediksi",
    "Deep learning adalah subset dari machine learning",
    "NLP memproses bahasa manusia dengan AI",
    "Computer vision mendeteksi objek dalam gambar",
    "Resep nasi goreng dengan bumbu sederhana",
    "Cara membuat kue brownies coklat",
    "Prediksi harga saham menggunakan LSTM",
])

results = engine.search("cara belajar artificial intelligence", top_k=3)
for r in results:
    print(f"[{r['score']:.3f}] {r['document']}")

# =============================================
# Dimensionality Reduction untuk Visualisasi
# =============================================
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import numpy as np

# Embeddings (384 dimensi) → 2D
tsne = TSNE(n_components=2, random_state=42, perplexity=30)
embeddings_2d = tsne.fit_transform(embeddings)

# Visualisasi
fig, ax = plt.subplots(figsize=(10, 8))
categories = ["AI", "AI", "Food", "Food", "AI", "Food", "AI"]
colors = {"AI": "blue", "Food": "red"}

for i, (x, y) in enumerate(embeddings_2d):
    cat = categories[i]
    ax.scatter(x, y, c=colors[cat], s=100)
    ax.annotate(texts[i][:20], (x, y), fontsize=8)

ax.set_title("t-SNE Visualization of Embeddings")
plt.savefig("embeddings_tsne.png", dpi=150)

# ----- UMAP (lebih cepat dan stabil) -----
# pip install umap-learn
import umap

reducer = umap.UMAP(n_components=2, random_state=42)
embeddings_2d = reducer.fit_transform(embeddings)

# ----- PCA (tercepat, linear) -----
from sklearn.decomposition import PCA

pca = PCA(n_components=2)
embeddings_2d_pca = pca.fit_transform(embeddings)
print(f"Explained variance: {pca.explained_variance_ratio_.sum():.2%}")

# =============================================
# Image & Multimodal Embeddings (CLIP)
# =============================================
from sentence_transformers import SentenceTransformer
from PIL import Image
import torch

# CLIP: bisa embed teks DAN gambar dalam ruang yang sama
model = SentenceTransformer("clip-ViT-B-32")

# Embed gambar
image = Image.open("gambar/kucing.jpg")
image_embedding = model.encode(image)

# Embed teks
text_embedding = model.encode(["foto seekor kucing"])

# Similarity antara gambar dan teks!
from sentence_transformers.util import cos_sim
similarity = cos_sim(image_embedding, text_embedding)
print(f"Image-Text similarity: {similarity:.3f}")

# Semantic search gambar
images = [
    Image.open("img/cat.jpg"),
    Image.open("img/dog.jpg"),
    Image.open("img/car.jpg"),
    Image.open("img/laptop.jpg"),
]
image_embeddings = model.encode(images)

query = "hewan peliharaan lucu"
query_embedding = model.encode(query)

similarities = cos_sim(query_embedding, image_embeddings)
print(f"Most similar: image {similarities.argmax()} ({similarities.max():.3f})")

# =============================================
# Evaluasi Embeddings
# =============================================

# ----- 1. Retrieval Metrics -----
def recall_at_k(relevant_docs, retrieved_docs, k):
    retrieved = set(retrieved_docs[:k])
    relevant = set(relevant_docs)
    return len(retrieved & relevant) / len(relevant)

def precision_at_k(relevant_docs, retrieved_docs, k):
    retrieved = set(retrieved_docs[:k])
    relevant = set(relevant_docs)
    return len(retrieved & relevant) / k

def mrr(relevant_docs, retrieved_docs):
    """Mean Reciprocal Rank"""
    for i, doc in enumerate(retrieved_docs):
        if doc in relevant_docs:
            return 1.0 / (i + 1)
    return 0.0

# Evaluasi
queries = {
    "apa itu AI": [0, 1, 3],       # Indices dokumen relevan
    "resep masakan": [5, 6],        # Indices dokumen relevan
}

for query, relevant in queries.items():
    results = engine.search(query, top_k=5)
    retrieved = [r["index"] for r in results]
    r5 = recall_at_k(relevant, retrieved, 5)
    p3 = precision_at_k(relevant, retrieved, 3)
    print(f"Query: '{query}' → R@5={r5:.2f}, P@3={p3:.2f}")

# ----- 2. Benchmark datasets -----
# MTEB (Massive Text Embedding Benchmark)
# https://huggingface.co/spaces/mteb/leaderboard
# Bandingkan model pada berbagai task dan bahasa