NLP dengan Transformers — BERT, GPT, Attention & Hugging Face

# =============================================
# Tokenization dengan Hugging Face
# =============================================
from transformers import AutoTokenizer

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# Tokenisasi teks
text = "Transformers merevolusi NLP dengan attention mechanism!"
tokens = tokenizer(text)
print(f"Input IDs: {tokens['input_ids']}")
print(f"Attention Mask: {tokens['attention_mask']}")

# Decode kembali ke teks
decoded = tokenizer.decode(tokens['input_ids'])
print(f"Decoded: {decoded}")

# Lihat token individual
tokens_list = tokenizer.tokenize(text)
print(f"Tokens: {tokens_list}")
# ['transform', '##ers', 'mer', '##evo', '##lusi', 'nlp', 'dengan', 
#  'attention', 'mechanism', '!']

# BERT menggunakan WordPiece: kata umum = 1 token, kata jarang dipecah
# "merevolusi" → ["mer", "##evo", "##lusi"] (subword pieces)

# Batch tokenization
texts = ["Halo dunia", "NLP sangat menarik", "Belajar AI"]
batch = tokenizer(texts, padding=True, truncation=True, max_length=128,
                  return_tensors="pt")
print(f"Batch shape: {batch['input_ids'].shape}")

# Tokenizer untuk GPT (BPE - Byte Pair Encoding)
gpt_tokenizer = AutoTokenizer.from_pretrained("gpt2")
gpt_tokens = gpt_tokenizer.tokenize(text)
print(f"GPT tokens: {gpt_tokens}")

# =============================================
# Visualisasi Attention Weights
# =============================================
from transformers import AutoTokenizer, AutoModel
import torch
import numpy as np

model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name, output_attentions=True)

text = "The cat sat on the mat"
inputs = tokenizer(text, return_tensors="pt")
tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])

with torch.no_grad():
    outputs = model(**inputs)

# Ambil attention dari layer terakhir, head pertama
attention = outputs.attentions[-1][0, 0].numpy()
# Shape: (seq_len, seq_len)

import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(8, 6))
im = ax.imshow(attention, cmap="Blues")
ax.set_xticks(range(len(tokens)))
ax.set_yticks(range(len(tokens)))
ax.set_xticklabels(tokens, rotation=45)
ax.set_yticklabels(tokens)
ax.set_title("Self-Attention Weights (Layer 12, Head 1)")
plt.colorbar(im)
plt.tight_layout()
plt.savefig("attention_heatmap.png", dpi=150)
print("Attention heatmap disimpan!")

# Multi-Head Attention
print(f"Jumlah layers: {len(outputs.attentions)}")  # 12
print(f"Jumlah heads per layer: {outputs.attentions[0].shape[1]}")  # 12
print(f"Sequence length: {outputs.attentions[0].shape[2]}")

# =============================================
# BERT untuk NLP Tasks
# =============================================
from transformers import pipeline

# ----- 1. Masked Language Model -----
fill_mask = pipeline("fill-mask", model="bert-base-uncased")
results = fill_mask("Transformers are [MASK] for NLP tasks.")
for r in results:
    print(f"  {r['token_str']}: {r['score']:.3f}")
# "good": 0.234, "used": 0.189, "great": 0.087, ...

# ----- 2. Sentence Embeddings -----
from transformers import AutoTokenizer, AutoModel
import torch

tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")

def get_embedding(text):
    inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=128)
    with torch.no_grad():
        outputs = model(**inputs)
    # Mean pooling
    embedding = outputs.last_hidden_state.mean(dim=1)
    return embedding[0]

emb1 = get_embedding("Saya suka belajar AI")
emb2 = get_embedding("AI sangat menarik untuk dipelajari")
emb3 = get_embedding("Resep nasi goreng enak")

# Cosine similarity
from torch.nn.functional import cosine_similarity
print(f"AI vs AI: {cosine_similarity(emb1, emb2):.3f}")   # ~0.85
print(f"AI vs Food: {cosine_similarity(emb1, emb3):.3f}") # ~0.15

# =============================================
# Text Generation dengan GPT
# =============================================
from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM

# ----- Menggunakan pipeline -----
generator = pipeline("text-generation", model="gpt2")
result = generator(
    "Artificial Intelligence di Indonesia",
    max_length=100,
    num_return_sequences=2,
    temperature=0.7,
    top_p=0.9,
    do_sample=True
)
for i, r in enumerate(result):
    print(f"Generated {i+1}: {r['generated_text']}")

# ----- GPT dengan Hugging Face -----
tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelForCausalLM.from_pretrained("gpt2")

# Manual generation
input_text = "Machine learning adalah"
input_ids = tokenizer.encode(input_text, return_tensors="pt")

output = model.generate(
    input_ids,
    max_length=150,
    temperature=0.7,
    top_k=50,
    top_p=0.95,
    do_sample=True,
    pad_token_id=tokenizer.eos_token_id
)
generated = tokenizer.decode(output[0], skip_special_tokens=True)
print(generated)

# ----- Menggunakan OpenAI API -----
from openai import OpenAI
client = OpenAI()

response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "system", "content": "Anda adalah penulis teknis."},
        {"role": "user", "content": "Jelaskan Transformers dalam 100 kata."}
    ],
    temperature=0.7,
    max_tokens=200
)
print(response.choices[0].message.content)

# =============================================
# Hugging Face Ecosystem
# =============================================
# pip install transformers datasets evaluate huggingface_hub

# ----- 1. Pipeline (Paling Mudah) -----
from transformers import pipeline

# Sentiment Analysis
classifier = pipeline("sentiment-analysis", model="indobenchmark/indobert-base-p1")
result = classifier("Film ini sangat bagus dan menarik!")
print(result)  # [{'label': 'POSITIVE', 'score': 0.98}]

# Text Summarization
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
summary = summarizer(long_text, max_length=100, min_length=30)

# Translation (Indonesian → English)
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-id-en")
result = translator("Saya suka belajar pemrograman")
print(result)  # [{'translation_text': 'I like learning programming'}]

# Named Entity Recognition
ner = pipeline("ner", model="dslim/bert-base-NER", grouped_entities=True)
entities = ner("Barack Obama lahir di Hawaii pada tahun 1961")
print(entities)

# Question Answering
qa = pipeline("question-answering", model="deepset/roberta-base-squad2")
result = qa(question="Siapa pendiri Tesla?", context="Tesla didirikan oleh Elon Musk...")
print(result)  # {'answer': 'Elon Musk', 'score': 0.95}

# ----- 2. Load Dataset dari Hub -----
from datasets import load_dataset
dataset = load_dataset("imdb", split="train[:1000]")
print(dataset)
print(dataset[0])  # {'text': '...', 'label': 1}

# ----- 3. Push Model ke Hub -----
from huggingface_hub import login
login(token="hf_...")

# Push model dan tokenizer
model.push_to_hub("username/my-indonesian-sentiment-model")
tokenizer.push_to_hub("username/my-indonesian-sentiment-model")

# =============================================
# Fine-tuning BERT untuk Sentiment Analysis
# =============================================
from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer
)
from datasets import load_dataset
import evaluate
import numpy as np

# Load dataset
dataset = load_dataset("imdb")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

def tokenize(batch):
    return tokenizer(batch["text"], padding=True, truncation=True, max_length=256)

tokenized = dataset.map(tokenize, batched=True)
tokenized = tokenized.rename_column("label", "labels")

# Load model
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased", num_labels=2
)

# Metrics
accuracy = evaluate.load("accuracy")
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return accuracy.compute(predictions=preds, references=labels)

# Training
args = TrainingArguments(
    output_dir="./bert-sentiment",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    learning_rate=2e-5,
    weight_decay=0.01,
    eval_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    report_to="none"
)

trainer = Trainer(
    model=model, args=args,
    train_dataset=tokenized["train"].shuffle(seed=42).select(range(5000)),
    eval_dataset=tokenized["test"].shuffle(seed=42).select(range(1000)),
    tokenizer=tokenizer,
    compute_metrics=compute_metrics
)

trainer.train()
results = trainer.evaluate()
print(f"Accuracy: {results['eval_accuracy']:.4f}")

# =============================================
# Deploy NLP Model dengan FastAPI
# =============================================
from fastapi import FastAPI
from pydantic import BaseModel
from transformers import pipeline

app = FastAPI()
classifier = pipeline("sentiment-analysis", model="./bert-sentiment")

class TextInput(BaseModel):
    text: str

@app.post("/predict")
async def predict(input: TextInput):
    result = classifier(input.text)
    return {"sentiment": result[0]["label"], "score": result[0]["score"]}

@app.post("/predict-batch")
async def predict_batch(texts: list[str]):
    results = classifier(texts)
    return {"results": results}

# Hugging Face Inference API (tanpa deploy)
# POST https://api-inference.huggingface.co/models/{model_id}
# Headers: {"Authorization": "Bearer hf_..."}
# Body: {"inputs": "teks Anda di sini"}