NLP dengan Transformers — BERT, GPT, Attention & Hugging Face

# =============================================
# BERT untuk NLP Tasks
# =============================================
from transformers import pipeline

# ----- 1. Masked Language Model -----
fill_mask = pipeline("fill-mask", model="bert-base-uncased")
results = fill_mask("Transformers are [MASK] for NLP tasks.")
for r in results:
    print(f"  {r['token_str']}: {r['score']:.3f}")
# "good": 0.234, "used": 0.189, "great": 0.087, ...

# ----- 2. Sentence Embeddings -----
from transformers import AutoTokenizer, AutoModel
import torch

tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")

def get_embedding(text):
    inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=128)
    with torch.no_grad():
        outputs = model(**inputs)
    # Mean pooling
    embedding = outputs.last_hidden_state.mean(dim=1)
    return embedding[0]

emb1 = get_embedding("Saya suka belajar AI")
emb2 = get_embedding("AI sangat menarik untuk dipelajari")
emb3 = get_embedding("Resep nasi goreng enak")

# Cosine similarity
from torch.nn.functional import cosine_similarity
print(f"AI vs AI: {cosine_similarity(emb1, emb2):.3f}")   # ~0.85
print(f"AI vs Food: {cosine_similarity(emb1, emb3):.3f}") # ~0.15

# =============================================
# Text Generation dengan GPT
# =============================================
from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM

# ----- Menggunakan pipeline -----
generator = pipeline("text-generation", model="gpt2")
result = generator(
    "Artificial Intelligence di Indonesia",
    max_length=100,
    num_return_sequences=2,
    temperature=0.7,
    top_p=0.9,
    do_sample=True
)
for i, r in enumerate(result):
    print(f"Generated {i+1}: {r['generated_text']}")

# ----- GPT dengan Hugging Face -----
tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelForCausalLM.from_pretrained("gpt2")

# Manual generation
input_text = "Machine learning adalah"
input_ids = tokenizer.encode(input_text, return_tensors="pt")

output = model.generate(
    input_ids,
    max_length=150,
    temperature=0.7,
    top_k=50,
    top_p=0.95,
    do_sample=True,
    pad_token_id=tokenizer.eos_token_id
)
generated = tokenizer.decode(output[0], skip_special_tokens=True)
print(generated)

# ----- Menggunakan OpenAI API -----
from openai import OpenAI
client = OpenAI()

response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "system", "content": "Anda adalah penulis teknis."},
        {"role": "user", "content": "Jelaskan Transformers dalam 100 kata."}
    ],
    temperature=0.7,
    max_tokens=200
)
print(response.choices[0].message.content)

# =============================================
# Hugging Face Ecosystem
# =============================================
# pip install transformers datasets evaluate huggingface_hub

# ----- 1. Pipeline (Paling Mudah) -----
from transformers import pipeline

# Sentiment Analysis
classifier = pipeline("sentiment-analysis", model="indobenchmark/indobert-base-p1")
result = classifier("Film ini sangat bagus dan menarik!")
print(result)  # [{'label': 'POSITIVE', 'score': 0.98}]

# Text Summarization
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
summary = summarizer(long_text, max_length=100, min_length=30)

# Translation (Indonesian → English)
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-id-en")
result = translator("Saya suka belajar pemrograman")
print(result)  # [{'translation_text': 'I like learning programming'}]

# Named Entity Recognition
ner = pipeline("ner", model="dslim/bert-base-NER", grouped_entities=True)
entities = ner("Barack Obama lahir di Hawaii pada tahun 1961")
print(entities)

# Question Answering
qa = pipeline("question-answering", model="deepset/roberta-base-squad2")
result = qa(question="Siapa pendiri Tesla?", context="Tesla didirikan oleh Elon Musk...")
print(result)  # {'answer': 'Elon Musk', 'score': 0.95}

# ----- 2. Load Dataset dari Hub -----
from datasets import load_dataset
dataset = load_dataset("imdb", split="train[:1000]")
print(dataset)
print(dataset[0])  # {'text': '...', 'label': 1}

# ----- 3. Push Model ke Hub -----
from huggingface_hub import login
login(token="hf_...")

# Push model dan tokenizer
model.push_to_hub("username/my-indonesian-sentiment-model")
tokenizer.push_to_hub("username/my-indonesian-sentiment-model")

# =============================================
# Fine-tuning BERT untuk Sentiment Analysis
# =============================================
from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer
)
from datasets import load_dataset
import evaluate
import numpy as np

# Load dataset
dataset = load_dataset("imdb")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

def tokenize(batch):
    return tokenizer(batch["text"], padding=True, truncation=True, max_length=256)

tokenized = dataset.map(tokenize, batched=True)
tokenized = tokenized.rename_column("label", "labels")

# Load model
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased", num_labels=2
)

# Metrics
accuracy = evaluate.load("accuracy")
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return accuracy.compute(predictions=preds, references=labels)

# Training
args = TrainingArguments(
    output_dir="./bert-sentiment",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    learning_rate=2e-5,
    weight_decay=0.01,
    eval_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    report_to="none"
)

trainer = Trainer(
    model=model, args=args,
    train_dataset=tokenized["train"].shuffle(seed=42).select(range(5000)),
    eval_dataset=tokenized["test"].shuffle(seed=42).select(range(1000)),
    tokenizer=tokenizer,
    compute_metrics=compute_metrics
)

trainer.train()
results = trainer.evaluate()
print(f"Accuracy: {results['eval_accuracy']:.4f}")

# =============================================
# Deploy NLP Model dengan FastAPI
# =============================================
from fastapi import FastAPI
from pydantic import BaseModel
from transformers import pipeline

app = FastAPI()
classifier = pipeline("sentiment-analysis", model="./bert-sentiment")

class TextInput(BaseModel):
    text: str

@app.post("/predict")
async def predict(input: TextInput):
    result = classifier(input.text)
    return {"sentiment": result[0]["label"], "score": result[0]["score"]}

@app.post("/predict-batch")
async def predict_batch(texts: list[str]):
    results = classifier(texts)
    return {"results": results}

# Hugging Face Inference API (tanpa deploy)
# POST https://api-inference.huggingface.co/models/{model_id}
# Headers: {"Authorization": "Bearer hf_..."}
# Body: {"inputs": "teks Anda di sini"}