Scikit-learn: Library Machine Learning untuk Semua

1. Pengenalan Scikit-learn

Scikit-learn (sklearn) adalah library machine learning paling populer untuk Python. Dibangun di atas NumPy dan SciPy, scikit-learn menyediakan API yang konsisten dan elegan untuk hampir semua algoritma ML klasik — dari regression sederhana sampai ensemble methods yang powerful.

Keunggulan utama scikit-learn: API yang konsisten. Semua model mengikuti pola yang sama — fit(), predict(), transform(). Ini membuat switching antar algoritma menjadi sangat mudah.

Fitur Utama

# Instalasi
pip install scikit-learn numpy pandas matplotlib

# Import dasar
import numpy as np
import pandas as pd
from sklearn import datasets, model_selection, preprocessing, metrics
import matplotlib.pyplot as plt

┌────────────┐  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐
│  Raw Data  │─→│  Preprocess  │─→│  Train Model │─→│  Evaluate    │
│            │  │              │  │              │  │              │
│ • CSV      │  │ • Scaling    │  │ • fit(X, y)  │  │ • Accuracy   │
│ • Database │  │ • Encoding   │  │ • Algoritma   │  │ • Precision  │
│ • API      │  │ • Imputation │  │   ML pilihan │  │ • Recall     │
│ • Scraping │  │ • Features   │  │              │  │ • F1-Score   │
└────────────┘  └──────────────┘  └──────────────┘  └──────┬───────┘
                                                           │
                    ┌──────────────┐  ┌──────────────┐     │
                    │  Deploy      │←─│  Tune &       │←───┘
                    │              │  │  Optimize     │
                    │ • Pickle     │  │ • GridSearch  │
                    │ • API        │  │ • CV          │
                    │ • Monitoring │  │ • Ensemble    │
                    └──────────────┘  └──────────────┘

2. Data Preparation & Feature Engineering

Data preparation sering menghabiskan 70-80% waktu dalam proyek ML. Scikit-learn menyediakan tools yang sangat lengkap untuk preprocessing.

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import (StandardScaler, MinMaxScaler, RobustScaler,
                                     LabelEncoder, OneHotEncoder, OrdinalEncoder)
from sklearn.impute import SimpleImputer, KNNImputer

# Load dataset contoh
from sklearn.datasets import load_iris
iris = load_iris()
X = iris.data
y = iris.target

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
print(f"Train: {X_train.shape}, Test: {X_test.shape}")
print(f"Class distribution train: {np.bincount(y_train)}")
print(f"Class distribution test: {np.bincount(y_test)}")

Feature Scaling

from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

# 1. StandardScaler: mean=0, std=1 (cocok untuk SVM, Logistic Regression)
scaler_standard = StandardScaler()
X_train_scaled = scaler_standard.fit_transform(X_train)
X_test_scaled = scaler_standard.transform(X_test)  # Gunakan transform() saja!
print(f"StandardScaler - Mean: {X_train_scaled.mean(axis=0)[0]:.4f}, Std: {X_train_scaled.std(axis=0)[0]:.4f}")

# 2. MinMaxScaler: range [0, 1] (cocok untuk neural networks)
scaler_minmax = MinMaxScaler()
X_train_mm = scaler_minmax.fit_transform(X_train)

# 3. RobustScaler: tahan terhadap outliers
scaler_robust = RobustScaler()
X_train_robust = scaler_robust.fit_transform(X_train)

# ⚠️ PENTING: Selalu fit di training data, transform di test data!
# JANGAN fit_transform di test data (data leakage!)

Encoding Kategorikal

import pandas as pd
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, OrdinalEncoder

# Data kategorikal
data = pd.DataFrame({
    'kota': ['Jakarta', 'Bandung', 'Surabaya', 'Jakarta', 'Bandung'],
    'pendidikan': ['S1', 'S2', 'SMA', 'S3', 'S1'],
    'gender': ['M', 'F', 'M', 'F', 'M'],
})

# 1. Label Encoding (untuk ordinal data)
ordinal = OrdinalEncoder(categories=[['SMA', 'S1', 'S2', 'S3']])
data['pendidikan_enc'] = ordinal.fit_transform(data[['pendidikan']])
print("Ordinal Encoding:")
print(data[['pendidikan', 'pendidikan_enc']])

# 2. One-Hot Encoding (untuk nominal data)
ohe = OneHotEncoder(sparse_output=False, drop='first')
kota_encoded = ohe.fit_transform(data[['kota']])
print(f"\nOne-Hot Encoding (kota): {kota_encoded}")

# 3. Pandas get_dummies (shortcut)
data_dummies = pd.get_dummies(data, columns=['kota', 'gender'], drop_first=True)
print(f"\nDummies:\n{data_dummies}")

Handling Missing Values

import numpy as np
from sklearn.impute import SimpleImputer, KNNImputer

# Data dengan missing values
X_missing = np.array([[1, 2, np.nan],
                       [4, np.nan, 6],
                       [7, 8, 9],
                       [np.nan, 11, 12]])

# 1. Simple Imputer (mean, median, most_frequent, constant)
imputer_mean = SimpleImputer(strategy='mean')
X_filled = imputer_mean.fit_transform(X_missing)
print("SimpleImputer (mean):")
print(X_filled)

# 2. KNN Imputer (lebih canggih, berdasarkan tetangga)
imputer_knn = KNNImputer(n_neighbors=2)
X_knn = imputer_knn.fit_transform(X_missing)
print(f"\nKNN Imputer:")
print(X_knn)

3. Classification

Classification adalah tugas memprediksi kategori/label dari data. Scikit-learn menyediakan berbagai algoritma classification yang bisa kamu coba.

import numpy as np
from sklearn.datasets import load_iris, load_wine
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import (accuracy_score, classification_report,
                               confusion_matrix, ConfusionMatrixDisplay)
import matplotlib.pyplot as plt

# Load dataset
X, y = load_wine(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Scaling
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# Bandingkan beberapa algoritma
models = {
    'Logistic Regression': LogisticRegression(max_iter=1000),
    'Decision Tree': DecisionTreeClassifier(max_depth=5, random_state=42),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'SVM': SVC(kernel='rbf', random_state=42),
    'KNN': KNeighborsClassifier(n_neighbors=5),
    'Naive Bayes': GaussianNB(),
    'Gradient Boosting': GradientBoostingClassifier(random_state=42),
}

print("Perbandingan Algoritma Classification:")
print(f"{'Model':<25} {'CV Mean':>8} {'CV Std':>8} {'Test Acc':>10}")
print("-" * 55)

best_score = 0
best_model_name = ""

for name, model in models.items():
    # Cross-validation pada training data
    cv_scores = cross_val_score(model, X_train_s, y_train, cv=5, scoring='accuracy')
    # Fit dan evaluasi di test data
    model.fit(X_train_s, y_train)
    test_acc = accuracy_score(y_test, model.predict(X_test_s))

    print(f"{name:<25} {cv_scores.mean():>8.4f} {cv_scores.std():>8.4f} {test_acc:>10.4f}")

    if cv_scores.mean() > best_score:
        best_score = cv_scores.mean()
        best_model_name = name

print(f"\n🏆 Terbaik: {best_model_name} (CV: {best_score:.4f})")

Detail Evaluasi Model

from sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay

# Ambil model terbaik dan evaluasi detail
best_model = models[best_model_name]
best_model.fit(X_train_s, y_train)
y_pred = best_model.predict(X_test_s)

# Classification Report
print("Classification Report:")
print(classification_report(y_test, y_pred, target_names=load_wine().target_names))

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
print(f"Confusion Matrix:\n{cm}")

# Visualisasi
fig, ax = plt.subplots(figsize=(8, 6))
ConfusionMatrixDisplay.from_predictions(y_test, y_pred,
    display_labels=load_wine().target_names,
    cmap='Blues', ax=ax)
ax.set_title(f'Confusion Matrix - {best_model_name}')
plt.tight_layout()
plt.savefig('confusion_matrix.png', dpi=150)
plt.show()

4. Regression

Regression memprediksi nilai kontinu (angka). Dari prediksi harga rumah, suhu, sampai pendapatan — semuanya regression.

import numpy as np
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
import matplotlib.pyplot as plt

# Load California Housing dataset
data = fetch_california_housing()
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# Model regression
models = {
    'Linear Regression': LinearRegression(),
    'Ridge (α=1)': Ridge(alpha=1.0),
    'Lasso (α=0.1)': Lasso(alpha=0.1),
    'ElasticNet': ElasticNet(alpha=0.1, l1_ratio=0.5),
    'Decision Tree': DecisionTreeRegressor(max_depth=10, random_state=42),
    'Random Forest': RandomForestRegressor(n_estimators=100, random_state=42),
    'Gradient Boosting': GradientBoostingRegressor(n_estimators=200, random_state=42),
}

print("Perbandingan Model Regression:")
print(f"{'Model':<25} {'RMSE':>8} {'MAE':>8} {'R²':>8}")
print("-" * 55)

for name, model in models.items():
    model.fit(X_train_s, y_train)
    y_pred = model.predict(X_test_s)

    rmse = np.sqrt(mean_squared_error(y_test, y_pred))
    mae = mean_absolute_error(y_test, y_pred)
    r2 = r2_score(y_test, y_pred)

    print(f"{name:<25} {rmse:>8.4f} {mae:>8.4f} {r2:>8.4f}")

# Visualisasi: Predicted vs Actual
best_model = GradientBoostingRegressor(n_estimators=200, random_state=42)
best_model.fit(X_train_s, y_train)
y_pred = best_model.predict(X_test_s)

plt.figure(figsize=(8, 8))
plt.scatter(y_test, y_pred, alpha=0.3, s=10)
plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', linewidth=2)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.title('Predicted vs Actual - Gradient Boosting')
plt.grid(True, alpha=0.3)
plt.savefig('regression_pred_vs_actual.png', dpi=150)
plt.show()

Feature Importance

# Feature importance dari Random Forest
rf = RandomForestRegressor(n_estimators=100, random_state=42)
rf.fit(X_train_s, y_train)

importances = rf.feature_importances_
feature_names = data.feature_names

# Sort
indices = np.argsort(importances)[::-1]

plt.figure(figsize=(10, 6))
plt.barh(range(len(importances)), importances[indices])
plt.yticks(range(len(importances)), [feature_names[i] for i in indices])
plt.xlabel('Feature Importance')
plt.title('Feature Importance - Random Forest')
plt.tight_layout()
plt.savefig('feature_importance.png', dpi=150)
plt.show()

for i in indices:
    print(f"  {feature_names[i]}: {importances[i]:.4f}")

5. Clustering

Clustering adalah unsupervised learning — mengelompokkan data tanpa label. Berguna untuk customer segmentation, anomaly detection, dan pattern discovery.

import numpy as np
from sklearn.datasets import make_blobs, make_moons
from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import silhouette_score, adjusted_rand_score
import matplotlib.pyplot as plt

# Generate data
X_blobs, y_blobs = make_blobs(n_samples=300, centers=4, cluster_std=0.8, random_state=42)
X_moons, y_moons = make_moons(n_samples=300, noise=0.1, random_state=42)

# ========== K-Means ==========
# Elbow method untuk menentukan K optimal
inertias = []
K_range = range(2, 10)
for k in K_range:
    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
    kmeans.fit(X_blobs)
    inertias.append(kmeans.inertia_)

plt.figure(figsize=(8, 5))
plt.plot(K_range, inertias, 'bo-')
plt.xlabel('K')
plt.ylabel('Inertia')
plt.title('Elbow Method - K-Means')
plt.grid(True, alpha=0.3)
plt.savefig('elbow.png', dpi=150)
plt.show()

# K-Means dengan K=4
kmeans = KMeans(n_clusters=4, random_state=42, n_init=10)
labels_km = kmeans.fit_predict(X_blobs)
sil_km = silhouette_score(X_blobs, labels_km)
print(f"K-Means Silhouette Score: {sil_km:.4f}")
print(f"K-Means Centers:\n{kmeans.cluster_centers_}")

# ========== DBSCAN ==========
# DBSCAN: density-based, bagus untuk cluster bentuk aneh
scaler = StandardScaler()
X_moons_s = scaler.fit_transform(X_moons)

dbscan = DBSCAN(eps=0.3, min_samples=5)
labels_db = dbscan.fit_predict(X_moons_s)

n_clusters = len(set(labels_db)) - (1 if -1 in labels_db else 0)
n_noise = list(labels_db).count(-1)
print(f"\nDBSCAN: {n_clusters} clusters, {n_noise} noise points")

if n_clusters > 1:
    sil_db = silhouette_score(X_moons_s, labels_db)
    print(f"DBSCAN Silhouette Score: {sil_db:.4f}")

# Visualisasi
fig, axes = plt.subplots(1, 3, figsize=(18, 5))

axes[0].scatter(X_blobs[:, 0], X_blobs[:, 1], c=labels_km, cmap='viridis', s=30)
axes[0].scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],
                c='red', marker='X', s=200, edgecolors='black')
axes[0].set_title(f'K-Means (Silhouette: {sil_km:.3f})')

axes[1].scatter(X_moons[:, 0], X_moons[:, 1], c=labels_db, cmap='Set1', s=30)
axes[1].set_title(f'DBSCAN ({n_clusters} clusters)')

# Agglomerative
agg = AgglomerativeClustering(n_clusters=2)
labels_agg = agg.fit_predict(X_moons)
axes[2].scatter(X_moons[:, 0], X_moons[:, 1], c=labels_agg, cmap='Set1', s=30)
axes[2].set_title('Agglomerative Clustering')

plt.tight_layout()
plt.savefig('clustering.png', dpi=150)
plt.show()

6. Model Evaluation

from sklearn.model_selection import (cross_val_score, KFold, StratifiedKFold,
                                       cross_validate, learning_curve)
from sklearn.metrics import (accuracy_score, precision_score, recall_score,
                               f1_score, roc_auc_score, roc_curve)
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
import matplotlib.pyplot as plt
import numpy as np

# Load binary classification dataset
X, y = load_breast_cancer(return_X_y=True)

# ========== Cross-Validation ==========
model = RandomForestClassifier(n_estimators=100, random_state=42)

# Stratified K-Fold (mempertahankan proporsi kelas)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
cv_results = cross_validate(model, X, y, cv=skf,
    scoring=['accuracy', 'precision', 'recall', 'f1', 'roc_auc'],
    return_train_score=True)

print("Cross-Validation Results (5-Fold):")
print(f"  Accuracy:  {cv_results['test_accuracy'].mean():.4f} ± {cv_results['test_accuracy'].std():.4f}")
print(f"  Precision: {cv_results['test_precision'].mean():.4f} ± {cv_results['test_precision'].std():.4f}")
print(f"  Recall:    {cv_results['test_recall'].mean():.4f} ± {cv_results['test_recall'].std():.4f}")
print(f"  F1-Score:  {cv_results['test_f1'].mean():.4f} ± {cv_results['test_f1'].std():.4f}")
print(f"  ROC-AUC:   {cv_results['test_roc_auc'].mean():.4f} ± {cv_results['test_roc_auc'].std():.4f}")

# ========== ROC Curve ==========
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

model.fit(X_train, y_train)
y_prob = model.predict_proba(X_test)[:, 1]
fpr, tpr, thresholds = roc_curve(y_test, y_prob)
auc = roc_auc_score(y_test, y_prob)

plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, 'b-', linewidth=2, label=f'ROC Curve (AUC = {auc:.4f})')
plt.plot([0, 1], [0, 1], 'r--', label='Random')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.grid(True, alpha=0.3)
plt.savefig('roc_curve.png', dpi=150)
plt.show()

# ========== Learning Curve ==========
train_sizes, train_scores, val_scores = learning_curve(
    model, X, y, cv=5, train_sizes=np.linspace(0.1, 1.0, 10),
    scoring='accuracy', random_state=42
)

plt.figure(figsize=(10, 6))
plt.plot(train_sizes, train_scores.mean(axis=1), 'o-', label='Training Score')
plt.plot(train_sizes, val_scores.mean(axis=1), 'o-', label='Validation Score')
plt.fill_between(train_sizes,
    train_scores.mean(axis=1) - train_scores.std(axis=1),
    train_scores.mean(axis=1) + train_scores.std(axis=1), alpha=0.1)
plt.fill_between(train_sizes,
    val_scores.mean(axis=1) - val_scores.std(axis=1),
    val_scores.mean(axis=1) + val_scores.std(axis=1), alpha=0.1)
plt.xlabel('Training Set Size')
plt.ylabel('Accuracy')
plt.title('Learning Curve')
plt.legend()
plt.grid(True, alpha=0.3)
plt.savefig('learning_curve.png', dpi=150)
plt.show()

7. Hyperparameter Tuning

from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_wine
from sklearn.model_selection import train_test_split
import numpy as np

X, y = load_wine(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# ========== Grid Search ==========
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [3, 5, 10, None],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4],
}

grid_search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1,        # Gunakan semua CPU cores
    verbose=1
)
grid_search.fit(X_train, y_train)

print(f"Best Parameters: {grid_search.best_params_}")
print(f"Best CV Score: {grid_search.best_score_:.4f}")
print(f"Test Score: {grid_search.score(X_test, y_test):.4f}")

# ========== Randomized Search (lebih cepat untuk param space besar) ==========
from scipy.stats import randint, uniform

param_distributions = {
    'n_estimators': randint(50, 500),
    'max_depth': [3, 5, 10, 15, 20, None],
    'min_samples_split': randint(2, 20),
    'min_samples_leaf': randint(1, 10),
    'max_features': ['sqrt', 'log2', None],
}

random_search = RandomizedSearchCV(
    RandomForestClassifier(random_state=42),
    param_distributions,
    n_iter=50,         # Coba 50 kombinasi random
    cv=5,
    scoring='accuracy',
    n_jobs=-1,
    random_state=42,
    verbose=1
)
random_search.fit(X_train, y_train)

print(f"\nRandom Search Best: {random_search.best_params_}")
print(f"Test Score: {random_search.score(X_test, y_test):.4f}")

8. Pipelines

Pipeline menggabungkan preprocessing dan model menjadi satu objek. Ini mencegah data leakage dan membuat kode lebih rapi.

import numpy as np
import pandas as pd
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

# Contoh data dengan tipe mixed
data = pd.DataFrame({
    'umur': [25, 30, np.nan, 45, 35, 50, 28, np.nan],
    'gaji': [5000, 8000, 12000, 15000, 7000, 20000, 6000, 9000],
    'kota': ['Jakarta', 'Bandung', 'Jakarta', 'Surabaya', 'Bandung', 'Jakarta', 'Surabaya', 'Bandung'],
    'pendidikan': ['S1', 'S2', 'S1', 'S3', 'S1', 'S2', 'SMA', 'S1'],
    'beli': [0, 1, 1, 1, 0, 1, 0, 1],
})

X = data.drop('beli', axis=1)
y = data['beli']

# Definisikan kolom berdasarkan tipe
numerik_features = ['umur', 'gaji']
kategorik_features = ['kota', 'pendidikan']

# Pipeline untuk numerik
numerik_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler()),
])

# Pipeline untuk kategorik
kategorik_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('encoder', OneHotEncoder(drop='first', sparse_output=False)),
])

# Gabungkan dengan ColumnTransformer
preprocessor = ColumnTransformer([
    ('num', numerik_pipeline, numerik_features),
    ('cat', kategorik_pipeline, kategorik_features),
])

# Full pipeline: preprocessing + model
full_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(n_estimators=100, random_state=42)),
])

# Fit dan evaluasi (hanya satu baris!)
scores = cross_val_score(full_pipeline, X, y, cv=3, scoring='accuracy')
print(f"Pipeline CV Score: {scores.mean():.4f} ± {scores.std():.4f}")

# Fit di seluruh data
full_pipeline.fit(X, y)

# Predict data baru
new_data = pd.DataFrame({
    'umur': [32],
    'gaji': [10000],
    'kota': ['Jakarta'],
    'pendidikan': ['S2'],
})
pred = full_pipeline.predict(new_data)
print(f"Prediksi untuk data baru: {'Beli' if pred[0] == 1 else 'Tidak Beli'}")

9. Feature Selection

import numpy as np
from sklearn.datasets import load_wine
from sklearn.feature_selection import SelectKBest, f_classif, RFE, SequentialFeatureSelector
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.preprocessing import StandardScaler

X, y = load_wine(return_X_y=True)
feature_names = load_wine().feature_names
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# 1. SelectKBest (statistical test)
selector = SelectKBest(f_classif, k=5)
X_train_selected = selector.fit_transform(X_train_s, y_train)
selected_mask = selector.get_support()
print("SelectKBest - Top 5 features:")
for name, score in sorted(zip(feature_names, selector.scores_), key=lambda x: -x[1])[:5]:
    print(f"  {name}: {score:.2f}")

# 2. RFE (Recursive Feature Elimination)
rfe = RFE(RandomForestClassifier(n_estimators=50, random_state=42), n_features_to_select=5)
rfe.fit(X_train_s, y_train)
print(f"\nRFE - Selected features:")
for name, rank, selected in zip(feature_names, rfe.ranking_, rfe.support_):
    if selected:
        print(f"  {name} (rank: {rank})")

# 3. Feature Importance from tree-based model
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train_s, y_train)
importances = rf.feature_importances_
print(f"\nTop 5 by Random Forest Importance:")
for idx in np.argsort(importances)[::-1][:5]:
    print(f"  {feature_names[idx]}: {importances[idx]:.4f}")

# Bandingkan performa: semua features vs selected
scores_all = cross_val_score(RandomForestClassifier(random_state=42), X_train_s, y_train, cv=5)
X_train_rfe = rfe.transform(X_train_s)
scores_selected = cross_val_score(RandomForestClassifier(random_state=42), X_train_rfe, y_train, cv=5)

print(f"\nSemua features ({X_train_s.shape[1]}): {scores_all.mean():.4f}")
print(f"Selected features ({X_train_rfe.shape[1]}): {scores_selected.mean():.4f}")

10. Ensemble Methods

from sklearn.ensemble import (RandomForestClassifier, GradientBoostingClassifier,
                                AdaBoostClassifier, VotingClassifier, StackingClassifier)
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import load_wine
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.preprocessing import StandardScaler

X, y = load_wine(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# Individual models
lr = LogisticRegression(max_iter=1000, random_state=42)
svm = SVC(probability=True, random_state=42)
dt = DecisionTreeClassifier(max_depth=5, random_state=42)

# 1. Voting Classifier (majority vote)
voting = VotingClassifier(
    estimators=[('lr', lr), ('svm', svm), ('dt', dt)],
    voting='soft'  # soft = probability averaging
)

# 2. Stacking Classifier (meta-learner)
stacking = StackingClassifier(
    estimators=[('lr', lr), ('svm', svm), ('dt', dt)],
    final_estimator=LogisticRegression(max_iter=1000),
    cv=5
)

# 3. AdaBoost
adaboost = AdaBoostClassifier(
    estimator=DecisionTreeClassifier(max_depth=1),
    n_estimators=100,
    random_state=42
)

# 4. Gradient Boosting
gb = GradientBoostingClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    random_state=42
)

models = {
    'Logistic Regression': lr,
    'SVM': svm,
    'Decision Tree': dt,
    'Random Forest': RandomForestClassifier(100, random_state=42),
    'Voting': voting,
    'Stacking': stacking,
    'AdaBoost': adaboost,
    'Gradient Boosting': gb,
}

print("Perbandingan Ensemble Methods:")
for name, model in models.items():
    scores = cross_val_score(model, X_train_s, y_train, cv=5, scoring='accuracy')
    print(f"  {name:<25} {scores.mean():.4f} ± {scores.std():.4f}")

11. Quiz Pemahaman