Random Forest: Ensemble Learning

import numpy as np

def gini_impurity(labels):
    """Hitung Gini Impurity dari sebuah node."""
    _, counts = np.unique(labels, return_counts=True)
    probabilities = counts / counts.sum()
    return 1 - np.sum(probabilities ** 2)

# Contoh: 10 data — 7 positif, 3 negatif
labels_node_a = np.array([1,1,1,1,1,1,1,0,0,0])
print(f"Gini Node A (7+, 3-): {gini_impurity(labels_node_a):.4f}")
# Output: Gini Node A (7+, 3-): 0.4200

# Contoh: node murni (semua sama)
labels_pure = np.array([1,1,1,1,1])
print(f"Gini Node Pure (5+, 0-): {gini_impurity(labels_pure):.4f}")
# Output: Gini Node Pure (5+, 0-): 0.0000

# Contoh: node campuran (50-50 — paling impure)
labels_mixed = np.array([1,1,1,0,0,0])
print(f"Gini Node Mixed (3+, 3-): {gini_impurity(labels_mixed):.4f}")
# Output: Gini Node Mixed (3+, 3-): 0.5000

import numpy as np

# Dataset asli: 8 data points
dataset = np.array([0, 1, 2, 3, 4, 5, 6, 7])
N = len(dataset)
print(f"Dataset asli: {dataset}")
print(f"Jumlah data: {N}")

np.random.seed(42)

# Bootstrap sample: sampling dengan replacement
bootstrap_indices = np.random.choice(N, size=N, replace=True)
bootstrap_sample = dataset[bootstrap_indices]

print(f"\nBootstrap sample: {bootstrap_sample}")
print(f"Indices yang dipilih: {bootstrap_indices}")

# Identifikasi OOB (Out-of-Bag) data
oob_mask = np.ones(N, dtype=bool)
oob_mask[bootstrap_indices] = False
oob_data = dataset[oob_mask]

print(f"\nOOB data (tidak terpilih): {oob_data}")
print(f"Jumlah OOB: {len(oob_data)} ({len(oob_data)/N*100:.1f}%)")
print(f"Data terpilih: {N - len(oob_data)} ({(N-len(oob_data))/N*100:.1f}%)")

# Demo: bandingkan beberapa bootstrap sample
print("\n--- 5 Bootstrap Sample Berbeda ---")
for i in range(5):
    idx = np.random.choice(N, size=N, replace=True)
    sample = dataset[idx]
    unique = np.unique(sample)
    print(f"Sample {i+1}: {sample} → {len(unique)} data unik dari {N}")

from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
from sklearn.metrics import accuracy_score, classification_report
from sklearn.model_selection import train_test_split

# Generate dataset
X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=15,
    n_redundant=3,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Random Forest DENGAN OOB scoring
rf_with_oob = RandomForestClassifier(
    n_estimators=100,
    oob_score=True,           # Aktifkan OOB scoring
    random_state=42,
    n_jobs=-1
)
rf_with_oob.fit(X_train, y_train)

print("=== OOB Evaluation ===")
print(f"OOB Score: {rf_with_oob.oob_score_:.4f}")

# Bandingkan dengan test set
y_pred = rf_with_oob.predict(X_test)
test_accuracy = accuracy_score(y_test, y_pred)
print(f"Test Score: {test_accuracy:.4f}")
print(f"\nSelisih: {abs(rf_with_oob.oob_score_ - test_accuracy):.4f}")
# Selisih biasanya kecil — OOB score ≈ cross-validation score!

import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split

# Dataset: two moons (non-linear boundary)
X, y = make_moons(n_samples=500, noise=0.3, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Random Forest dengan 5 trees saja (untuk visualisasi)
rf_small = RandomForestClassifier(n_estimators=5, random_state=42)
rf_small.fit(X_train, y_train)

fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# Plot 5 individual trees
for i, tree in enumerate(rf_small.estimators_):
    ax = axes[i // 3, i % 3]
    xx, yy = np.meshgrid(
        np.linspace(X[:, 0].min()-0.5, X[:, 0].max()+0.5, 200),
        np.linspace(X[:, 1].min()-0.5, X[:, 1].max()+0.5, 200)
    )
    Z = tree.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    ax.contourf(xx, yy, Z, alpha=0.3, cmap='RdYlBu')
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap='RdYlBu', 
               edgecolors='k', s=20)
    ax.set_title(f'Tree #{i+1} (acc: {tree.score(X_test, y_test):.2f})')

# Plot final ensemble prediction
ax = axes[1, 2]
Z_rf = rf_small.predict(np.c_[xx.ravel(), yy.ravel()])
Z_rf = Z_rf.reshape(xx.shape)
ax.contourf(xx, yy, Z_rf, alpha=0.3, cmap='RdYlBu')
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap='RdYlBu',
           edgecolors='k', s=20)
ax.set_title(f'Ensemble (acc: {rf_small.score(X_test, y_test):.2f})')

plt.suptitle('Random Forest: 5 Trees → 1 Ensemble', fontsize=14)
plt.tight_layout()
plt.show()

import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split

# Load dataset breast cancer
data = load_breast_cancer()
X, y = data.data, data.target
feature_names = data.feature_names

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Train Random Forest
rf = RandomForestClassifier(
    n_estimators=500,
    random_state=42,
    n_jobs=-1
)
rf.fit(X_train, y_train)

print(f"Test Accuracy: {rf.score(X_test, y_test):.4f}")

# === 1. MDI Feature Importance (Gini Importance) ===
importances_mdi = rf.feature_importances_
indices_mdi = np.argsort(importances_mdi)[::-1]

print("\n=== MDI Feature Importance (Top 10) ===")
for i in range(10):
    print(f"  {i+1}. {feature_names[indices_mdi[i]]}: {importances_mdi[indices_mdi[i]]:.4f}")

# Visualisasi
fig, axes = plt.subplots(1, 2, figsize=(16, 6))

# MDI Importance
axes[0].barh(range(10), importances_mdi[indices_mdi[:10]][::-1])
axes[0].set_yticks(range(10))
axes[0].set_yticklabels([feature_names[i] for i in indices_mdi[:10]][::-1])
axes[0].set_title('MDI Feature Importance (Gini)')
axes[0].set_xlabel('Importance')

# === 2. Permutation Importance ===
from sklearn.inspection import permutation_importance

perm_importance = permutation_importance(
    rf, X_test, y_test, n_repeats=10, random_state=42, n_jobs=-1
)
indices_perm = np.argsort(perm_importance.importances_mean)[::-1]

print("\n=== Permutation Feature Importance (Top 10) ===")
for i in range(10):
    print(f"  {i+1}. {feature_names[indices_perm[i]]}: "
          f"{perm_importance.importances_mean[indices_perm[i]]:.4f} "
          f"± {perm_importance.importances_std[indices_perm[i]]:.4f}")

axes[1].barh(range(10), perm_importance.importances_mean[indices_perm[:10]][::-1])
axes[1].set_yticks(range(10))
axes[1].set_yticklabels([feature_names[i] for i in indices_perm[:10]][::-1])
axes[1].set_title('Permutation Feature Importance')
axes[1].set_xlabel('Mean Accuracy Decrease')

plt.tight_layout()
plt.show()

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import (
    train_test_split, cross_val_score, 
    StratifiedKFold, learning_curve
)
from sklearn.metrics import (
    accuracy_score, precision_score, recall_score,
    f1_score, roc_auc_score, confusion_matrix,
    classification_report, roc_curve
)
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
import warnings
warnings.filterwarnings('ignore')

# =============================================
# 1. LOAD & PERSIAPAN DATA
# =============================================
data = load_breast_cancer()
X = pd.DataFrame(data.data, columns=data.feature_names)
y = pd.Series(data.target, name='target')

print("=" * 60)
print("DATASET BREAST CANCER")
print("=" * 60)
print(f"Jumlah samples: {X.shape[0]}")
print(f"Jumlah fitur: {X.shape[1]}")
print(f"Kelas: {dict(zip(data.target_names, np.bincount(y)))}")
print(f"\n5 baris pertama:")
print(X.head())

# Train-test split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
print(f"\nTrain size: {X_train.shape[0]}")
print(f"Test size: {X_test.shape[0]}")

# =============================================
# 2. TRAINING RANDOM FOREST
# =============================================
rf_model = RandomForestClassifier(
    n_estimators=300,        # Jumlah tree
    max_features='sqrt',     # Fitur per split (sqrt untuk klasifikasi)
    max_depth=None,          # Tanpa batas kedalaman
    min_samples_split=2,     # Minimum sample untuk split
    min_samples_leaf=1,      # Minimum sample di leaf
    bootstrap=True,          # Gunakan bootstrap
    oob_score=True,          # Aktifkan OOB scoring
    random_state=42,
    n_jobs=-1,               # Gunakan semua CPU core
    class_weight='balanced'  # Handle class imbalance
)
rf_model.fit(X_train, y_train)

# =============================================
# 3. EVALUASI MODEL
# =============================================
y_pred = rf_model.predict(X_test)
y_prob = rf_model.predict_proba(X_test)[:, 1]

print("\n" + "=" * 60)
print("EVALUASI MODEL")
print("=" * 60)
print(f"OOB Score: {rf_model.oob_score_:.4f}")
print(f"Test Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"Precision: {precision_score(y_test, y_pred):.4f}")
print(f"Recall: {recall_score(y_test, y_pred):.4f}")
print(f"F1-Score: {f1_score(y_test, y_pred):.4f}")
print(f"ROC AUC: {roc_auc_score(y_test, y_prob):.4f}")
print(f"\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=data.target_names))

# =============================================
# 4. CROSS VALIDATION
# =============================================
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
cv_scores = cross_val_score(rf_model, X, y, cv=cv, scoring='accuracy')

print("=" * 60)
print("CROSS VALIDATION (5-Fold)")
print("=" * 60)
print(f"Scores: {cv_scores}")
print(f"Mean: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")

# =============================================
# 5. VISUALISASI
# =============================================
fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# 5a. Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
            xticklabels=data.target_names,
            yticklabels=data.target_names, ax=axes[0, 0])
axes[0, 0].set_title('Confusion Matrix')
axes[0, 0].set_xlabel('Prediksi')
axes[0, 0].set_ylabel('Aktual')

# 5b. ROC Curve
fpr, tpr, _ = roc_curve(y_test, y_prob)
axes[0, 1].plot(fpr, tpr, 'b-', linewidth=2,
                label=f'RF (AUC = {roc_auc_score(y_test, y_prob):.3f})')
axes[0, 1].plot([0, 1], [0, 1], 'k--', alpha=0.5)
axes[0, 1].set_title('ROC Curve')
axes[0, 1].set_xlabel('False Positive Rate')
axes[0, 1].set_ylabel('True Positive Rate')
axes[0, 1].legend()

# 5c. Feature Importance (Top 15)
importances = rf_model.feature_importances_
indices = np.argsort(importances)[::-1][:15]
axes[1, 0].barh(range(15), importances[indices][::-1])
axes[1, 0].set_yticks(range(15))
axes[1, 0].set_yticklabels([data.feature_names[i] for i in indices][::-1])
axes[1, 0].set_title('Top 15 Feature Importance')

# 5d. Learning Curve
train_sizes, train_scores, val_scores = learning_curve(
    rf_model, X_train, y_train, cv=5,
    train_sizes=np.linspace(0.1, 1.0, 10),
    scoring='accuracy', n_jobs=-1
)
axes[1, 1].plot(train_sizes, train_scores.mean(axis=1), 'o-', label='Training')
axes[1, 1].plot(train_sizes, val_scores.mean(axis=1), 'o-', label='Validation')
axes[1, 1].fill_between(train_sizes,
                         train_scores.mean(axis=1) - train_scores.std(axis=1),
                         train_scores.mean(axis=1) + train_scores.std(axis=1), alpha=0.2)
axes[1, 1].fill_between(train_sizes,
                         val_scores.mean(axis=1) - val_scores.std(axis=1),
                         val_scores.mean(axis=1) + val_scores.std(axis=1), alpha=0.2)
axes[1, 1].set_title('Learning Curve')
axes[1, 1].set_xlabel('Training Set Size')
axes[1, 1].set_ylabel('Accuracy')
axes[1, 1].legend()

plt.suptitle('Random Forest — Analisis Hasil', fontsize=14, y=1.02)
plt.tight_layout()
plt.show()

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from scipy.stats import randint, uniform
import numpy as np

# Dataset
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# =============================================
# 1. RANDOMIZED SEARCH (Lebih cepat, recommended)
# =============================================
param_distributions = {
    'n_estimators': randint(100, 500),
    'max_depth': [None, 10, 15, 20, 30],
    'max_features': ['sqrt', 'log2', 0.2, 0.3, 0.5],
    'min_samples_split': randint(2, 15),
    'min_samples_leaf': randint(1, 10),
    'class_weight': [None, 'balanced', 'balanced_subsample']
}

rf_random = RandomizedSearchCV(
    estimator=RandomForestClassifier(random_state=42),
    param_distributions=param_distributions,
    n_iter=100,           # 100 kombinasi random
    cv=5,
    scoring='f1_weighted',
    random_state=42,
    n_jobs=-1,
    verbose=1
)

rf_random.fit(X_train, y_train)

print("=" * 60)
print("RANDOMIZED SEARCH RESULTS")
print("=" * 60)
print(f"Best Score (CV): {rf_random.best_score_:.4f}")
print(f"Best Parameters:")
for param, value in rf_random.best_params_.items():
    print(f"  {param}: {value}")

best_rf = rf_random.best_estimator_
print(f"Test Accuracy: {best_rf.score(X_test, y_test):.4f}")

# =============================================
# 2. GRID SEARCH (Lebih teliti, lebih lambat)
# =============================================
# Hanya setelahRandomizedSearch menemukan range yang bagus
param_grid = {
    'n_estimators': [200, 300, 400],
    'max_depth': [None, 15, 20, 25],
    'max_features': ['sqrt', 0.3],
    'min_samples_split': [2, 3, 5],
    'min_samples_leaf': [1, 2, 3]
}

rf_grid = GridSearchCV(
    estimator=RandomForestClassifier(random_state=42),
    param_grid=param_grid,
    cv=5,
    scoring='f1_weighted',
    n_jobs=-1,
    verbose=1
)

rf_grid.fit(X_train, y_train)

print("\n" + "=" * 60)
print("GRID SEARCH RESULTS")
print("=" * 60)
print(f"Best Score (CV): {rf_grid.best_score_:.4f}")
print(f"Best Parameters:")
for param, value in rf_grid.best_params_.items():
    print(f"  {param}: {value}")

best_rf_grid = rf_grid.best_estimator_
print(f"Test Accuracy: {best_rf_grid.score(X_test, y_test):.4f}")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestRegressor
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

# Generate dataset regresi
X, y = make_regression(
    n_samples=1000,
    n_features=10,
    n_informative=7,
    noise=15,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Random Forest Regressor
rf_reg = RandomForestRegressor(
    n_estimators=300,
    max_depth=20,
    max_features='sqrt',
    min_samples_split=5,
    min_samples_leaf=2,
    oob_score=True,
    random_state=42,
    n_jobs=-1
)
rf_reg.fit(X_train, y_train)

# Prediksi
y_pred = rf_reg.predict(X_test)

# Evaluasi
print("=" * 50)
print("RANDOM FOREST REGRESSOR — HASIL")
print("=" * 50)
print(f"OOB Score (R²): {rf_reg.oob_score_:.4f}")
print(f"Test R² Score: {r2_score(y_test, y_pred):.4f}")
print(f"RMSE: {np.sqrt(mean_squared_error(y_test, y_pred)):.4f}")
print(f"MAE: {mean_absolute_error(y_test, y_pred):.4f}")

# Visualisasi: Prediksi vs Aktual
fig, axes = plt.subplots(1, 3, figsize=(18, 5))

# Plot 1: Prediksi vs Aktual
axes[0].scatter(y_test, y_pred, alpha=0.5, s=20)
axes[0].plot([y_test.min(), y_test.max()], 
             [y_test.min(), y_test.max()], 'r--', linewidth=2)
axes[0].set_xlabel('Nilai Aktual')
axes[0].set_ylabel('Nilai Prediksi')
axes[0].set_title(f'Prediksi vs Aktual (R²={r2_score(y_test, y_pred):.3f})')

# Plot 2: Residuals
residuals = y_test - y_pred
axes[1].scatter(y_pred, residuals, alpha=0.5, s=20)
axes[1].axhline(y=0, color='r', linestyle='--', linewidth=2)
axes[1].set_xlabel('Prediksi')
axes[1].set_ylabel('Residual (Aktual - Prediksi)')
axes[1].set_title('Residual Plot')

# Plot 3: Feature Importance
importances = rf_reg.feature_importances_
indices = np.argsort(importances)[::-1]
axes[2].bar(range(10), importances[indices])
axes[2].set_xticks(range(10))
axes[2].set_xticklabels([f'Feature {i}' for i in indices], rotation=45)
axes[2].set_title('Feature Importance')

plt.tight_layout()
plt.show()

# Perbandingan jumlah tree
print("\n--- Pengaruh Jumlah Tree ---")
n_trees_range = [10, 25, 50, 100, 200, 300, 500, 800]
scores = []
for n in n_trees_range:
    rf_temp = RandomForestRegressor(n_estimators=n, random_state=42, n_jobs=-1)
    cv_score = cross_val_score(rf_temp, X_train, y_train, cv=5, 
                                scoring='r2', n_jobs=-1)
    scores.append(cv_score.mean())
    print(f"  n_estimators={n:4d}: CV R² = {cv_score.mean():.4f} "
          f"(±{cv_score.std():.4f})")

plt.figure(figsize=(10, 5))
plt.plot(n_trees_range, scores, 'bo-', linewidth=2)
plt.xlabel('Jumlah Trees')
plt.ylabel('CV R² Score')
plt.title('Pengaruh Jumlah Trees terhadap Performa')
plt.grid(True, alpha=0.3)
plt.show()