Decision Trees: Pohon Keputusan

import numpy as np

def gini_impurity(labels):
    """Menghitung Gini Impurity dari sekumpulan label."""
    classes, counts = np.unique(labels, return_counts=True)
    probabilities = counts / len(labels)
    gini = 1 - np.sum(probabilities ** 2)
    return gini

# Contoh: Node dengan 10 data, 7 positif, 3 negatif
labels_mixed = ['Ya'] * 7 + ['Tidak'] * 3
print(f"Node campur (7 Ya, 3 Tidak): Gini = {gini_impurity(labels_mixed):.4f}")

# Node murni
labels_pure = ['Ya'] * 10
print(f"Node murni (10 Ya, 0 Tidak):  Gini = {gini_impurity(labels_pure):.4f}")

# Node paling campur
labels_worst = ['Ya'] * 5 + ['Tidak'] * 5
print(f"Node paling campur (5:5):     Gini = {gini_impurity(labels_worst):.4f}")

import numpy as np

def entropy(labels):
    """Menghitung Entropy dari sekumpulan label."""
    classes, counts = np.unique(labels, return_counts=True)
    probabilities = counts / len(labels)
    ent = -np.sum(probabilities * np.log2(probabilities + 1e-10))  # +1e-10 untuk hindari log(0)
    return ent

def information_gain(parent, left_child, right_child):
    """Menghitung Information Gain dari split."""
    weight_left = len(left_child) / len(parent)
    weight_right = len(right_child) / len(parent)
    ig = entropy(parent) - (weight_left * entropy(left_child) + weight_right * entropy(right_child))
    return ig

# Contoh dataset sederhana: apakah bermain tenis?
# [Cuaca, Suhu, Bermain?]
data = ['Ya', 'Ya', 'Ya', 'Tidak', 'Ya', 'Tidak', 'Ya', 'Ya', 'Tidak', 'Ya', 'Tidak', 'Ya', 'Ya', 'Tidak']

print(f"Entropy parent: {entropy(data):.4f}")

# Split berdasarkan cuaca
# Cerah → [Ya, Ya, Tidak, Ya, Tidak, Ya] (6 data)
# Hujan → [Ya, Ya, Tidak, Ya, Ya, Tidak, Ya, Tidak] (8 data)
cerah = ['Ya', 'Ya', 'Tidak', 'Ya', 'Tidak', 'Ya']
hujan = ['Ya', 'Ya', 'Tidak', 'Ya', 'Ya', 'Tidak', 'Ya', 'Tidak']

ig = information_gain(data, cerah, hujan)
print(f"Entropy cerah:  {entropy(cerah):.4f}")
print(f"Entropy hujan:  {entropy(hujan):.4f}")
print(f"Information Gain: {ig:.4f}")
print(f"\nSemakin besar IG, semakin baik split tersebut.")

import numpy as np
import pandas as pd
from sklearn.tree import DecisionTreeClassifier, export_text
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

# ===== DATASET: Prediksi Kelulusan Mahasiswa =====
np.random.seed(42)
n = 500

data = pd.DataFrame({
    'ipk': np.random.uniform(2.0, 4.0, n).round(2),
    'kehadiran_persen': np.random.uniform(40, 100, n).round(1),
    'jam_belajar_per_minggu': np.random.uniform(1, 30, n).round(1),
    'tugas_lengkap': np.random.randint(0, 2, n),  # 0=Tidak, 1=Ya
    'aktif_organisasi': np.random.randint(0, 2, n),  # 0=Tidak, 1=Ya
})

# Buat label lulus/tidak berdasarkan aturan + noise
skor = (
    data['ipk'] * 15
    + data['kehadiran_persen'] * 0.3
    + data['jam_belajar_per_minggu'] * 1.2
    + data['tugas_lengkap'] * 10
    - data['aktif_organisasi'] * 3
    + np.random.normal(0, 5, n)
)
data['lulus'] = (skor > np.percentile(skor, 40)).astype(int)

print("Dataset Kelulusan:")
print(data.head(10))
print(f"\nDistribusi kelas:")
print(data['lulus'].value_counts())

# ===== SPLIT & TRAIN =====
X = data.drop('lulus', axis=1)
y = data['lulus']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Decision Tree Classifier
dt_clf = DecisionTreeClassifier(
    criterion='gini',
    max_depth=5,
    min_samples_split=10,
    min_samples_leaf=5,
    random_state=42
)

dt_clf.fit(X_train, y_train)

# ===== EVALUASI =====
y_pred = dt_clf.predict(X_test)

print(f"\n=== HASIL EVALUASI ===")
print(f"Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"\nClassification Report:")
print(classification_report(y_test, y_pred, target_names=['Tidak Lulus', 'Lulus']))
print(f"Confusion Matrix:")
print(confusion_matrix(y_test, y_pred))

# ===== LIHAT STRUKTUR POHON =====
print(f"\n=== STRUKTUR POHON (Text) ===")
tree_rules = export_text(dt_clf, feature_names=list(X.columns), max_depth=4)
print(tree_rules)

import matplotlib.pyplot as plt

# Feature importance
importances = dt_clf.feature_importances_
feature_names = X.columns

# Urutkan dari paling penting
indices = np.argsort(importances)[::-1]

print("=== FEATURE IMPORTANCE ===")
for i, idx in enumerate(indices):
    print(f"  {i+1}. {feature_names[idx]:<25}: {importances[idx]:.4f} {'█' * int(importances[idx] * 40)}")

# Visualisasi
plt.figure(figsize=(10, 5))
plt.bar(range(len(importances)), importances[indices], color='steelblue', edgecolor='white')
plt.xticks(range(len(importances)), [feature_names[i] for i in indices], rotation=45, ha='right')
plt.title('Feature Importance — Decision Tree', fontsize=14)
plt.ylabel('Importance')
plt.tight_layout()
plt.savefig('feature_importance.png', dpi=150)
plt.show()

import numpy as np
import pandas as pd
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

# Dataset: Prediksi harga properti
np.random.seed(42)
n = 300

data_reg = pd.DataFrame({
    'luas_m2': np.random.uniform(30, 200, n),
    'kamar': np.random.randint(1, 5, n),
    'usia_bangunan': np.random.uniform(0, 30, n),
    'jarak_transportasi': np.random.uniform(0.1, 15, n).round(1),
})

# Harga non-linear
data_reg['harga_juta'] = (
    100
    + 5 * data_reg['luas_m2']
    + 0.02 * data_reg['luas_m2'] ** 2  # Non-linear!
    + 30 * data_reg['kamar']
    - 2 * data_reg['usia_bangunan']
    - 10 * data_reg['jarak_transportasi']
    + np.random.normal(0, 50, n)
).round(1)

X = data_reg.drop('harga_juta', axis=1)
y = data_reg['harga_juta']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# ===== DECISION TREE REGRESSOR =====
dt_reg = DecisionTreeRegressor(
    max_depth=6,
    min_samples_split=15,
    min_samples_leaf=8,
    random_state=42
)
dt_reg.fit(X_train, y_train)

y_pred_train = dt_reg.predict(X_train)
y_pred_test = dt_reg.predict(X_test)

print("=== DECISION TREE REGRESSOR ===")
print(f"Train R²:  {r2_score(y_train, y_pred_train):.4f}")
print(f"Test R²:   {r2_score(y_test, y_pred_test):.4f}")
print(f"Train RMSE: {np.sqrt(mean_squared_error(y_train, y_pred_train)):.2f}")
print(f"Test RMSE:  {np.sqrt(mean_squared_error(y_test, y_pred_test)):.2f}")

# Visualisasi: Actual vs Predicted
plt.figure(figsize=(10, 5))
plt.scatter(y_test, y_pred_test, alpha=0.5, color='steelblue')
plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', linewidth=2)
plt.xlabel('Harga Aktual (Juta Rp)')
plt.ylabel('Harga Prediksi (Juta Rp)')
plt.title('Decision Tree Regressor: Actual vs Predicted')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

# Visualisasi: Perbandingan kedalaman pohon
depths = range(1, 20)
train_scores = []
test_scores = []

for d in depths:
    dt = DecisionTreeRegressor(max_depth=d, random_state=42)
    dt.fit(X_train, y_train)
    train_scores.append(r2_score(y_train, dt.predict(X_train)))
    test_scores.append(r2_score(y_test, dt.predict(X_test)))

plt.figure(figsize=(10, 5))
plt.plot(depths, train_scores, 'b-o', label='Train R²')
plt.plot(depths, test_scores, 'r-s', label='Test R²')
plt.xlabel('Max Depth')
plt.ylabel('R² Score')
plt.title('Effect of Tree Depth on Performance')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score
import numpy as np

# ===== PRE-PRUNING: Eksperimen dengan berbagai parameter =====
param_configs = [
    {'max_depth': None, 'min_samples_split': 2, 'min_samples_leaf': 1},  # Tanpa batasan
    {'max_depth': 5, 'min_samples_split': 10, 'min_samples_leaf': 5},    # Moderat
    {'max_depth': 3, 'min_samples_split': 20, 'min_samples_leaf': 10},   # Konservatif
    {'max_depth': 10, 'min_samples_split': 5, 'min_samples_leaf': 3},    # Agresif
]

print(f"{'Config':<40} {'Train Acc':>10} {'Test Acc':>10} {'CV Mean':>10}")
print("-" * 74)

for i, params in enumerate(param_configs):
    dt = DecisionTreeClassifier(random_state=42, **params)
    dt.fit(X_train, y_train)

    train_acc = dt.score(X_train, y_train)
    test_acc = dt.score(X_test, y_test)
    cv_scores = cross_val_score(dt, X_train, y_train, cv=5)

    desc = ', '.join(f'{k}={v}' for k, v in params.items())
    print(f"{desc:<40} {train_acc:>10.4f} {test_acc:>10.4f} {cv_scores.mean():>10.4f}")

print("\n→ Perhatikan: tanpa batasan, train accuracy sangat tinggi tapi test accuracy turun (overfitting!)")

from sklearn.tree import DecisionTreeClassifier
import matplotlib.pyplot as plt
import numpy as np

# Dapatkan path cost complexity pruning
dt_full = DecisionTreeClassifier(random_state=42)
dt_full.fit(X_train, y_train)

# Mendapatkan nilai ccp_alpha yang mungkin
path = dt_full.cost_complexity_pruning_path(X_train, y_train)
ccp_alphas = path.ccp_alphas
impurities = path.impurities

# Latih model untuk setiap ccp_alpha
train_scores = []
test_scores = []
node_counts = []

for alpha in ccp_alphas:
    dt = DecisionTreeClassifier(ccp_alpha=alpha, random_state=42)
    dt.fit(X_train, y_train)
    train_scores.append(dt.score(X_train, y_train))
    test_scores.append(dt.score(X_test, y_test))
    node_counts.append(dt.tree_.node_count)

# Visualisasi
fig, axes = plt.subplots(1, 3, figsize=(18, 5))

# Plot 1: Accuracy vs Alpha
axes[0].plot(ccp_alphas, train_scores, 'b-', label='Train')
axes[0].plot(ccp_alphas, test_scores, 'r-', label='Test')
axes[0].set_xlabel('ccp_alpha')
axes[0].set_ylabel('Accuracy')
axes[0].set_title('Accuracy vs ccp_alpha')
axes[0].legend()
axes[0].grid(True, alpha=0.3)

# Plot 2: Number of nodes vs Alpha
axes[1].plot(ccp_alphas, node_counts, 'g-')
axes[1].set_xlabel('ccp_alpha')
axes[1].set_ylabel('Number of Nodes')
axes[1].set_title('Tree Size vs ccp_alpha')
axes[1].grid(True, alpha=0.3)

# Plot 3: Impurity vs Alpha
axes[2].plot(ccp_alphas, impurities, 'm-')
axes[2].set_xlabel('ccp_alpha')
axes[2].set_ylabel('Impurity')
axes[2].set_title('Impurity vs ccp_alpha')
axes[2].grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

# Pilih alpha terbaik
best_idx = np.argmax(test_scores)
best_alpha = ccp_alphas[best_idx]
print(f"Best ccp_alpha: {best_alpha:.6f}")
print(f"Best test accuracy: {test_scores[best_idx]:.4f}")

# Model final dengan pruning optimal
dt_pruned = DecisionTreeClassifier(ccp_alpha=best_alpha, random_state=42)
dt_pruned.fit(X_train, y_train)
print(f"Nodes setelah pruning: {dt_pruned.tree_.node_count}")

from sklearn.tree import plot_tree, export_text, export_graphviz
import matplotlib.pyplot as plt

# ===== VISUALISASI 1: Matplotlib plot_tree =====
fig, ax = plt.subplots(figsize=(20, 10))
plot_tree(
    dt_pruned,
    feature_names=list(X.columns),
    class_names=['Tidak Lulus', 'Lulus'],
    filled=True,           # Warna berdasarkan kelas mayoritas
    rounded=True,          # Node berbentuk rounded
    fontsize=9,
    max_depth=3,           # Batasi tampilan 3 level
    ax=ax
)
plt.title('Visualisasi Decision Tree (max_depth=3)', fontsize=16)
plt.tight_layout()
plt.savefig('decision_tree_visual.png', dpi=150, bbox_inches='tight')
plt.show()

# ===== VISUALISASI 2: Text representation =====
print("=== STRUKTUR POHON (TEXT) ===")
tree_text = export_text(dt_pruned, feature_names=list(X.columns), max_depth=4)
print(tree_text[:2000])  # Batasi output

# ===== VISUALISASI 3: Graphviz (untuk visualisasi lebih cantik) =====
# Uncomment jika graphviz terinstal:
# export_graphviz(
#     dt_pruned,
#     out_file='decision_tree.dot',
#     feature_names=list(X.columns),
#     class_names=['Tidak Lulus', 'Lulus'],
#     filled=True, rounded=True
# )
# # Jalankan di terminal: dot -Tpng decision_tree.dot -o decision_tree.png

from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report, accuracy_score
import numpy as np
import matplotlib.pyplot as plt

# ===== RANDOM FOREST CLASSIFIER =====
rf_clf = RandomForestClassifier(
    n_estimators=200,        # Jumlah pohon
    max_depth=10,            # Kedalaman setiap pohon
    min_samples_split=5,
    min_samples_leaf=3,
    max_features='sqrt',     # √jumlah fitur di setiap split
    bootstrap=True,          # Aktifkan bagging
    random_state=42,
    n_jobs=-1                # Gunakan semua CPU
)

rf_clf.fit(X_train, y_train)

# Evaluasi
y_pred_rf = rf_clf.predict(X_test)

print("=== RANDOM FOREST CLASSIFIER ===")
print(f"Accuracy: {accuracy_score(y_test, y_pred_rf):.4f}")
print(f"\nClassification Report:")
print(classification_report(y_test, y_pred_rf, target_names=['Tidak Lulus', 'Lulus']))

# Cross-validation
cv_scores = cross_val_score(rf_clf, X, y, cv=5, scoring='accuracy')
print(f"CV Accuracy: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")

# ===== PERBANDINGAN: Decision Tree vs Random Forest =====
dt_single = DecisionTreeClassifier(max_depth=10, random_state=42)
dt_single.fit(X_train, y_train)

print(f"\n=== PERBANDINGAN ===")
print(f"{'Model':<25} {'Train Acc':>10} {'Test Acc':>10}")
print("-" * 47)
print(f"{'Decision Tree':<25} {dt_single.score(X_train, y_train):>10.4f} {dt_single.score(X_test, y_test):>10.4f}")
print(f"{'Random Forest':<25} {rf_clf.score(X_train, y_train):>10.4f} {rf_clf.score(X_test, y_test):>10.4f}")

# Feature importance dari Random Forest
importances = rf_clf.feature_importances_
std = np.std([tree.feature_importances_ for tree in rf_clf.estimators_], axis=0)

feature_names = X.columns
indices = np.argsort(importances)[::-1]

print("=== FEATURE IMPORTANCE (Random Forest) ===")
for i, idx in enumerate(indices):
    print(f"  {i+1}. {feature_names[idx]:<25}: {importances[idx]:.4f} ± {std[idx]:.4f}")

# Visualisasi dengan error bar
plt.figure(figsize=(10, 5))
plt.bar(range(len(importances)), importances[indices], yerr=std[indices],
        color='forestgreen', edgecolor='white', alpha=0.8, capsize=5)
plt.xticks(range(len(importances)), [feature_names[i] for i in indices], rotation=45, ha='right')
plt.title('Feature Importance — Random Forest', fontsize=14)
plt.ylabel('Importance')
plt.tight_layout()
plt.show()

from sklearn.model_selection import GridSearchCV

# Parameter grid untuk tuning
param_grid = {
    'n_estimators': [100, 200, 300],
    'max_depth': [5, 10, 15, None],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 3, 5],
    'max_features': ['sqrt', 'log2']
}

# Grid Search dengan Cross Validation
rf_grid = GridSearchCV(
    RandomForestClassifier(random_state=42, n_jobs=-1),
    param_grid=param_grid,
    cv=3,
    scoring='accuracy',
    n_jobs=-1,
    verbose=1
)

rf_grid.fit(X_train, y_train)

print(f"Best Parameters: {rf_grid.best_params_}")
print(f"Best CV Score: {rf_grid.best_score_:.4f}")
print(f"Test Score: {rf_grid.best_estimator_.score(X_test, y_test):.4f}")

from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score

# Random Forest untuk regresi
rf_reg = RandomForestRegressor(
    n_estimators=200,
    max_depth=12,
    min_samples_split=10,
    random_state=42,
    n_jobs=-1
)

rf_reg.fit(X_train, y_train)
y_pred_rf = rf_reg.predict(X_test)

print("=== RANDOM FOREST REGRESSOR ===")
print(f"R² Score: {r2_score(y_test, y_pred_rf):.4f}")
print(f"RMSE: {np.sqrt(mean_squared_error(y_test, y_pred_rf)):.2f}")
print(f"MAE: {np.mean(np.abs(y_test - y_pred_rf)):.2f}")

# Perbandingan dengan Decision Tree tunggal
dt_reg = DecisionTreeRegressor(max_depth=10, random_state=42)
dt_reg.fit(X_train, y_train)
y_pred_dt = dt_reg.predict(X_test)

print(f"\n=== PERBANDINGAN REGRESI ===")
print(f"{'Model':<25} {'R²':>8} {'RMSE':>10}")
print("-" * 45)
print(f"{'Decision Tree':<25} {r2_score(y_test, y_pred_dt):>8.4f} {np.sqrt(mean_squared_error(y_test, y_pred_dt)):>10.2f}")
print(f"{'Random Forest':<25} {r2_score(y_test, y_pred_rf):>8.4f} {np.sqrt(mean_squared_error(y_test, y_pred_rf)):>10.2f}")