CNN: Convolutional Neural Networks

import numpy as np

def conv2d_manual(image, kernel):
    """Implementasi convolution 2D secara manual."""
    h, w = image.shape
    kh, kw = kernel.shape
    out_h = h - kh + 1
    out_w = w - kw + 1
    output = np.zeros((out_h, out_w))
    
    for i in range(out_h):
        for j in range(out_w):
            # Ambil patch dari image
            patch = image[i:i+kh, j:j+kw]
            # Element-wise multiplication + sum
            output[i, j] = np.sum(patch * kernel)
    
    return output

# Contoh input
image = np.array([
    [1, 2, 0, 1, 3],
    [0, 1, 2, 3, 1],
    [2, 3, 1, 0, 2],
    [1, 0, 3, 2, 1],
    [3, 2, 1, 0, 3]
], dtype=float)

# Vertical edge detector
kernel_vert = np.array([
    [-1, 0, 1],
    [-1, 0, 1],
    [-1, 0, 1]
], dtype=float)

# Horizontal edge detector
kernel_horiz = np.array([
    [-1, -1, -1],
    [ 0,  0,  0],
    [ 1,  1,  1]
], dtype=float)

print("Input Image:")
print(image)
print(f"\nVertical Edge Kernel:")
print(kernel_vert)

output_vert = conv2d_manual(image, kernel_vert)
output_horiz = conv2d_manual(image, kernel_horiz)

print(f"\nOutput (Vertical Edge):")
print(output_vert)
print(f"\nOutput (Horizontal Edge):")
print(output_horiz)

# Dimensi output
h, w = image.shape
kh, kw = kernel_vert.shape
print(f"\nInput size: {h}×{w}")
print(f"Kernel size: {kh}×{kw}")
print(f"Output size: {h-kh+1}×{w-kw+1} = {h-kh+1}×{w-kw+1}")

def calc_output_size(input_size, kernel_size, padding=0, stride=1):
    """Hitung output size dari convolution layer."""
    return (input_size - kernel_size + 2 * padding) // stride + 1

# Test berbagai konfigurasi
configs = [
    (32, 3, 0, 1, "Valid, stride=1"),
    (32, 3, 1, 1, "Same, stride=1"),
    (32, 3, 1, 2, "Downsample 2x"),
    (28, 5, 0, 1, "5x5 kernel, no pad"),
    (28, 5, 2, 1, "5x5 kernel, same pad"),
    (224, 7, 3, 2, "ResNet first layer"),
]

print("Input  Kernel  Pad  Stride  Output  Deskripsi")
print("-" * 65)
for inp, ker, pad, stride, desc in configs:
    out = calc_output_size(inp, ker, pad, stride)
    print(f"  {inp:3d}    {ker:2d}x{ker:<2d}  {pad:2d}    {stride:2d}     {out:3d}x{out:<3d}   {desc}")

# Multi-layer CNN output size tracker
print("\n=== Arsitektur CNN Sederhana ===")
layers = [
    ("Conv1", 32, 3, 1, 1),
    ("Conv2", 32, 3, 1, 1),
    ("Pool1", 32, 2, 0, 2),  # MaxPool 2x2
    ("Conv3", 16, 3, 1, 1),
    ("Conv4", 16, 3, 1, 1),
    ("Pool2", 16, 2, 0, 2),  # MaxPool 2x2
]

size = 28
print(f"Input: {size}x{size}")
for name, size_val, ker, pad, stride in layers:
    size = calc_output_size(size, ker, pad, stride)
    print(f"  {name}: kernel={ker}x{ker}, pad={pad}, stride={stride} → {size}x{size}")
print(f"Flatten output: {size*size*16} = {size}×{size}×16")

import numpy as np
import matplotlib.pyplot as plt

x = np.linspace(-5, 5, 200)

# Fungsi aktivasi
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def tanh(x):
    return np.tanh(x)

def relu(x):
    return np.maximum(0, x)

def leaky_relu(x, alpha=0.01):
    return np.where(x > 0, x, alpha * x)

def swish(x):
    return x * sigmoid(x)

activations = {
    'Sigmoid': sigmoid(x),
    'Tanh': tanh(x),
    'ReLU': relu(x),
    'Leaky ReLU (α=0.01)': leaky_relu(x),
    'Swish (SiLU)': swish(x)
}

fig, axes = plt.subplots(2, 3, figsize=(15, 10))
for ax, (name, values) in zip(axes.ravel(), activations.items()):
    ax.plot(x, values, 'b-', linewidth=2)
    ax.axhline(y=0, color='k', linewidth=0.5)
    ax.axvline(x=0, color='k', linewidth=0.5)
    ax.set_title(name, fontsize=12)
    ax.grid(True, alpha=0.3)
    ax.set_ylim(-2, 5)

# Hide last subplot
axes[1, 2].axis('off')
plt.suptitle('Fungsi Aktivasi Neural Network', fontsize=14)
plt.tight_layout()
plt.show()

# ReLU vs Sigmoid: gradient comparison
print("=== Gradient Comparison ===")
print(f"Sigmoid gradient max: {sigmoid(np.array([0])) * (1 - sigmoid(np.array([0]))):.4f}")
print(f"Saturated sigmoid: {sigmoid(np.array([5])) * (1 - sigmoid(np.array([5]))):.6f}")
print(f"ReLU gradient (x=5): {1.0:.4f}")
print(f"ReLU gradient (x=-5): {0.0:.4f}")
print("\n→ Sigmoid mengalami vanishing gradient!")
print("→ ReLU: gradient selalu 0 atau 1 (menghindari vanishing gradient)")

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt
import numpy as np

# =============================================
# 1. DEVICE & HYPERPARAMETERS
# =============================================
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Device: {device}")

BATCH_SIZE = 64
LEARNING_RATE = 0.001
NUM_EPOCHS = 20
NUM_CLASSES = 10

# =============================================
# 2. DATA LOADING & AUGMENTATION
# =============================================
transform_train = transforms.Compose([
    transforms.RandomHorizontalFlip(p=0.5),
    transforms.RandomRotation(10),
    transforms.RandomAffine(0, translate=(0.1, 0.1)),
    transforms.ColorJitter(brightness=0.2, contrast=0.2),
    transforms.ToTensor(),
    transforms.Normalize(
        mean=[0.4914, 0.4822, 0.4465],
        std=[0.2470, 0.2435, 0.2616]
    )
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(
        mean=[0.4914, 0.4822, 0.4465],
        std=[0.2470, 0.2435, 0.2616]
    )
])

# Download CIFAR-10
train_dataset = torchvision.datasets.CIFAR10(
    root='./data', train=True, download=True, transform=transform_train
)
test_dataset = torchvision.datasets.CIFAR10(
    root='./data', train=False, download=True, transform=transform_test
)

train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, 
                          shuffle=True, num_workers=2)
test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE,
                         shuffle=False, num_workers=2)

classes = ('airplane', 'automobile', 'bird', 'cat', 'deer',
           'dog', 'frog', 'horse', 'ship', 'truck')

print(f"Training samples: {len(train_dataset)}")
print(f"Test samples: {len(test_dataset)}")

# =============================================
# 3. MODEL DEFINITION
# =============================================
class CNNCIFAR10(nn.Module):
    def __init__(self, num_classes=10):
        super(CNNCIFAR10, self).__init__()
        
        # Block 1: Conv → BN → ReLU → Conv → BN → ReLU → MaxPool
        self.block1 = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(inplace=True),
            nn.Conv2d(32, 32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, 2),         # 32×32 → 16×16
            nn.Dropout2d(0.25)
        )
        
        # Block 2
        self.block2 = nn.Sequential(
            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, 2),         # 16×16 → 8×8
            nn.Dropout2d(0.25)
        )
        
        # Block 3
        self.block3 = nn.Sequential(
            nn.Conv2d(64, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, 2),         # 8×8 → 4×4
            nn.Dropout2d(0.25)
        )
        
        # Global Average Pooling + Classifier
        self.global_pool = nn.AdaptiveAvgPool2d(1)  # 4×4 → 1×1
        
        self.classifier = nn.Sequential(
            nn.Linear(128, 256),
            nn.ReLU(inplace=True),
            nn.Dropout(0.5),
            nn.Linear(256, num_classes)
        )
    
    def forward(self, x):
        x = self.block1(x)    # → (32, 16, 16)
        x = self.block2(x)    # → (64, 8, 8)
        x = self.block3(x)    # → (128, 4, 4)
        x = self.global_pool(x)  # → (128, 1, 1)
        x = x.view(x.size(0), -1)  # → (128)
        x = self.classifier(x)
        return x

model = CNNCIFAR10(NUM_CLASSES).to(device)

# Print model summary
total_params = sum(p.numel() for p in model.parameters())
trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f"\nModel Parameters: {total_params:,} total, {trainable_params:,} trainable")

# =============================================
# 4. TRAINING LOOP
# =============================================
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-4)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)

train_losses = []
train_accs = []
test_accs = []

for epoch in range(NUM_EPOCHS):
    model.train()
    running_loss = 0.0
    correct = 0
    total = 0
    
    for batch_idx, (images, labels) in enumerate(train_loader):
        images, labels = images.to(device), labels.to(device)
        
        optimizer.zero_grad()
        outputs = model(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        
        running_loss += loss.item()
        _, predicted = outputs.max(1)
        total += labels.size(0)
        correct += predicted.eq(labels).sum().item()
    
    scheduler.step()
    
    train_loss = running_loss / len(train_loader)
    train_acc = 100. * correct / total
    train_losses.append(train_loss)
    train_accs.append(train_acc)
    
    # Test evaluation
    model.eval()
    test_correct = 0
    test_total = 0
    
    with torch.no_grad():
        for images, labels in test_loader:
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)
            _, predicted = outputs.max(1)
            test_total += labels.size(0)
            test_correct += predicted.eq(labels).sum().item()
    
    test_acc = 100. * test_correct / test_total
    test_accs.append(test_acc)
    
    print(f"Epoch [{epoch+1:2d}/{NUM_EPOCHS}] "
          f"Loss: {train_loss:.4f} | "
          f"Train Acc: {train_acc:.2f}% | "
          f"Test Acc: {test_acc:.2f}%")

# =============================================
# 5. VISUALISASI
# =============================================
fig, axes = plt.subplots(1, 3, figsize=(18, 5))

# Loss curve
axes[0].plot(train_losses, 'b-', linewidth=2)
axes[0].set_title('Training Loss')
axes[0].set_xlabel('Epoch')
axes[0].set_ylabel('Loss')
axes[0].grid(True, alpha=0.3)

# Accuracy curve
axes[1].plot(train_accs, 'b-', linewidth=2, label='Train')
axes[1].plot(test_accs, 'r-', linewidth=2, label='Test')
axes[1].set_title('Accuracy')
axes[1].set_xlabel('Epoch')
axes[1].set_ylabel('Accuracy (%)')
axes[1].legend()
axes[1].grid(True, alpha=0.3)

# Sample predictions
dataiter = iter(test_loader)
images, labels = next(dataiter)
images_gpu = images[:8].to(device)
outputs = model(images_gpu)
_, predicted = outputs.max(1)

mean = torch.tensor([0.4914, 0.4822, 0.4465])
std = torch.tensor([0.2470, 0.2435, 0.2616])

for i in range(8):
    ax = axes[2] if i < 1 else None
    img = images[i].permute(1, 2, 0) * std + mean
    img = img.clamp(0, 1)
    axes[2].imshow(img)

plt.suptitle(f'CNN CIFAR-10 — Final Test Acc: {test_accs[-1]:.2f}%', fontsize=14)
plt.tight_layout()
plt.show()

import torch
import torch.nn as nn
import torchvision.models as models

# =============================================
# TRANSFER LEARNING: Pre-trained ResNet-18
# =============================================

# Load pre-trained ResNet-18
model = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)

print("=== Original ResNet-18 ===")
print(f"Total params: {sum(p.numel() for p in model.parameters()):,}")

# =============================================
# STRATEGY 1: Feature Extraction
# =============================================
# Freeze ALL convolution layers
for param in model.parameters():
    param.requires_grad = False

# Replace classifier (final FC layer)
# ResNet-18 final layer: model.fc (512 → 1000)
model.fc = nn.Sequential(
    nn.Linear(512, 256),
    nn.ReLU(),
    nn.Dropout(0.3),
    nn.Linear(256, 10)  # 10 classes for CIFAR-10
)

trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
total_params = sum(p.numel() for p in model.parameters())
print(f"\n=== After Feature Extraction ===")
print(f"Total params: {total_params:,}")
print(f"Trainable params: {trainable_params:,}")
print(f"Frozen params: {total_params - trainable_params:,}")

# =============================================
# STRATEGY 2: Fine-Tuning
# =============================================
model_ft = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)

# Freeze early layers (layer1, layer2)
for name, param in model_ft.named_parameters():
    if 'layer1' in name or 'layer2' in name:
        param.requires_grad = False

# Unfreeze later layers (layer3, layer4) + FC
for name, param in model_ft.named_parameters():
    if 'layer3' in name or 'layer4' in name:
        param.requires_grad = True

# Replace FC
model_ft.fc = nn.Sequential(
    nn.Linear(512, 256),
    nn.ReLU(),
    nn.Dropout(0.3),
    nn.Linear(256, 10)
)

trainable_ft = sum(p.numel() for p in model_ft.parameters() if p.requires_grad)
print(f"\n=== After Fine-Tuning ===")
print(f"Trainable params: {trainable_ft:,}")
print(f"Frozen early layers: layer1, layer2")
print(f"Unfrozen: layer3, layer4, fc")

# =============================================
# DIFFERENT LEARNING RATES (LR Discriminative)
# =============================================
# Berikan learning rate berbeda per layer group
param_groups = [
    {'params': [p for n, p in model_ft.named_parameters()
                if 'layer1' in n or 'layer2' in n],
     'lr': 1e-5},          # Frozen layer: very small LR (if any)
    {'params': [p for n, p in model_ft.named_parameters()
                if 'layer3' in n or 'layer4' in n],
     'lr': 1e-4},          # Middle layers: small LR
    {'params': model_ft.fc.parameters(),
     'lr': 1e-3}           # New layers: larger LR
]

optimizer_ft = torch.optim.Adam(param_groups, weight_decay=1e-4)
print("\n=== Discriminative Learning Rates ===")
print("  Early layers: 1e-5")
print("  Middle layers: 1e-4")
print("  FC layer:     1e-3")