Pandas & NumPy: Manipulasi Data Python

# Instalasi NumPy dan Pandas
pip install numpy pandas

# Verifikasi
python -c "import numpy; print('NumPy', numpy.__version__)"
python -c "import pandas; print('Pandas', pandas.__version__)"

# Import convention (standar di komunitas data science)
# import numpy as np
# import pandas as pd

import numpy as np

# Membuat array dari list Python
arr1d = np.array([1, 2, 3, 4, 5])
print(arr1d)          # [1 2 3 4 5]
print(arr1d.shape)    # (5,)
print(arr1d.dtype)    # int64

# Array 2D (matriks)
arr2d = np.array([
    [1, 2, 3],
    [4, 5, 6],
    [7, 8, 9]
])
print(arr2d.shape)    # (3, 3)
print(arr2d.ndim)     # 2 (jumlah dimensi)

# Array dengan tipe data spesifik
arr_float = np.array([1, 2, 3], dtype=np.float64)
print(arr_float.dtype)  # float64

# Array khusus
zeros = np.zeros((3, 4))      # Matriks 3x4 berisi 0
ones = np.ones((2, 3))        # Matriks 2x3 berisi 1
identity = np.eye(4)           # Matriks identitas 4x4
range_arr = np.arange(0, 10, 2)   # [0 2 4 6 8]
linspace = np.linspace(0, 1, 5)   # [0.   0.25 0.5  0.75 1.  ]

# Random arrays
rng = np.random.default_rng(42)
random_arr = rng.random((3, 3))        # Random 0-1
random_int = rng.integers(1, 100, 5)   # 5 random int 1-99
normal = rng.normal(0, 1, (2, 3))      # Normal distribution

print(f"Zeros:\n{zeros}")
print(f"Random integers: {random_int}")

import numpy as np

a = np.array([10, 20, 30, 40, 50])
b = np.array([1, 2, 3, 4, 5])

# Operasi aritmatika element-wise
print(a + b)      # [11 22 33 44 55]
print(a - b)      # [ 9 18 27 36 45]
print(a * b)      # [ 10  40  90 160 250]
print(a / b)      # [10. 10. 10. 10. 10.]
print(a ** 2)     # [ 100  400  900 1600 2500]

# Operasi statistik
print(f"Mean:    {a.mean()}")      # 30.0
print(f"Median:  {np.median(a)}")  # 30.0
print(f"Std Dev: {a.std():.2f}")   # 14.14
print(f"Min:     {a.min()}")       # 10
print(f"Max:     {a.max()}")       # 50
print(f"Sum:     {a.sum()}")       # 150

# Operasi pada matriks
matriks = np.array([
    [1, 2, 3],
    [4, 5, 6],
    [7, 8, 9]
])

print(f"Sum per kolom: {matriks.sum(axis=0)}")  # [12 15 18]
print(f"Sum per baris: {matriks.sum(axis=1)}")  # [ 6 15 24]
print(f"Transpose:\n{matriks.T}")
print(f"Diagonal: {np.diag(matriks)}")          # [1 5 9]

import numpy as np

arr = np.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100])

# Basic indexing
print(arr[0])       # 10 (elemen pertama)
print(arr[-1])      # 100 (elemen terakhir)
print(arr[3])       # 40

# Slicing [start:stop:step]
print(arr[2:7])     # [30 40 50 60 70]
print(arr[::2])     # [10 30 50 70 90] (setiap 2)
print(arr[::-1])    # [100 90 80 ... 10] (reversed)

# 2D array indexing
mat = np.array([
    [1, 2, 3, 4],
    [5, 6, 7, 8],
    [9, 10, 11, 12]
])

print(mat[0, 0])     # 1 (baris 0, kolom 0)
print(mat[1, 2])     # 7 (baris 1, kolom 2)
print(mat[0, :])     # [1 2 3 4] (seluruh baris 0)
print(mat[:, 1])     # [2 6 10] (seluruh kolom 1)
print(mat[0:2, 1:3]) # [[2 3] [6 7]] (sub-matriks)

# Boolean indexing (filtering)
data = np.array([15, 22, 8, 42, 33, 5, 18, 55])
mask = data > 20
print(data[mask])    # [22 42 33 55]

# Fancy indexing
indices = [0, 3, 5]
print(data[indices]) # [15 42  5]

import numpy as np

# Contoh Broadcasting
matriks = np.array([
    [1, 2, 3],
    [4, 5, 6],
    [7, 8, 9]
])

# Scalar broadcasting
print(matriks * 10)
# [[10 20 30]
#  [40 50 60]
#  [70 80 90]]

# Vector broadcasting (baris)
kolom = np.array([[10], [20], [30]])  # shape (3,1)
print(matriks + kolom)
# [[11 12 13]
#  [24 25 26]
#  [37 38 39]]

# Normalisasi data (praktik umum)
data = np.array([
    [170, 65, 25],  # [tinggi_cm, berat_kg, umur]
    [180, 80, 30],
    [165, 55, 22],
    [175, 70, 28],
])

# Z-score normalisasi dengan broadcasting
mean = data.mean(axis=0)   # Mean per kolom
std = data.std(axis=0)     # Std per kolom
normalized = (data - mean) / std

print(f"Mean per kolom: {mean}")
print(f"Std per kolom:  {std}")
print(f"Normalized:\n{normalized}")

import numpy as np

# Reshape
arr = np.arange(12)
print(arr.reshape(3, 4))
# [[ 0  1  2  3]
#  [ 4  5  6  7]
#  [ 8  9 10 11]]

# Stacking arrays
a = np.array([[1, 2], [3, 4]])
b = np.array([[5, 6], [7, 8]])

print(np.vstack((a, b)))  # Vertical stack
# [[1 2] [3 4] [5 6] [7 8]]

print(np.hstack((a, b)))  # Horizontal stack
# [[1 2 5 6] [3 4 7 8]]

# Dot product (perkalian matriks)
A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])
print(np.dot(A, B))
# [[19 22]
#  [43 50]]

# Atau dengan operator @
print(A @ B)  # Sama dengan np.dot

# Linear algebra
m = np.array([[2, 1], [5, 3]])
print(f"Determinan: {np.linalg.det(m):.0f}")      # 1.0
print(f"Inverse:\n{np.linalg.inv(m)}")
print(f"Eigenvalues: {np.linalg.eig(m)[0]}")

# Simpan dan load array
np.save('data_array.npy', data)
loaded = np.save('data_array.npy')
np.savetxt('data_array.csv', data, delimiter=',')
loaded_csv = np.loadtxt('data_array.csv', delimiter=',')

import pandas as pd

# Series dari list
s = pd.Series([10, 20, 30, 40, 50])
print(s)
# 0    10
# 1    20
# 2    30
# 3    40
# 4    50
# dtype: int64

# Series dengan custom index
suhu = pd.Series(
    [28.5, 30.2, 27.8, 31.5, 29.0],
    index=['Senin', 'Selasa', 'Rabu', 'Kamis', 'Jumat'],
    name='Suhu Harian'
)
print(suhu)
# Senin     28.5
# Selasa    30.2
# Rabu      27.8
# Kamis     31.5
# Jumat     29.0
# Name: Suhu Harian, dtype: float64

# Series dari dictionary
penduduk = pd.Series({
    'Jakarta': 10560000,
    'Surabaya': 2870000,
    'Bandung': 2500000,
    'Medan': 2430000,
    'Semarang': 1750000
})
print(penduduk)

# Akses elemen
print(suhu['Rabu'])        # 27.8 (label-based)
print(suhu.iloc[2])        # 27.8 (position-based)
print(suhu['Senin':'Rabu'])  # Slicing (inklusif di Pandas!)

import pandas as pd

nilai = pd.Series({
    'Matematika': 85,
    'Fisika': 78,
    'Kimia': 92,
    'Biologi': 88,
    'Bahasa': 75
})

# Statistik
print(f"Rata-rata: {nilai.mean():.1f}")      # 83.6
print(f"Tertinggi: {nilai.idxmax()}")          # Kimia
print(f"Terendah:  {nilai.idxmin()}")          # Bahasa

# Filtering
print(nilai[nilai >= 85])  # Nilai >= 85

# Sorting
print(nilai.sort_values(ascending=False))  # Sort descending

# Apply function
def grade(score):
    if score >= 90: return 'A'
    elif score >= 80: return 'B'
    elif score >= 70: return 'C'
    else: return 'D'

grades = nilai.apply(grade)
print(grades)

# Value counts
print(grades.value_counts())

import pandas as pd

# DataFrame dari dictionary
karyawan = pd.DataFrame({
    'Nama': ['Andi', 'Budi', 'Citra', 'Dewi', 'Eka'],
    'Departemen': ['IT', 'Finance', 'IT', 'HR', 'Finance'],
    'Usia': [28, 35, 24, 31, 29],
    'Gaji': [8500000, 12000000, 7000000, 9500000, 11000000],
    'Rating': [4.5, 3.8, 4.2, 4.0, 4.7]
})

print(karyawan)
#     Nama Departemen  Usia      Gaji  Rating
# 0   Andi         IT    28   8500000     4.5
# 1   Budi    Finance    35  12000000     3.8
# 2  Citra         IT    24   7000000     4.2
# 3   Dewi         HR    31   9500000     4.0
# 4    Eka    Finance    29  11000000     4.7

# Informasi DataFrame
print(f"Shape: {karyawan.shape}")         # (5, 5)
print(f"Kolom: {list(karyawan.columns)}")
print(f"Tipe data:\n{karyawan.dtypes}")
print(f"Info statistik:\n{karyawan.describe()}")

import pandas as pd

karyawan = pd.DataFrame({
    'Nama': ['Andi', 'Budi', 'Citra', 'Dewi', 'Eka'],
    'Departemen': ['IT', 'Finance', 'IT', 'HR', 'Finance'],
    'Usia': [28, 35, 24, 31, 29],
    'Gaji': [8500000, 12000000, 7000000, 9500000, 11000000],
    'Rating': [4.5, 3.8, 4.2, 4.0, 4.7]
})

# Tambah kolom baru
karyawan['Gaji_Tahunan'] = karyawan['Gaji'] * 12

# Tambah kolom berdasarkan kondisi
import numpy as np
karyawan['Grade'] = np.where(karyawan['Rating'] >= 4.5, 'Top',
                    np.where(karyawan['Rating'] >= 4.0, 'Good', 'Average'))

# Hapus kolom
karyawan_baru = karyawan.drop(columns=['Gaji_Tahunan'])

# Rename kolom
karyawan_baru = karyawan_baru.rename(columns={'Rating': 'Skor_Kinerja'})

# Sort values
print(karyawan_baru.sort_values('Gaji', ascending=False))

# Tambah baris baru
new_row = pd.DataFrame([{
    'Nama': 'Fajar',
    'Departemen': 'IT',
    'Usia': 26,
    'Gaji': 7500000,
    'Skor_Kinerja': 4.3,
    'Grade': 'Good'
}])
karyawan_final = pd.concat([karyawan_baru, new_row], ignore_index=True)
print(karyawan_final)

import pandas as pd
import numpy as np

# DataFrame dengan missing values
df = pd.DataFrame({
    'Nama': ['Andi', 'Budi', 'Citra', 'Dewi', 'Eka'],
    'Usia': [28, np.nan, 24, 31, np.nan],
    'Gaji': [8500000, 12000000, np.nan, 9500000, 11000000],
    'Kota': ['Jakarta', 'Bandung', None, 'Surabaya', 'Medan']
})

# Cek missing values
print(df.isnull().sum())
# Nama    0
# Usia    2
# Gaji    1
# Kota    1

# Hapus baris dengan missing values
df_clean = df.dropna()

# Isi missing values
df_filled = df.copy()
df_filled['Usia'] = df_filled['Usia'].fillna(df_filled['Usia'].mean())
df_filled['Gaji'] = df_filled['Gaji'].fillna(df_filled['Gaji'].median())
df_filled['Kota'] = df_filled['Kota'].fillna('Tidak Diketahui')

print(df_filled)

import pandas as pd

df = pd.DataFrame({
    'Nama': ['Andi', 'Budi', 'Citra', 'Dewi', 'Eka'],
    'Departemen': ['IT', 'Finance', 'IT', 'HR', 'Finance'],
    'Usia': [28, 35, 24, 31, 29],
    'Gaji': [8500000, 12000000, 7000000, 9500000, 11000000],
    'Rating': [4.5, 3.8, 4.2, 4.0, 4.7]
})

# loc — label-based
print(df.loc[0, 'Nama'])               # Andi
print(df.loc[0:2, ['Nama', 'Gaji']])   # Baris 0-2, kolom Nama & Gaji

# iloc — position-based
print(df.iloc[0, 0])                    # Andi (baris 0, kolom 0)
print(df.iloc[0:3, 0:3])               # Baris 0-2, kolom 0-2

# Boolean indexing (filtering)
# Karyawan dengan gaji > 9 juta
high_salary = df[df['Gaji'] > 9000000]
print(high_salary)

# Filter dengan multiple kondisi
# IT departemen DAN rating >= 4.0
it_top = df[(df['Departemen'] == 'IT') & (df['Rating'] >= 4.0)]
print(it_top)

# Filter dengan .isin()
target_dept = df[df['Departemen'].isin(['IT', 'HR'])]
print(target_dept)

# Filter dengan .str.contains()
# Cari nama yang mengandung huruf 'a'
names_with_a = df[df['Nama'].str.contains('a', case=False)]
print(names_with_a)

# .query() — sintaks yang lebih bersih
result = df.query('Departemen == "Finance" and Gaji > 10000000')
print(result)

# Set index
df_indexed = df.set_index('Nama')
print(df_indexed.loc['Andi'])  # Akses baris berdasarkan nama

import pandas as pd
import numpy as np

# Dataset penjualan
np.random.seed(42)
penjualan = pd.DataFrame({
    'Tanggal': pd.date_range('2026-01-01', periods=20, freq='D'),
    'Produk': np.random.choice(['Laptop', 'HP', 'Tablet'], 20),
    'Kota': np.random.choice(['Jakarta', 'Bandung', 'Surabaya'], 20),
    'Jumlah': np.random.randint(1, 10, 20),
    'Harga_Satuan': np.random.choice([500000, 1000000, 800000], 20)
})

penjualan['Total'] = penjualan['Jumlah'] * penjualan['Harga_Satuan']

# GroupBy satu kolom
print("=== Total Penjualan per Produk ===")
produk_stats = penjualan.groupby('Produk')['Total'].agg([
    'sum', 'mean', 'count'
]).round(0)
print(produk_stats)

# GroupBy multiple kolom
print("\n=== Penjualan per Produk per Kota ===")
pivot = penjualan.groupby(['Kota', 'Produk'])['Total'].sum().unstack(fill_value=0)
print(pivot)

# Aggregasi dengan multiple fungsi
print("\n=== Statistik per Kota ===")
kota_agg = penjualan.groupby('Kota').agg({
    'Total': ['sum', 'mean', 'max'],
    'Jumlah': ['sum', 'mean'],
    'Produk': 'count'
})
print(kota_agg)

# Custom aggregation
print("\n=== Top Seller per Kota ===")
def top_product(group):
    return group.groupby('Produk')['Total'].sum().idxmax()

top = penjualan.groupby('Kota').apply(top_product)
print(top)

# Pivot Table — seperti Excel pivot
pivot_table = pd.pivot_table(
    penjualan,
    values='Total',
    index='Kota',
    columns='Produk',
    aggfunc='sum',
    fill_value=0,
    margins=True,           # Tambahkan total
    margins_name='TOTAL'
)
print(pivot_table)

# Pivot dengan multiple aggregasi
pivot_multi = pd.pivot_table(
    penjualan,
    values='Total',
    index='Kota',
    columns='Produk',
    aggfunc=['sum', 'mean'],
    fill_value=0
)
print(pivot_multi)

import pandas as pd

# DataFrame 1: Pesanan
pesanan = pd.DataFrame({
    'order_id': [1, 2, 3, 4, 5],
    'customer_id': [101, 102, 103, 104, 105],
    'produk': ['Laptop', 'HP', 'Tablet', 'Laptop', 'HP'],
    'jumlah': [1, 2, 1, 3, 1]
})

# DataFrame 2: Pelanggan
pelanggan = pd.DataFrame({
    'customer_id': [101, 102, 103, 106, 107],
    'nama': ['Andi', 'Budi', 'Citra', 'Fajar', 'Gina'],
    'kota': ['Jakarta', 'Bandung', 'Surabaya', 'Medan', 'Semarang']
})

# Inner join (default)
inner = pd.merge(pesanan, pelanggan, on='customer_id', how='inner')
print("=== Inner Join ===")
print(inner)
# Hanya customer_id 101, 102, 103 yang muncul (ada di kedua tabel)

# Left join
left = pd.merge(pesanan, pelanggan, on='customer_id', how='left')
print("\n=== Left Join ===")
print(left)
# Semua pesanan ditampilkan, pelanggan yang tidak match = NaN

# Outer join
outer = pd.merge(pesanan, pelanggan, on='customer_id', how='outer')
print("\n=== Outer Join ===")
print(outer)
# Semua data dari kedua tabel

# Merge dengan kolom berbeda
orders_v2 = pesanan.rename(columns={'customer_id': 'cust_id'})
merged = pd.merge(orders_v2, pelanggan, left_on='cust_id', right_on='customer_id')
print(merged)

# Concat (menggabungkan baris)
df1 = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
df2 = pd.DataFrame({'A': [5, 6], 'B': [7, 8]})
combined = pd.concat([df1, df2], ignore_index=True)
print(combined)
#    A  B
# 0  1  3
# 1  2  4
# 2  5  7
# 3  6  8

import pandas as pd

# ========================
# CSV
# ========================
# Baca CSV
df = pd.read_csv('data.csv')

# Baca CSV dengan opsi
df = pd.read_csv(
    'data.csv',
    sep=',',                    # Separator (',' atau ';')
    encoding='utf-8',           # Encoding ('utf-8', 'latin-1')
    index_col=0,                # Kolom pertama sebagai index
    parse_dates=['tanggal'],    # Auto-parse kolom tanggal
    na_values=['N/A', '-', ''], # Nilai yang dianggap NaN
    thousands='.',              # Pemisah ribuan
    decimal=','                 # Pemisah desimal
)

# Simpan ke CSV
df.to_csv(
    'output.csv',
    index=False,          # Jangan simpan index
    encoding='utf-8',
    sep=','
)

# ========================
# EXCEL
# ========================
# Baca Excel
df_excel = pd.read_excel(
    'data.xlsx',
    sheet_name='Sheet1',   # Nama sheet
    header=0               # Baris header
)

# Simpan ke Excel
with pd.ExcelWriter('output.xlsx', engine='openpyxl') as writer:
    df.to_excel(writer, sheet_name='Data', index=False)
    df_excel.to_excel(writer, sheet_name='Lainnya', index=False)

# ========================
# JSON
# ========================
# Baca JSON
df_json = pd.read_json('data.json', orient='records')

# Simpan ke JSON
df.to_json('output.json', orient='records', indent=2, force_ascii=False)

# ========================
# SQL (SQLite contoh)
# ========================
import sqlite3
conn = sqlite3.connect('database.db')

# Baca dari SQL
df_sql = pd.read_sql('SELECT * FROM karyawan', conn)

# Tulis ke SQL
df.to_sql('karyawan_baru', conn, if_exists='replace', index=False)

conn.close()

# ========================
# PARQUET (format efisien untuk data besar)
# ========================
df.to_parquet('data.parquet', engine='pyarrow')
df_parquet = pd.read_parquet('data.parquet')