Python CSV Processing: Panduan Lengkap

# File: data_karyawan.csv
nama,departemen,gaji,tanggal_masuk
Budi Santoso,Engineering,15000000,2023-01-15
Siti Rahayu,Marketing,12000000,2023-03-20
Ahmad Hidayat,Engineering,18000000,2022-06-01
Dewi Lestari,HR,11000000,2023-08-10
Rizki Pratama,Marketing,13000000,2024-01-05

# Masalah umum CSV:
# 1. Nilai dengan koma: "Jakarta, Indonesia"
# 2. Nilai dengan kutip: 'Dia berkata "Halo"'
# 3. Baris baru dalam nilai: "Baris 1\nBaris 2"
# 4. Delimiter berbeda: tab, semikolon, pipe
# 5. Encoding berbeda: UTF-8, Latin-1, CP1252

import csv

# Membaca CSV dengan csv.reader
with open('data_karyawan.csv', 'r', newline='', encoding='utf-8') as f:
    reader = csv.reader(f)
    
    # Baris pertama adalah header
    header = next(reader)
    print(f"Header: {header}")
    # ['nama', 'departemen', 'gaji', 'tanggal_masuk']
    
    # Sisa baris adalah data
    for row in reader:
        print(f"  {row}")
    # ['Budi Santoso', 'Engineering', '15000000', '2023-01-15']
    # ['Siti Rahayu', 'Marketing', '12000000', '2023-03-20']
    # ...

# Mengubah ke list
with open('data_karyawan.csv', 'r', newline='', encoding='utf-8') as f:
    reader = csv.reader(f)
    semua_data = list(reader)

print(f"Total baris (termasuk header): {len(semua_data)}")
print(f"Header: {semua_data[0]}")
print(f"Data pertama: {semua_data[1]}")

import csv

# CSV dengan delimiter titik koma (umum di Eropa/Indonesia)
# data_indonesia.csv:
# nama;kelas;nilai
# Budi;X-A;85
# Siti;X-B;92

with open('data_indonesia.csv', 'r', newline='', encoding='utf-8') as f:
    reader = csv.reader(f, delimiter=';', quotechar='"')
    for row in reader:
        print(row)
# ['nama', 'kelas', 'nilai']
# ['Budi', 'X-A', '85']
# ['Siti', 'X-B', '92']

# CSV dengan tab delimiter (TSV)
with open('data.tsv', 'r', newline='', encoding='utf-8') as f:
    reader = csv.reader(f, delimiter='\t')
    for row in reader:
        print(row)

# Register custom dialect
csv.register_dialect('pipe_delimited', delimiter='|', quoting=csv.QUOTE_MINIMAL)

with open('data_pipe.csv', 'r', newline='') as f:
    reader = csv.reader(f, dialect='pipe_delimited')
    for row in reader:
        print(row)

# List semua dialect yang terdaftar
print(csv.list_dialects())  # ['excel', 'excel-tab', 'unix', 'pipe_delimited']

import csv

# Menulis CSV dari list
data = [
    ['nama', 'departemen', 'gaji'],
    ['Budi Santoso', 'Engineering', 15000000],
    ['Siti Rahayu', 'Marketing', 12000000],
    ['Ahmad Hidayat', 'Engineering', 18000000],
]

with open('output_karyawan.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f)
    writer.writerows(data)  # Tulis semua sekaligus

# Atau satu baris sekaligus
with open('output_karyawan2.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f)
    writer.writerow(['nama', 'departemen', 'gaji'])  # Header
    writer.writerow(['Budi Santoso', 'Engineering', 15000000])
    writer.writerow(['Siti Rahayu', 'Marketing', 12000000])

# Menggunakan quoting
with open('output_quoted.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f, quoting=csv.QUOTE_ALL)
    writer.writerow(['nama', 'kota', 'alamat'])
    writer.writerow(['Budi', 'Jakarta', 'Jl. Sudirman No. 1'])
# Output: "Budi","Jakarta","Jl. Sudirman No. 1"

# QUOTE_NONNUMERIC — quote semua non-angka
with open('output_mixed.csv', 'w', newline='') as f:
    writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
    writer.writerow(['nama', 'umur', 'kota'])
    writer.writerow(['Budi', 25, 'Jakarta'])
# Output: "Budi",25,"Jakarta"

import csv

# DictReader — setiap baris jadi OrderedDict/dict
with open('data_karyawan.csv', 'r', newline='', encoding='utf-8') as f:
    reader = csv.DictReader(f)
    
    # Akses dengan nama kolom (bukan index!)
    for row in reader:
        print(f"  {row['nama']} | {row['departemen']} | Rp{int(row['gaji']):,}")
    # Budi Santoso | Engineering | Rp15,000,000
    # Siti Rahayu | Marketing | Rp12,000,000

# Filter data
with open('data_karyawan.csv', 'r', newline='', encoding='utf-8') as f:
    reader = csv.DictReader(f)
    engineers = [
        row for row in reader 
        if row['departemen'] == 'Engineering'
    ]
    
    for emp in engineers:
        print(f"  {emp['nama']} - Gaji: Rp{int(emp['gaji']):,}")


# DictWriter — tulis dari dict
karyawan_baru = [
    {'nama': 'Rizki Pratama', 'departemen': 'IT', 'gaji': '14000000', 'tanggal_masuk': '2024-06-01'},
    {'nama': 'Ayu Putri', 'departemen': 'Finance', 'gaji': '13500000', 'tanggal_masuk': '2024-07-15'},
]

with open('karyawan_baru.csv', 'w', newline='', encoding='utf-8') as f:
    fieldnames = ['nama', 'departemen', 'gaji', 'tanggal_masuk']
    writer = csv.DictWriter(f, fieldnames=fieldnames)
    
    writer.writeheader()  # Tulis header
    writer.writerows(karyawan_baru)  # Tulis data

# Output:
# nama,departemen,gaji,tanggal_masuk
# Rizki Pratama,IT,14000000,2024-06-01
# Ayu Putri,Finance,13500000,2024-07-15

import pandas as pd

# Baca CSV dasar
df = pd.read_csv('data_karyawan.csv')
print(df.head())
#            nama  departemen      gaji tanggal_masuk
# 0  Budi Santoso  Engineering  15000000    2023-01-15
# 1    Siti Rahayu    Marketing  12000000    2023-03-20

print(df.shape)          # (5, 4) — 5 baris, 4 kolom
print(df.columns.tolist())  # ['nama', 'departemen', 'gaji', 'tanggal_masuk']
print(df.dtypes)         # Tipe data per kolom

# Baca dengan parameter kustom
df = pd.read_csv(
    'data.csv',
    sep=';',                    # Delimiter titik koma
    encoding='utf-8',           # Encoding
    header=0,                   # Baris ke-0 sebagai header
    names=['A', 'B', 'C'],     # Custom column names (jika tidak ada header)
    index_col='nama',          # Kolom 'nama' sebagai index
    usecols=['nama', 'gaji'],  # Hanya baca kolom tertentu
    dtype={'gaji': float},     # Tentukan tipe data
    parse_dates=['tanggal'],   # Auto-parse tanggal
    na_values=['N/A', ''],     # Nilai yang dianggap NaN
    skiprows=[1, 3],           # Skip baris tertentu
    nrows=100,                 # Hanya baca 100 baris pertama
    skip_blank_lines=True,     # Skip baris kosong
)

# Baca dari URL
df = pd.read_csv('https://example.com/data.csv')

# Baca dari string
csv_text = """nama,umur,kota
Budi,25,Jakarta
Siti,30,Bandung"""
df = pd.read_csv(pd.io.common.StringIO(csv_text))
print(df)

import pandas as pd

df = pd.read_csv('data_karyawan.csv')

# --- Filtering ---
engineering = df[df['departemen'] == 'Engineering']
gaji_tinggi = df[df['gaji'] > 13000000]
multi_filter = df[(df['departemen'] == 'Engineering') & (df['gaji'] > 15000000)]

# --- Sorting ---
df_sorted = df.sort_values('gaji', ascending=False)
df_multi = df.sort_values(['departemen', 'gaji'], ascending=[True, False])

# --- Aggregation ---
print(f"Rata-rata gaji: Rp{df['gaji'].mean():,.0f}")
print(f"Gaji max: Rp{df['gaji'].max():,.0f}")
print(f"Gaji min: Rp{df['gaji'].min():,.0f}")

# Group by
dept_stats = df.groupby('departemen').agg({
    'gaji': ['mean', 'min', 'max', 'count'],
    'nama': 'count'
}).round(0)
print(dept_stats)

# --- Kolom Baru ---
df['gaji_tahunan'] = df['gaji'] * 13  # Termasuk THR
df['departemen_upper'] = df['departemen'].str.upper()

# --- Pivot Table ---
pivot = pd.pivot_table(
    df,
    values='gaji',
    index='departemen',
    aggfunc=['mean', 'count']
)

# --- Filtering string ---
nama_budi = df[df['nama'].str.contains('Budi', case=False)]
dept_starting_e = df[df['departemen'].str.startswith('E')]

# --- Menghapus duplikat ---
df_unique = df.drop_duplicates(subset=['departemen'], keep='first')

# --- Menghapus kolom ---
df_dropped = df.drop(columns=['tanggal_masuk'])

print(df.head())

import pandas as pd

df = pd.read_csv('data_karyawan.csv')

# Tulis ke CSV
df.to_csv('output.csv', index=False, encoding='utf-8')

# Tulis dengan separator titik koma
df.to_csv('output_semicolon.csv', sep=';', index=False)

# Tulis hanya kolom tertentu
df.to_csv('subset.csv', columns=['nama', 'gaji'], index=False)

# Tulis tanpa index
df.to_csv('no_index.csv', index=False)

# Tulis sebagian data
df.head(10).to_csv('first_10.csv', index=False)

# Tulis ke string (bukan file)
csv_string = df.to_csv(index=False)
print(csv_string)

# Tulis dengan format angka kustom
df['gaji_formatted'] = df['gaji'].apply(lambda x: f"Rp{x:,.0f}")
df.to_csv('formatted.csv', index=False)

import pandas as pd
import csv

# Pandas: baca per chunk
chunk_size = 10000
total_rows = 0
total_gaji = 0

for chunk in pd.read_csv('data_besar.csv', chunksize=chunk_size):
    total_rows += len(chunk)
    total_gaji += chunk['gaji'].sum()
    # Proses chunk di sini
    print(f"  Processed {total_rows} rows...")

print(f"Total: {total_rows} baris, Total Gaji: Rp{total_gaji:,.0f}")


# csv module: baca baris per baris (sangat memory-efficient)
total = 0
with open('data_besar.csv', 'r', newline='', encoding='utf-8') as f:
    reader = csv.reader(f)
    header = next(reader)
    for row in reader:
        total += 1
        # Proses setiap baris di sini

print(f"Total baris: {total}")

import pandas as pd

# Hanya baca kolom yang dibutuhkan
df = pd.read_csv(
    'data_besar.csv',
    usecols=['nama', 'gaji'],  # Hanya 2 dari 50+ kolom
    dtype={'gaji': float},     # Tentukan tipe untuk efisiensi memory
)

# Optimasi memory
df = pd.read_csv('data_besar.csv')
print(f"Memory usage: {df.memory_usage(deep=True).sum() / 1024 / 1024:.1f} MB")

# Konversi kolom untuk hemat memory
df['departemen'] = df['departemen'].astype('category')  # String → Category
df['gaji'] = pd.to_numeric(df['gaji'], downcast='float')  # float64 → float32

print(f"Memory setelah optimasi: {df.memory_usage(deep=True).sum() / 1024 / 1024:.1f} MB")

import chardet

# Deteksi encoding file
with open('misteri.csv', 'rb') as f:
    raw_data = f.read(10000)  # Baca 10KB pertama
    result = chardet.detect(raw_data)
    print(result)
    # {'encoding': 'UTF-8-SIG', 'confidence': 0.99, 'language': ''}

# Atau gunakan fallback chain
def baca_csv_aman(filepath):
    """Baca CSV dengan auto-detect encoding."""
    encodings = ['utf-8', 'utf-8-sig', 'latin-1', 'cp1252', 'iso-8859-1']
    
    for enc in encodings:
        try:
            df = pd.read_csv(filepath, encoding=enc)
            print(f"  Berhasil dengan encoding: {enc}")
            return df
        except UnicodeDecodeError:
            continue
    
    # Fallback: deteksi otomatis
    with open(filepath, 'rb') as f:
        detected = chardet.detect(f.read())
    df = pd.read_csv(filepath, encoding=detected['encoding'])
    print(f"  Auto-detected: {detected['encoding']} (confidence: {detected['confidence']})")
    return df

df = baca_csv_aman('data_indonesia.csv')

import pandas as pd
import numpy as np

# Simulasi data kotor
data = {
    'nama': ['Budi', 'Siti', None, 'Dewi ', ' Rizki', 'Budi'],
    'gaji': ['15jt', '12.000.000', 'N/A', '11jt', '13,5jt', '15jt'],
    'email': ['budi@email.com', 'siti@', 'invalid', 'dewi@email.com', '', 'budi@email.com'],
    'umur': [25, 30, -1, 28, 200, 25],
}
df = pd.DataFrame(data)

# 1. Hapus duplikat
print(f"Sebelum: {len(df)} baris")
df = df.drop_duplicates()
print(f"Sesudah: {len(df)} baris")

# 2. Handle missing values
df['nama'] = df['nama'].fillna('Tidak Diketahui')
df['nama'] = df['nama'].str.strip()  # Hapus spasi di awal/akhir

# 3. Parse gaji yang formatnya beragam
def parse_gaji(val):
    """Parse berbagai format gaji ke angka."""
    if pd.isna(val) or val == 'N/A':
        return np.nan
    val = str(val).strip()
    val = val.replace('.', '').replace(',', '')
    if 'jt' in val:
        val = val.replace('jt', '').strip()
        return float(val) * 1_000_000
    try:
        return float(val)
    except ValueError:
        return np.nan

df['gaji'] = df['gaji'].apply(parse_gaji)

# 4. Validasi email
import re
def validasi_email(email):
    if not email or pd.isna(email):
        return np.nan
    pattern = r'^[\w\.-]+@[\w\.-]+\.\w+$'
    return email if re.match(pattern, str(email)) else np.nan

df['email'] = df['email'].apply(validasi_email)

# 5. Clamp outlier
df.loc[df['umur'] > 100, 'umur'] = np.nan
df.loc[df['umur'] < 0, 'umur'] = np.nan

# 6. Tampilkan hasil
print(df)
print(f"\nNull counts:\n{df.isnull().sum()}")