A comprehensive guide to data cleaning and efficient analysis in Pandas

import pandas as pd

# Read large data sets in chunkschunk_iter = pd.read_csv('large_dataset.csv', chunksize=100000)
for chunk in chunk_iter:
    process(chunk)  # Custom processing functions
# Read only the required columnscols = ['id', 'name', 'value']
df = pd.read_csv('', usecols=cols)

# Specify data type to reduce memory usagedtypes = {'id': 'int32', 'price': 'float32'}
df = pd.read_csv('', dtype=dtypes)

# Display statistics for all columns (including non-numeric columns)(include='all')

# Check memory usage(memory_usage='deep')
# Advanced display of unique values ​​and their countsfor col in df.select_dtypes(include=['object']).columns:
    print(f"\n{col}Value distribution:")
    print(df[col].value_counts(dropna=False).head(10))

# Visualize missing valuesimport missingno as msno
(df)

# Fill in missing values ​​based on rulesdf['salary'] = ('department')['salary'].apply(
    lambda x: (())
)

# Create missing value indicator featuredf['age_missing'] = df['age'].isna().astype(int)

# Use IQR method to detect outliersdef detect_outliers(df, col):
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    return ~df[col].between(lower_bound, upper_bound)

outliers = detect_outliers(df, 'price')
df['price_cleaned'] = (outliers, , df['price'])

# Use Z-score to handle outliersfrom scipy import stats
df['z_score'] = ((df['value']))
df['value_cleaned'] = (df['z_score'] > 3, , df['value'])

# Use regular expressions to extract informationdf['phone_area'] = df['phone'].(r'\((\d{3})\)')

# Vectorized string operationsdf['name'] = df['first_name'].(df['last_name'], sep=' ')

# Use fuzzywuzzy for fuzzy matchingfrom fuzzywuzzy import fuzz
df['similarity'] = (
    lambda x: (x['name1'], x['name2']), axis=1
)

# Calculate multiple aggregate functions simultaneouslyagg_funcs = {
    'sales': ['sum', 'mean', 'max'],
    'profit': lambda x: (x > 0).mean()  # Profit ratio}
result = ('region').agg(agg_funcs)

# Use transform to maintain original DataFrame shapedf['dept_avg_salary'] = ('department')['salary'].transform('mean')

# Use pivot_table for perspectivepd.pivot_table(df, values='sales', index='region',
               columns='quarter', aggfunc=,
               margins=True, margins_name='total')

# Quick index-based merge(df2, how='left')

# Use merge's indicator parameter to track merge sources(df1, df2, on='key', how='outer', indicator=True)

# Use concat for axial merge([df1, df2], axis=1, keys=['2022', '2023'])

# Resampling and scrolling windowdf.set_index('date').resample('W').mean()  # Resample by week(window='30D').mean()  # 30-day rolling average
# Processing time zonedf['timestamp'] = df['timestamp'].dt.tz_localize('UTC').dt.tz_convert('Asia/Shanghai')

# Time Characteristic Engineeringdf['hour'] = df['timestamp'].
df['is_weekend'] = df['timestamp']. >= 5

# Automatically optimize data typesdef optimize_dtypes(df):
    for col in :
        col_type = df[col].dtype
        
        if col_type == 'object':
            num_unique = df[col].nunique()
            if num_unique / len(df) < 0.5:
                df[col] = df[col].astype('category')
                
        elif col_type == 'float64':
            df[col] = pd.to_numeric(df[col], downcast='float')
            
        elif col_type == 'int64':
            df[col] = pd.to_numeric(df[col], downcast='integer')
    
    return df

​​​​​​​df = optimize_dtypes(df)

# Use swifter to speed up apply operationsimport swifter
df['new_col'] = df['text'].(process_text)

# Use modin instead of pandas to implement parallel processingimport  as mpd
df = mpd.read_csv('large_file.csv')

# Performance comparison of various iterative methodsdef iterrows_example(df):
    for index, row in ():
        process(row)

def itertuples_example(df):
    for row in ():
        process(row)

def vectorized_example(df):
    df['new_col'] = df['col1'] + df['col2']
    
# Vectorization operations are usually 100-1000 times faster than iteration

# 1. Data loading and preliminary cleaningdf = pd.read_csv('', parse_dates=['order_date'])
df = df[df['order_amount'] > 0]  # Filter invalid orders
# 2. RFM Analysissnapshot_date = df['order_date'].max() + (days=1)
rfm = ('customer_id').agg({
    'order_date': lambda x: (snapshot_date - ()).days,
    'order_id': 'count',
    'order_amount': 'sum'
})
 = ['recency', 'frequency', 'monetary']

# 3. RFM binning and ratingrfm['R_score'] = (rfm['recency'], 5, labels=[5,4,3,2,1])
rfm['F_score'] = (rfm['frequency'], 5, labels=[1,2,3,4,5])
rfm['M_score'] = (rfm['monetary'], 5, labels=[1,2,3,4,5])
rfm['RFM_score'] = rfm[['R_score','F_score','M_score']].sum(axis=1)

# 4. Customer Layoutseg_map = {
    r'[12-15]': 'High value customers',
    r'[9-11]': 'Potential Client',
    r'[6-8]': 'General Customer',
    r'[3-5]': 'Cause Risk Customer'
}
rfm['segment'] = rfm['RFM_score'].astype('str').replace(seg_map, regex=True)

('result = (col1 + col2) / col3', inplace=True)