Python - Bank Marketing

import pandas as pd

df = pd.read_csv("bank-marketing.csv", sep=";")

# Preview dataframe
print(df.head())

# To see how many columns and rows
df.shape

# To view dataframe
df

# Get summary info
df.info()

# To get descriptive summary
df.describe()

# Import necessary packages
import matplotlib.pyplot as plt
from scipy import stats

# To check for outliers we can use Z-scores
# Calculate the Z-scores
z_scores = stats.zscore(df['campaign'])

# Define outliers as Z-scores greater than 3 or less than -3
outliers = df[abs(z_scores) > 3]

print("Outliers based on Z-score:")
print(outliers)
      
# Here we see that we have outliers, let us first see the performance of the model with outliers.

# We can also identify the outliers using IQR
Q1 = df["campaign"].quantile(0.25)
Q3 = df["campaign"].quantile(0.75)
IQR = Q3 - Q1

# Define bounds
lower = Q1 - 1.5 * IQR
upper = Q3 + 1.5 * IQR

# Remove outliers
clean_df = df[(df['campaign'] >= lower) & (df['campaign'] <= upper)]

clean_df

# Based on the info above, there is no null values
# But to look for null values
df.isnull().sum()
	# No null values

# To look for duplicates
dup = df.duplicated()

print(dup[dup] == True)
	# there are duplicated rows
    
df = df.drop_duplicates() # remove duplicated rows

df

df['job'].unique() # to check the entries under job

# Let us group by job and subscription, and count the occurrences
j_y_distribution = df.groupby(['job', 'y']).size().unstack().fillna(0)

# Create a bar chart for the distribution
j_y_distribution.plot(kind='bar', stacked=False)

# Customizing plot
plt.title('Job vs Subscription')
plt.xlabel('Job')
plt.ylabel('Frequency')
plt.xticks(rotation=45, ha='right')
plt.legend(title='Job Titles', bbox_to_anchor=(1.05, 1), loc='upper left')

# Show the plot
plt.show()

# Let us look at other variables by creating a heatmap
# But first let us perform one-hot encoding for 'y'
df['y'].replace(['no', 'yes'], [0, 1], inplace=True)
df['job'].replace(['housemaid', 'services', 'admin.', 'blue-collar', 'technician',
       'retired', 'management', 'unemployed', 'self-employed', 'unknown',
       'entrepreneur', 'student'], list(range(12)), inplace=True)

import seaborn as sns
                  
# Now let us create a heatmap
sns.heatmap(df.corr(), cmap="coolwarm", annot=True, fmt=".2f")
plt.title("Correlation Heatmap")
plt.show()

# Here we can see duration has a positive moderate relationship with y while others are considered weak

# Now let us see the trend of customer's subscription by creating a line garph
# However, let us first arranged the months
# Define the correct month order
month_order = ['may', 'jun', 'jul', 'aug', 'oct', 'nov', 'dec', 'mar', 'apr', 'sep']

# Sorting the months
df['month'] = pd.Categorical(df['month'], categories = month_order, ordered = True)

# Filter the dataframe for only those who are subscribed
df_y = df[df['y'] == 1]

# Sort the month
df_y = df_y.sort_values('month')

# Count the number of subscriptions per month
subscription_counts = df_y.groupby(['month', 'y']).size().unstack().fillna(0)
                        

# Assuming subscription_counts is already defined
subscription_counts.plot(kind='line', figsize=(10, 6))

# Adding labels and title
plt.title('Monthly Subscription Trend Each Month')
plt.xlabel('Month')
plt.ylabel('Number of Subscriptions')
plt.legend(title='Year', bbox_to_anchor=(1.05, 1), loc='upper left')

# Displaying the plot
plt.tight_layout()
plt.show()

# We need to identify if there is an association between the y variable and other variables

# But let us first check if the last campaign has an impact to the customers subscription

# Here we can use ANOVA or T-test
# However, we need to check for assumptions
# 2 groups: subscribed and not subscribed customers are independent groups

import seaborn as sns

# To check for normality
# Extract the 2 groups
df_yes = df[df['y'] == 1] ['campaign']
df_no = df[df['y'] == 0]['campaign']


# Shapiro-wilk test
shapiro_stat1, shapiro_p_value1 = stats.shapiro(df_yes)
print(f'Shapiro-Wilk Test: statistic = {shapiro_stat1}, p-value = {shapiro_p_value1}')

if shapiro_p_value1 > 0.05:
    print("The 'Subscribed' group follows a normal distribution. ")
    #fail to reject null hypothesis
else:
    print("The 'Subscribed' group does not follow a normal distribution.")
    #reject null
    
shapiro_stat2, shapiro_p_value2 = stats.shapiro(df_no)
print(f'Shapiro-Wilk Test: statistic = {shapiro_stat2}, p-value = {shapiro_p_value2}')

if shapiro_p_value2 > 0.05:
    print("The 'Not subscribed' group follows a normal distribution. ")
    #fail to reject null hypothesis
else:
    print("The 'Not subscribed' group does not follow a normal distribution.")
    #reject null
    
# Histogram
plt.figure(figsize=(8, 6))
sns.histplot(df_yes, kde=True)
plt.title('Histogram for Subscribed Group')
plt.show()

plt.figure(figsize=(8, 6))
sns.histplot(df_no, kde=True)
plt.title('Histogram for Not Subscribed Group')
plt.show()
    
# Since it does not follow a normal distribution we can't use ANOVA or T-test