Project: Hypothesis Testing with Men's and Women's Soccer Matches

# Imports
import pandas as pd
import matplotlib.pyplot as plt
import pingouin
from scipy.stats import mannwhitneyu

# Load the CSV file into a DataFrame
df_one = pd.read_csv('men_results.csv')
df_two = pd.read_csv('women_results.csv')

# Print the first few rows of the DataFrames
pd.set_option('display.max_columns', None)
print(df_two.head(10))
print(df_two.info())
print(df_two.value_counts())

#Exploratory data analaysis including data cleaning and shaping 

# Convert the 'date' column to datetime format for both mens and womens data
df_one['date'] = pd.to_datetime(df_one['date'])
df_two['date'] = pd.to_datetime(df_two['date'])

# Filter the DataFrame using the .isin() method for both mens and womens data
filtered_df_one = df_one[df_one['tournament'].isin(['FIFA World Cup']) & (df_one['date'] > '2002-01-01')]
filtered_df_two = df_two[df_two['tournament'].isin(['FIFA World Cup']) & (df_two['date'] > '2002-01-01')]

# Display the filtered DataFrame
pd.set_option('display.max_columns', None)
print(filtered_df_one.head(5))
print(filtered_df_one.describe())
print(filtered_df_one.info())

# Create group and goals_scored columns for mens data
filtered_df_one["group"] = "men"
filtered_df_one["goals_scored"] = filtered_df_one["home_score"] + filtered_df_one["away_score"]

# Create group and goals_scored columns for womens data
filtered_df_two["group"] = "women"
filtered_df_two["goals_scored"] = filtered_df_two["home_score"] + filtered_df_two["away_score"]

# Determine normality using histograms (mens)
filtered_df_one["goals_scored"].hist()
plt.show()
plt.clf()

# Determine normality using histograms (women)
filtered_df_two["goals_scored"].hist()
plt.show()
plt.clf()

# Not normally distributed so will use the wilcoxon-mann-whitney test of two groups 

# Combining the filtered men's and women's FIFA World Cup data into a single DataFrame
# allows for direct comparison and analysis between the two groups.
# This is especially useful for statistical tests (e.g., Mann-Whitney U test)
# and for creating visualizations that compare distributions or summary statistics
# of goals scored in men's vs. women's matches.

both = pd.concat([filtered_df_two, filtered_df_one], axis=0, ignore_index=True)
pd.set_option('display.max_columns', None)
print(both.head())

# The transformation is done to prepare the data for the Mann-Whitney U test using pingouin,
# which expects the data in a wide format: one column per group (e.g., 'men', 'women'),
# with each row representing a match's goals scored for that group.
# This allows the test to compare the distributions of goals scored between the two groups.

both_subset = both[["goals_scored", "group"]] # this is in long format
both_subset_wide = both_subset.pivot(columns="group", values="goals_scored") # this is the wide format
both_subset_wide

# The right-tailed Wilcoxon-Mann-Whitney test (alternative="greater") is used here to test
# whether the distribution of goals scored in women's matches is statistically greater than
# that in men's matches. This non-parametric test is appropriate because the data were found
# to be not normally distributed (as shown by the histograms). The test compares the medians
# of two independent groups (women vs. men) without assuming normality, making it suitable
# for this scenario where we want to know if women's matches tend to have more goals scored
# than men's matches.
results_pg = pingouin.mwu(x=both_subset_wide["women"],
                          y=both_subset_wide["men"],
                          alternative="greater")

# Alternative SciPy solution: Perform right-tailed Wilcoxon-Mann-Whitney test with scipy
results_scipy = mannwhitneyu(x=filtered_df_two["goals_scored"],
                             y=filtered_df_one["goals_scored"],
                             alternative="greater")

# The P-value is a measure used in hypothesis testing to help determine the statistical significance of the observed results.
# It represents the probability of obtaining test results at least as extreme as the observed results, assuming that the null hypothesis is true.
# In this context, the null hypothesis is that the distribution of goals scored in women's matches is not greater than that in men's matches.
# A small P-value (typically less than 0.05) indicates strong evidence against the null hypothesis, suggesting that women's matches tend to have more goals scored than men's matches.
# Conversely, a large P-value suggests that there is not enough evidence to conclude that women's matches have more goals than men's matches.

p_val  # This is the P-value from the right-tailed Wilcoxon-Mann-Whitney test comparing women's and men's match goals.

# Extract p-value as a float
p_val = results_pg["p-val"].values[0]

# Determine hypothesis test result using sig. level
if p_val <= 0.01:
    result = "reject"
else:
    result = "fail to reject"

result_dict = {"p_val": p_val, "result": result}