Measles Data

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

measles = pd.read_csv("data/measles.csv")
print(measles.shape)
measles.head(10)

# Initial exploratory analysis

measles.describe()
measles.shape
# -1 is used for missing data in overall and mmr vaccination rate columns

# Noticed in .head() that Arizona is the first state
# alphabetically should be Alabama
measles[measles['state'] == 'Alabama'] # no data for Alabama
# how many schools from each state?
measles['state'].value_counts() # good to keep in mind based on state population
# how many states are included?
print(measles['state'].nunique(), 'unique states are included in this dataset.') # 32 states only

# missing data
measles.isna().sum().plot(kind = 'bar', title = 'Missing Data, by Column', rot = 45)
plt.xlabel('Column Name')
plt.ylabel('Observation Count')

# Q1: Which states have the highest average mmr vaccination rates?

## Missing Data & Cleaning

# I noticed that some states have an average mmr vax rate of -1, likely due to missing data
# let's drop those for now and do further analysis on them later
measles_f = measles[measles['mmr']>=0] # subsetting only valid vax rates
print(measles_f[['state','name','mmr']].sort_values(by = 'mmr',ascending = True))
# now all mmr vaccination rates are positive, so we can proceed
print('The number of states with data for MMR vaccination rates is', measles_f['state'].nunique()) # check we still have 32 states

# only 21 states left: this means that 11 states had only missing data for mmr vaccination rates
# which states were excluded? (i.e. have no data available for mmr vax rates)
states_list = measles['state'].unique() # unique states in original dataset
states_f_list = measles_f['state'].unique() # unique states after removing missing data
states_m_list = [] # empty list of missing states

# using for loop to find missing states 
for i in states_list:
    if i not in states_f_list:
        states_m_list.append(i)
# print missing states
print('States without MMR vaccination rate data are', states_m_list)

# Grouping by state and taking mean mmr vaccination rate
measles_states = measles_f.groupby('state')['mmr'].mean()
measles_states = measles_states.to_frame()
# sorting in descending order
sort_measles_states = measles_states.sort_values(by = 'mmr',ascending = False)

# Visualizations
measles_states.plot(kind = 'bar', title = 'Average MMR Vaccination Rate, by State', rot = 75)
plt.ylim(75,100)
plt.xlabel('State')
plt.show()

## Trends in Types of Schools w/MMR vax data

# mean vaccination rate by school type (only mmr >= 0)
measles_type = measles_f.groupby('type')['mmr'].mean()
measles_type = measles_type.to_frame()

# visualization
measles_type.plot(kind = 'bar', title = 'Average MMR Vaccination Rate, by School Type', rot = 45)
plt.xlabel('School Type')
plt.ylabel('Vaccination Rate (%)')
plt.show()

# BOCES schools have the highest average MMR vaccination rate, likely because these schools are only in NY which
# has a high vaccination rate as a whole. Charter schools have slightly lower MMR vaccination rates than other 
# school types, but this difference doesn't seem to be significant.

# Are certain types of schools more likely to have missing data on MMR vaccination rates?

# take all missing MMR data
measles_missing = measles[measles['mmr'] < 0]
# counting missing data by school type. divide by total of each type
measles_type['missing'] = measles_missing['type'].value_counts() / measles['type'].value_counts() * 100
# replacing NaN with 0
measles_type_filled = measles_type.fillna(0)
# plotting
measles_type_filled.plot(kind = 'bar', y = 'missing', title = 'Percentage (%) Missing Data on MMR Vaccination Rates, by School Type', rot = 45)
plt.ylabel('Percentage of MMR Data Missing')
plt.xlabel('School Type')
plt.show()

# In 85% of nonpublic schools, data on MMR vaccination rates is missing. This means our previous finding that
# nonpublic schools have a similar MMR vaccination rate to private and public schools may be inaccurate.
# Additionally, as only ~ 20% of charter schools are missing data on MMR vax rates, the previous finding that charter
# schools have lower average vaccination rates is relatively accurate.

# Nonpublic and private schools have the highest rates of missing data for MMR vaccination rates, which makes sense
# since these schools are not bound to the same legal vaccination & reporting standards as public schools.

## Investigating States with Missing MMR Vaccination data
# Q3: Do states with missing MMR data share any common traits?

# subsetting for states without mmr data (i.e. mmr = -1)
measles_m = measles[measles['state'].isin(states_m_list)].iloc[:,1:14]
print(measles_m.head())
# check what other data is missing (looks like xrel, xmed, xper - makes sense since based on mmr)
measles_m.isna().any() # missing year, type, city, county, and district data too

# what percentage is missing?
print(round(measles_m['city'].isna().sum() / len(measles_m['city']) *100, 2), '% of schools are missing data on city.')
print(round(measles_m['type'].isna().sum() / len(measles_m['type']) *100, 2), '% of schools are missing data on school type.')
print(round(measles_m['year'].isna().sum() / len(measles_m['year']) *100, 2), '% of schools are missing data on year.')
print(round(measles_m['county'].isna().sum() / len(measles_m['county']) *100, 2), '% of schools are missing data on county.')
print(round(measles_m['district'].isna().sum() / len(measles_m['district']) *100, 2), '% of schools are missing data on district.')

# All schools in states that do not have any data on MMR vaccination rates are also missing data on the type of school (public, private, charter) and on which school district they belong to, with most (78%) also missing data on their city, and few (17%, 21%) missing data on county and year, respectively.
# Would be interesting to investigate co-missingness trends!

# Predicting Vaccination Rates