Associate Data Scientist Preparation

# Use this cell to write your code for Task 1

import pandas as pd
import numpy as np

df = pd.read_csv("house_sales.csv")
print(df.head())
##### Data inspection #####

print("\n"* 2,' BREAK '.center(80, '#') ,"\n" * 2)
print(df.info())
print("\n"* 2,' BREAK '.center(80, '#') ,"\n" * 2)
print(df.isna().sum())
print("\n"* 2,' BREAK '.center(80, '#') ,"\n" * 2)
print(df["city"].value_counts(dropna=False))
missing_city = 73
print(missing_city)

# Use this cell to write your code for Task 2
print(df.info())

### replacing missing values in "city" and converting the column to categories
work_df = df.copy()

work_df.loc[work_df["city"] == "--", "city"] = "Unknown"
work_df["city"] = work_df["city"].astype("category")
work_df["house_id"] = work_df["house_id"].astype("category")

### Checking the date column, converting all entries to a datetime object worked without a problem

work_df["sale_date"] = pd.to_datetime(work_df["sale_date"],errors="coerce")

### inspectin Sale price

print(work_df["sale_price"].describe())

### Imputing missing values, I could have used fillna() but I wanted more controll this time

mean_months_listed = work_df["months_listed"].mean().round(1)
work_df.loc[work_df["months_listed"].isna(), "months_listed"] = mean_months_listed
work_df["months_listed"] = work_df["months_listed"].round(1)

### Checking the house type unique values and changing them to the corresponding category
house_types = work_df["house_type"].unique()
print(house_types)
print(work_df["house_type"].value_counts())

work_df["house_type"].replace("Det.", "Detached", inplace=True)
work_df["house_type"].replace("Semi", "Semi-detached", inplace=True)
work_df["house_type"].replace("Terr.", "Terraced", inplace=True)
df['house_type'] = pd.Categorical(
    df['house_type'],
    categories=['Terraced', 'Semi-detached', 'Detached'],
    ordered=True
)
work_df["house_type"] = work_df["house_type"].astype("category")

print(work_df["house_type"].value_counts())

### Last column, from the info() I could already see that something is wrong with this continous variable. I have to get rid of the unit

work_df["area"] = work_df["area"].replace(r'\D',"",regex=True).astype(float).round(1)
print(work_df["area"].value_counts())

# In the end, I assigned the adapted dataframe to the desired variable clean_data
clean_data = work_df

clean_data.info()

# Use this cell to write your code for Task 3


price_by_rooms = work_df.groupby('bedrooms')['sale_price'].agg(
    avg_price='mean', 
    var_price='var'
).reset_index()

price_by_rooms = price_by_rooms.round({'avg_price': 1, 'var_price': 1})


print(price_by_rooms)

# Use this cell to write your code for Task 4

#I decided to use a DecisionTreeRegressor for my first model and see to where I could get. Depending on the results, I would have chosen a model for the comparison. Since the RMSE of the first model was roughly 50% of the maximum threshold, I decided to take a simple LinearRegression model for my second model.

from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score, KFold
from sklearn.metrics import mean_squared_error as MSE

train_data = pd.read_csv("train.csv")
validaton = pd.read_csv("validation.csv")
print(validaton.head())

# I decided to drop the sale_date column and encoded the  categorical data
X1 = train_data[["city","months_listed","bedrooms","house_type","area"]]
X1_encoded = pd.get_dummies(X1, columns=["city","house_type"], drop_first=True)
y1 = train_data["sale_price"]

X_train, X_test, y_train, y_test = train_test_split(X1_encoded,y1,
                                                    test_size=0.3,
                                                    random_state=42)

dt = DecisionTreeRegressor(max_depth=8,min_samples_leaf=0.004,random_state=42)

print(X_train)

MSE_CV = - cross_val_score(dt, X_train, y_train, cv=10,scoring="neg_mean_squared_error", n_jobs=-1)

dt.fit(X_train,y_train)
y_predict_train = dt.predict(X_train)
y_predict_test = dt.predict(X_test)

print(MSE_CV.mean()**(1/2))
print(MSE(y_train,y_predict_train)**(1/2))
print(MSE(y_test, y_predict_test)**(1/2))

X_validation = validaton[["city","months_listed","bedrooms","house_type","area"]]
X_validation_encoded = pd.get_dummies(X_validation, columns=["city","house_type"], drop_first=True)
X_validation_encoded = X_validation_encoded.reindex(columns=X_train.columns, fill_value=0)



y_validation_pred = dt.predict(X_validation_encoded)

base_result = validaton.copy()
base_result["price"] = y_validation_pred
base_result = base_result[["house_id","price"]]

print(base_result)

# Use this cell to write your code for Task 5

# My linreg model achieved a mean RSME of 23465.56, so even my simpler model is below the threshold, which makes me confident that my I will pass the requirements
from sklearn.linear_model import LinearRegression

train_data2 = pd.read_csv("train.csv")
X2 = train_data[["city","months_listed","bedrooms","house_type","area"]]
X2_encoded = pd.get_dummies(X1, columns=["city","house_type"], drop_first=True)
y2 = train_data["sale_price"]

X_train2, X_test2, y_train2, y_test2 = train_test_split(X2_encoded,y2,
                                                    test_size=0.2,
                                                    random_state=42)

kf = KFold(n_splits=5, shuffle=True,random_state=42)

linreg = LinearRegression()
linreg_cv = cross_val_score(linreg, X_train2, y_train2, cv=kf,scoring="neg_mean_squared_error")
print(np.sqrt(-linreg_cv))

linreg.fit(X_train2, y_train2)


y_predict_test2 = linreg.predict(X_test2)
print(MSE(y_test2, y_predict_test2)**(1/2))


y_validation_pred2 = linreg.predict(X_validation_encoded)

compare_result = validaton.copy()
compare_result["price"] = y_validation_pred2
compare_result = compare_result[["house_id","price"]]

print(compare_result)