# Imports
import warnings
warnings.filterwarnings('ignore')

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

%matplotlib inline

train = pd.read_csv("train-file.csv")
test = pd.read_csv("test-file.csv")

print(train.shape)
display(train.columns.values.reshape((12,1)))

train.head()

print('Categorical Variables:')
print('----------------------')
for x in train.dtypes.index:
    if train.dtypes[x]=='object':
        print(x)

def my_summary(df):
    df1 = df.count()
    df2 = df.apply(lambda x: sum(x.isnull()))  # missing values
    df3 = df.apply(lambda x: len(x.unique()))  # missing values

    dfx = pd.concat([df1, df2, df3], axis=1)
    dfx.columns = ['count', 'NaN', 'Unique']
    print(dfx)

my_summary(train)  # Details variables

cat_var = ['Item_Identifier', 'Item_Fat_Content', 'Item_Type', 'Outlet_Identifier',
               'Outlet_Size', 'Outlet_Location_Type', 'Outlet_Type']

for col in cat_var:
    print(train[col].value_counts())
    print('---')

num_var = ['Item_Weight', 'Item_Visibility', 'Item_MRP', 'Outlet_Establishment_Year']

train[num_var].head(10)

plt.hist(train.Item_Visibility, bins=100);

for outlet in train.Outlet_Identifier.unique():
    print(outlet, train[train.Outlet_Identifier==outlet].Item_Visibility.mean())

for outlet in train.Outlet_Type.unique():
    print(outlet, train[train.Outlet_Type==outlet].Item_Visibility.mean())

train['Item_Visibility_Rate'] = 'Low'
train.loc[(train.Outlet_Identifier == 'OUT010') | (train.Outlet_Identifier == 'OUT019'), 'Item_Visibility_Rate'] = 'High'

from scipy.stats import chi2_contingency

# Outlet_Size vs. Outlet_Type
tab = pd.crosstab(train.Outlet_Size, train.Outlet_Type)
chi2, p, dof, ex = chi2_contingency(tab.values, correction=False)
print(tab)
print('Chi2 statistics p-value: ', p)
print('------------------------------------------------------------')

# Outlet_Size vs. Outlet_Location_Type
tab = pd.crosstab(train.Outlet_Size, train.Outlet_Location_Type)
chi2, p, dof, ex = chi2_contingency(tab.values, correction=False)
print(tab)
print('Chi2 statistics p-value: ', p)
print('------------------------------------------------------------')

# Outlet_Size vs. Outlet_Establishment_Year
tab = pd.crosstab(train.Outlet_Size, train.Outlet_Establishment_Year)
chi2, p, dof, ex = chi2_contingency(tab.values, correction=False)
print(tab)
print('Chi2 statistics p-value: ', p)
print('------------------------------------------------------------')

tab = pd.crosstab(train[train.Outlet_Size.isnull()].Outlet_Type, train[train.Outlet_Size.isnull()].Outlet_Location_Type)
chi2, p, dof, ex = chi2_contingency(tab.values, correction=False)
print(tab)

train[(~train.Outlet_Size.isnull()) & \
      (train.Outlet_Location_Type == 'Tier 2') & \
      (train.Outlet_Type == 'Supermarket Type1')].Outlet_Size.unique()

train[(~train.Outlet_Size.isnull()) & \
      (train.Outlet_Location_Type == 'Tier 3') & \
      (train.Outlet_Type == 'Grocery Store')].Outlet_Size.unique()

train[(~train.Outlet_Size.isnull()) & \
      (train.Outlet_Type == 'Grocery Store')].Outlet_Size.unique()

codes = train.Item_Identifier.apply(lambda x: x[:2])
codes.unique()

train[codes == 'FD'].Item_Type.head(5)

train[codes == 'DR'].Item_Type.head(5)

train[codes == 'NC'].Item_Type.head(5)

plt.hist(train.Item_MRP, bins=200);
plt.show()

import numpy as np
import pandas as pd
import math
from sklearn.base import TransformerMixin
from sklearn.preprocessing import OneHotEncoder, MinMaxScaler
from sklearn.feature_extraction import DictVectorizer
from sklearn.pipeline import Pipeline
from collections import defaultdict

class SimpleColumnTransformer(TransformerMixin):
    '''
    This class will perform 5 transformations.
    By default, all 5 will be take place, but they can be skipped.
    '''
    def __init__(self, item_id=True, fat_content=True, outlet_year=True, item_mrp=True, outlet_size=True):
        self.item_id = item_id
        self.fat_content = fat_content
        self.outlet_year = outlet_year
        self.item_mrp = item_mrp
        self.outlet_size = outlet_size

    def fit(self, X, y=None):  # stateless transformer
        return self

    def transform(self, X):  # assumes X is a DataFrame
        if self.item_id:
            X.loc[:, 'Item_Category'] = np.nan
            codes = X['Item_Identifier'].apply(lambda x: x[:2])
            X.loc[:, 'Item_Category'] = codes

        if self.fat_content:
            X.Item_Fat_Content = X.Item_Fat_Content.replace(['low fat', 'LF'], 'Low Fat')
            X.Item_Fat_Content = X.Item_Fat_Content.replace('reg', 'Regular')

        if self.outlet_year:
            X.loc[:, 'Outlet_Age'] = 2013 - X.Outlet_Establishment_Year
            X = X.drop('Outlet_Establishment_Year', axis=1)  # drop the redundant column

        if self.item_mrp:
            X.loc[:, 'Item_Price_Category'] = np.nan  # create empty column first
            X.loc[X.Item_MRP < 68, 'Item_Price_Category'] = 'Cheap'
            X.loc[(X.Item_MRP > 68) & (X.Item_MRP < 135),'Item_Price_Category'] = 'Affordable'
            X.loc[(X.Item_MRP > 135) & (X.Item_MRP < 202), 'Item_Price_Category'] = 'Pricey'
            X.loc[X.Item_MRP > 202, 'Item_Price_Category'] = 'Expensive'

        if self.outlet_size:      
            X.loc[(X.Outlet_Size.isnull()) & \
                  (X.Outlet_Location_Type == 'Tier 2') & \
                  (X.Outlet_Type == 'Supermarket Type1'), 'Outlet_Size'] = 'Small'

            X.loc[(X.Outlet_Size.isnull()) & \
                  (X.Outlet_Location_Type == 'Tier 3') & \
                  (X.Outlet_Type == 'Grocery Store'), 'Outlet_Size'] = 'Small'       

        return X

class TreatWeight(TransformerMixin):

    def __init__(self):
        self.weights = None

    def fit(self, X, y=None):
        d = defaultdict(int)
        for item in X.Item_Identifier:
            replacement = X[(X.Item_Identifier == item) & (~X.Item_Weight.isnull())].Item_Weight.mean()
            if math.isnan(replacement):
                d[item] = X.Item_Weight.mean().round(2)
            else:
                d[item] = replacement.round(2)
        self.weights = d
        return self

    def transform(self, X):
        # assumes X is a DataFrame
        for index, item in X[X.Item_Weight.isnull()].Item_Identifier.iteritems():
            X.loc[index, 'Item_Weight'] = self.weights[item]
        return X

class TreatVisibility(TransformerMixin):

    def __init__(self):
        self.visibs = None

    def fit(self, X, y=None):
        d = defaultdict(int)
        for item in X.Item_Identifier:
            replacement = X[(X.Item_Identifier == item) & (X.Item_Visibility != 0.0)].Item_Visibility.mean()
            if math.isnan(replacement):
                d[item] = X.Item_Visibility.mean()
            else:
                d[item] = replacement
        self.visibs = d
        return self

    def transform(self, X):
        # assumes X is a DataFrame
        for index, item in X[X.Item_Visibility == 0.0].Item_Identifier.iteritems():
            X.loc[index, 'Item_Visibility'] = self.visibs[item]
        X.loc[:, 'Item_Visibility'] = np.sqrt(X.Item_Visibility)
        return X

class DummyTransformer(TransformerMixin):

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        # assumes X is a DataFrame
        X = pd.get_dummies(X, drop_first=True)
        return X

class ZeroOneScaler(TransformerMixin):

    def __init__(self):
        self.mm = None

    def fit(self, X, y=None):
        self.mm = MinMaxScaler()
        self.mm.fit(X)
        return self

    def transform(self, X):
        # assumes X is a DataFrame
        X.loc[:,:] = self.mm.transform(X)
        return X

class ColumnExtractor(TransformerMixin):

    def __init__(self, cols):
        # assign columns
        self.cols = cols

    def fit(self, X, y=None):
        # stateless transformer
        return self

    def transform(self, X):
        # assumes X is a DataFrame
        Xcols = X[self.cols]
        return Xcols

FEATURES = ['Item_Category', 'Item_Weight', 'Item_Fat_Content',
             'Item_Visibility', 'Item_MRP', 'Item_Type', 'Item_Price_Category',
             'Outlet_Type', 'Outlet_Location_Type', 'Outlet_Size', 'Outlet_Age']

pipeline = Pipeline([
            ('transform', SimpleColumnTransformer()),
            ('weight', TreatWeight()),
            ('visibility', TreatVisibility()),
            ('extract', ColumnExtractor(FEATURES)),
            ('dummy', DummyTransformer())
        ])

pipeline.fit(train)

train_t = pipeline.transform(train)
my_summary(train_t)

train_t.head()

test_t = pipeline.transform(test)

# The target variable
target = train['Item_Outlet_Sales']

from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split

X_train, X_val, y_train, y_val = train_test_split(train, target, test_size=0.33, random_state=42)

from sklearn.base import BaseEstimator

class Baseline_Estimator(BaseEstimator):
    def __init__(self):
        self.predicts = None

    def fit(self, XX, y=None):
        d = defaultdict(int)
        for item in XX.Item_Identifier:
            d[item] = XX[(XX.Item_Identifier == item)].Item_Outlet_Sales.mean()
        self.predicts = d
        return self

    def predict(self, XX):
        # assumes X is a DataFrame
        for index, item in XX.Item_Identifier.iteritems():
            XX.loc[index, 'Item_Outlet_Sales'] = self.predicts[item]
        return XX.Item_Outlet_Sales

base_model = Baseline_Estimator()
base_model.fit(X_train)
y_pred = base_model.predict(X_val)
print("Baseline RMSE = ", np.sqrt(mean_squared_error(y_val, y_pred)))

# Final model
base_model = Baseline_Estimator()
base_model.fit(train)
test_target = base_model.predict(test)

# Export for submission
test.loc[:, 'Item_Otlet_Sales'] = test_target
output = test[['Item_Identifier', 'Outlet_Identifier', 'Item_Outlet_Sales']]
output.to_csv('baseline_solution.csv', index=False)

# Split the transformed data (output of pipeline)
X_train, X_val, y_train, y_val = train_test_split(train_t, target, test_size=0.33, random_state=42)

from sklearn.feature_selection import RFECV
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score

estimator = LinearRegression()
selector = RFECV(estimator, step=1, cv=3, min_features_to_select=1)

selector = selector.fit(X_train, y_train)
print('Selected features:')
print(train_t.columns[selector.support_])

# Predict
y_pred = selector.predict(X_val)
print("Linear Regression with feature elimination RMSE = ", np.sqrt(mean_squared_error(y_val, y_pred)))

# Final model
selector = selector.fit(train_t, target)
test_target = selector.predict(test_t)

# Export
test['Item_Outlet_Sales'] = test_target
output = test[['Item_Identifier', 'Outlet_Identifier', 'Item_Outlet_Sales']]
output.to_csv('lr_rfecv.csv', index=False)

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV

# Create the parameter grid based on the results of random search
param_grid = {
    'bootstrap': [True],
    'max_depth': [2, 3, 4, 5, 6, 7, 8, 9, 10],
    'n_estimators': [100, 200, 300, 400, 500, 1000],
    'min_samples_leaf': [1, 2, 4]
}

# Create a based model
rf = RandomForestRegressor()

# Instantiate the grid search model
grid_search = GridSearchCV(estimator = rf, param_grid = param_grid,
                          cv = 3, n_jobs = -1, verbose = 2)

# Fit the grid search to the data
grid_search.fit(X_train, y_train)
best_model = grid_search.best_estimator_

y_pred = best_model.predict(X_val)
print("RF hyp. opt. RMSE = ", np.sqrt(mean_squared_error(y_val, y_pred)))

grid_search.best_params_

features = X_train.columns.values
importances = best_model.feature_importances_

important_features = [features[i] for i, imp in enumerate(importances) if imp > 0.001]
important_features

# Fit the grid search to the data
rf = RandomForestRegressor(max_depth=5, min_samples_leaf=4, n_estimators=100, random_state=42)
rf.fit(X_train[important_features], y_train)

y_pred = rf.predict(X_val[important_features])
print("RF hyp. opt. w/ feat. sel. RMSE = ", np.sqrt(mean_squared_error(y_val, y_pred)))

# Final model
rf.fit(train_t[important_features], target)
test_target = rf.predict(test_t[important_features])

# Export
test['Item_Outlet_Sales'] = test_target
output = test[['Item_Identifier', 'Outlet_Identifier', 'Item_Outlet_Sales']]
output.to_csv('rf_opt.csv', index=False)

import xgboost as xgb

xg_reg = xgb.XGBRegressor(objective ='reg:squarederror', colsample_bytree=0.3, learning_rate=0.1,
                max_depth=1, alpha=10, n_estimators=100, booster='gbtree')

for i in range(1,11):
    xg_reg.set_params(max_depth=i)
    xg_reg.fit(X_train, y_train)
    y_pred = xg_reg.predict(X_val)

    rmse = np.sqrt(mean_squared_error(y_val, y_pred))
    print(i)
    print("RMSE: %f" % (rmse))
    print('Test:', r2_score(y_val, y_pred))
    print('Train:', r2_score(y_train, xg_reg.predict(X_train)))

xg_reg.set_params(max_depth=4);

for i in range(10,110,10):
    xg_reg.set_params(n_estimators=i)
    xg_reg.fit(X_train, y_train)
    y_pred = xg_reg.predict(X_val)

    rmse = np.sqrt(mean_squared_error(y_val, y_pred))
    print(i)
    print("RMSE: %f" % (rmse))
    print('Test:', r2_score(y_val, y_pred))
    print('Train:', r2_score(y_train, xg_reg.predict(X_train)))

xg_reg.set_params(n_estimators=80)
xg_reg.fit(X_train, y_train)
y_pred = xg_reg.predict(X_val)
print("XGBoost RMSE = ", np.sqrt(mean_squared_error(y_val, y_pred)))

# Final model
xg_reg.fit(train_t, target)
test_target = xg_reg.predict(test_t)

# Export
test['Item_Outlet_Sales'] = test_target
output = test[['Item_Identifier', 'Outlet_Identifier', 'Item_Outlet_Sales']]
output.to_csv('xgboost.csv', index=False)