%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns; sns.set_style('whitegrid')
oj = pd.read_csv('../../datasets/OJ.csv', index_col=0)
oj = oj.reset_index(drop=True)
oj.head()

oj.info()

# drop superfluous variables
oj = oj.drop(columns=['STORE', 'Store7'])

oj.columns

# one hot encode categoricals
cat_vars = ['StoreID', 'Purchase']

oj = pd.get_dummies(data=oj, columns=cat_vars, drop_first=True)
oj.head()

oj.info()

from sklearn.model_selection import train_test_split

X, y = oj.drop(columns=['Purchase_MM']), oj['Purchase_MM']
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=800, random_state=27)
X_train.shape

from sklearn.tree import DecisionTreeClassifier

clf_tree = DecisionTreeClassifier(random_state=27)
clf_tree.fit(X_train, y_train)

# training error rate
clf_tree.score(X_train, y_train)

# test error rate
clf_tree.score(X_test, y_test)

estimator = clf_tree
n_nodes = estimator.tree_.node_count
children_left = estimator.tree_.children_left
children_right = estimator.tree_.children_right
feature = estimator.tree_.feature
threshold = estimator.tree_.threshold


# The tree structure can be traversed to compute various properties such
# as the depth of each node and whether or not it is a leaf.
node_depth = np.zeros(shape=n_nodes, dtype=np.int64)
is_leaves = np.zeros(shape=n_nodes, dtype=bool)
stack = [(0, -1)]  # seed is the root node id and its parent depth
while len(stack) > 0:
    node_id, parent_depth = stack.pop()
    node_depth[node_id] = parent_depth + 1

    # If we have a test node
    if (children_left[node_id] != children_right[node_id]):
        stack.append((children_left[node_id], parent_depth + 1))
        stack.append((children_right[node_id], parent_depth + 1))
    else:
        is_leaves[node_id] = True

# number of leaves = number of terminal nodes
np.sum(is_leaves)

# feature importances
clf_tree_feat_imp = pd.DataFrame({'feature': X_train.columns, 
                            'importance': clf_tree.feature_importances_})
clf_tree_feat_imp.sort_values(by='importance', ascending=False)

from graphviz import Source
from sklearn import tree
from IPython.display import SVG
graph = Source(tree.export_graphviz(clf_tree, out_file=None, filled=True, 
                                    feature_names=X.columns))
display(SVG(graph.pipe(format='svg')))

from sklearn.metrics import confusion_matrix

y_act = pd.Series(['Purchase_CH' if entry == 0 else "Purchase_MM" for entry in y_test],
                  name='Actual')
y_pred = pd.Series(['Purchase_CH' if entry == 0 else "Purchase_MM" for entry in clf_tree.predict(X_test)],
                  name='Predicted')
clf_tree_test_conf = pd.crosstab(y_act, y_pred, margins=True)
clf_tree_test_conf

from sklearn.model_selection import GridSearchCV

params = {'max_depth': list(range(1, 800)) + [None]}
clf_tree = DecisionTreeClassifier(random_state=27)
clf_tree_search = GridSearchCV(estimator=clf_tree, 
                               param_grid=params,
                               cv=8,
                               scoring='accuracy')

%timeit -n1 -r1 clf_tree_search.fit(X_train, y_train)

clf_tree_search.best_params_

clf_tree_search.best_score_

clf_tree_search_df = pd.DataFrame(clf_tree_search.cv_results_)
clf_tree_search_df.head()

x, y = clv_tree_search_df['param_max_depth'], clv_tree_search_df['mean_test_score']
plt.plot(x, y)

plt.plot(x.loc[:30, ], y.loc[:30, ])

plt.plot(np.arange(y))

from sklearn.model_selection import RandomizedSearchCV

params = {'max_features': np.arange(1, 18),
          'max_leaf_nodes': np.append(np.arange(2, 21), None),
          'min_impurity_decrease': np.linspace(0, 1, 10),
          'min_samples_leaf': np.arange(1, 11)
          }
clf_tree2 = DecisionTreeClassifier(random_state=27, max_depth=2)

# Randomized search to cover a large region of parameter space
clf_tree2_rvsearch = RandomizedSearchCV(estimator=clf_tree2,
                                        param_distributions=params,
                                        cv=8,
                                        scoring='accuracy',
                                        n_iter=1000,
                                        random_state=27)
%timeit -n1 -r1 clf_tree2_rvsearch.fit(X_train, y_train)

clf_tree2_rvsearch.best_params_

# Grid search nearby randomized search results
params = {'min_samples_leaf': np.arange(4, 8),
          'max_features': np.arange(10, 16)
         }
clf_tree3 = DecisionTreeClassifier(random_state=27, 
                                   max_depth=2,
                                   max_leaf_nodes=None)
clf_tree2_gridsearch = GridSearchCV(estimator=clf_tree3,
                                    param_grid=params,
                                    cv=8,
                                    scoring='accuracy')
%timeit -n1 -r1 clf_tree2_gridsearch.fit(X_train, y_train)

clf_tree2_gridsearch.best_params_

clf_tree2_gridsearch.best_score_

from sklearn.metrics import accuracy_score

# trees for comparison
clf_tree = clf_tree_search.best_estimator_
pruned_clf_tree = clf_tree2_gridsearch.best_estimator_

# train and test errors
train = [accuracy_score(clf_tree.predict(X_train), y_train), 
         accuracy_score(pruned_clf_tree.predict(X_train), y_train)]
test = [accuracy_score(clf_tree.predict(X_test), y_test), 
         accuracy_score(pruned_clf_tree.predict(X_test), y_test)]

# df for results
comp_df = pd.DataFrame({'train_error': train, 'test_error': test}, 
                       index=['clf_tree', 'pruned_clf_tree'])
comp_df