diff --git a/notebooks/[WIP] Untitled.ipynb b/notebooks/[WIP] Untitled.ipynb
index 8aa5872..d7ffb4f 100644
--- a/notebooks/[WIP] Untitled.ipynb	
+++ b/notebooks/[WIP] Untitled.ipynb	
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 259,
    "id": "f13010f3",
    "metadata": {},
    "outputs": [],
@@ -13,6 +13,9 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import param\n",
+    "#from pycaret.utils import check_metric\n",
+    "\n",
+    "#from feature_selection.run_pycaret_setup import run_pycaret_setup\n",
     "\n",
     "\n",
     "class FeatureSelection(param.Parameterized):\n",
@@ -103,10 +106,10 @@
     "    target = param.String(\"goal_2.5\")\n",
     "    number_features = param.Number(\n",
     "        0.5,\n",
-    "        bounds=(0, None),\n",
-    "        inclusive_bounds=(False, True),\n",
-    "        doc=\"Number of features selected each iteration. Only the first nth features \"\n",
-    "        \"(where n is given by 'number_features') will be kept for the next iteration.\",\n",
+    "        bounds=(0, 1),\n",
+    "        inclusive_bounds=(False, False),\n",
+    "        doc=\"Number of features (percentage) selected each iteration. Only the first nth \"\n",
+    "        \"features will be kept for the next iteration.\",\n",
     "    )\n",
     "    target_features = param.Number(\n",
     "        0.3,\n",
@@ -115,12 +118,11 @@
     "        doc=\"Final total number of features. The goal of the package is to reduce \"\n",
     "        \"the incoming columns of the dataset to this 'target_features' number.\",\n",
     "    )\n",
-    "    feature_division = param.Number(3, bounds=(1, 100))\n",
     "    ## Metric parameters\n",
     "    filter_metrics = param.Dict(_filter_metric)\n",
     "    ## Model setup and model optimization parameters\n",
     "    numerics = param.List(_numerics)\n",
-    "    ignore_features = param.List(default=None, allow_None=True)\n",
+    "    ignore_features = param.List(default=[], allow_None=True)\n",
     "    setup_kwargs = param.Dict(_setup_kwargs)\n",
     "    include = param.List(default=None, item_type=str, allow_None=True)\n",
     "    exclude = param.List([\"qda\", \"knn\", \"nb\"], item_type=str)\n",
@@ -144,10 +146,12 @@
     "        dataset = dataset.copy()\n",
     "        # Compute the upper bound of number_features, target_features, number_models\n",
     "        total_features = dataset.shape[1]\n",
-    "        self.param.number_features.bounds = (0, total_features)\n",
     "        self.param.target_features.bounds = (0, total_features)\n",
     "        if \"include\" in kwargs:\n",
-    "            self.param.number_models.bounds = (2, len(include))\n",
+    "            print('hola')\n",
+    "            self.param.number_models.default = len(kwargs[\"include\"])\n",
+    "            self.param.number_models.bounds = (0, len(kwargs[\"include\"]))\n",
+    "            print(self.param.number_models.bounds)\n",
     "        # Call super\n",
     "        super(FeatureSelection, self).__init__(dataset=dataset, **kwargs)\n",
     "        # Get the features of the dataframe\n",
@@ -158,7 +162,7 @@
     "            number_features=self.target_features, features=self.feature_list\n",
     "        )\n",
     "        # Get the evaluator and the arguments. Depends on the \"include\" parameter\n",
-    "        self.obj, self.args = self._decide_model_eval()\n",
+    "        self._training_function, self._args = self._decide_model_eval()\n",
     "        # Get all the columns whose type is numeric\n",
     "        self.numeric_features = self._compute_numeric_features(df=self.dataset[self.feature_list])\n",
     "\n",
@@ -180,63 +184,42 @@
     "        list.\n",
     "        \"\"\"\n",
     "        args = {\"n_select\": self.number_models, \"sort\": self.sort, \"verbose\": False}\n",
-    "        obj = np.random.randint\n",
+    "        training_function = np.random.randint\n",
     "        if not self.include:\n",
     "            args[\"exclude\"] = self.exclude\n",
     "        elif len(self.include) == 1:\n",
-    "            obj = range\n",
-    "            args = {\"estimator\": self.include[0]}\n",
+    "            training_function = lambda *rgs, **kwargs: [range(*rgs, **kwargs)]\n",
+    "            args = {\"estimator\": self.include[0], \"verbose\": False}\n",
     "        else:\n",
     "            args[\"include\"] = self.include\n",
-    "        return obj, args\n",
+    "        return training_function, args\n",
     "\n",
     "    @staticmethod\n",
     "    def calculate_number_features(\n",
     "        number_features: Union[int, float], features: Union[pd.DataFrame, List]\n",
     "    ) -> int:\n",
     "        n_features = (\n",
-    "            int(number_features) if (number_features >= 1) else int(number_features * len(features))\n",
+    "            int(number_features)\n",
+    "            if (number_features >= 1)\n",
+    "            else int(number_features * len(features))\n",
     "        )\n",
-    "        return n_features\n",
-    "\n",
-    "    def train_model(self):\n",
-    "        \"\"\"Preprocess the data and select self.number_models top models.\"\"\"\n",
-    "        # Selected dataset\n",
-    "        selected_cols = self.feature_list + [self.target]\n",
-    "        train_data = self.dataset[selected_cols] if self.x_df.empty else self.x_df[selected_cols]\n",
-    "        # Numeric features\n",
-    "        self.setup_kwargs[\"numeric_features\"] = [\n",
-    "            c for c in self.numeric_features if c in self.feature_list\n",
-    "        ]\n",
-    "        # Ignore features\n",
-    "        self.setup_kwargs[\"ignore_features\"] = [\n",
-    "            c for c in self.ignore_features if c in self.feature_list\n",
-    "        ]\n",
-    "        # Initialize pycaret setup\n",
-    "        setup(train_data=train_data, target=self.target, **self.setup_kwargs)\n",
-    "        # Get train dataset and preprocessed dataframe\n",
-    "        self.x_train = get_config(\"X_train\")\n",
-    "        if self.x_df.empty:  # TODO change x_df by dataset and add flag?\n",
-    "            self.x_df = pd.concat([get_config(\"X\"), get_config(\"y\")], axis=1)\n",
-    "            self.setup_kwargs[\"preprocess\"] = False  # Turn off preprocessing\n",
-    "        # Compare models\n",
-    "        self.top_models = self.obj(**self.args)"
+    "        return n_features"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 260,
    "id": "028d5b40",
    "metadata": {},
    "outputs": [],
    "source": [
     "dataset = pd.DataFrame({'a': [1,3,3,4,5], 'b': [13,3,4,4,5], 'c': ['a', 'b', 'c', 'd', 'd']})\n",
-    "a = FeatureSelection(dataset, target='c', target_features=1)"
+    "pepe = FeatureSelection(dataset, target='c', target_features=1)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 139,
    "id": "c8453ac0",
    "metadata": {},
    "outputs": [
@@ -246,7 +229,7 @@
        "['a', 'b']"
       ]
      },
-     "execution_count": 31,
+     "execution_count": 139,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -257,11 +240,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "3bd6c126",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/vicentearjona/Library/Python/3.8/lib/python/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (2012,2016) have mixed types.Specify dtype option on import or set low_memory=False.\n",
+      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
+     ]
+    }
+   ],
+   "source": [
+    "predictions = pd.read_csv('predictions.csv')"
+   ]
   },
   {
    "cell_type": "code",
@@ -281,7 +275,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 3,
    "id": "aba4a1d2-96b4-444a-9648-05c48864faae",
    "metadata": {},
    "outputs": [
@@ -796,396 +790,6 @@
     "pn.Row(a.param)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 35,
-   "id": "63e48759-5207-4631-a004-574ff32eb6e1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from itertools import product\n",
-    "from typing import Dict, List, Optional, Union\n",
-    "\n",
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "import param\n",
-    "#from pycaret.classification import compare_models, get_config, predict_model, tune_model\n",
-    "#from pycaret.utils import check_metric\n",
-    "\n",
-    "#from feature_selection.run_pycaret_setup import run_pycaret_setup\n",
-    "\n",
-    "\n",
-    "class FeatureSelection(param.Parameterized):\n",
-    "\n",
-    "    # Class attributes\n",
-    "    model_class_to_name = {\n",
-    "        \"RidgeClassifier\": \"ridge\",\n",
-    "        \"LogisticRegression\": \"lr\",\n",
-    "        \"LinearDiscriminantAnalysis\": \"lda\",\n",
-    "        \"GradientBoostingClassifier\": \"gbc\",\n",
-    "        \"QuadraticDiscriminantAnalysis\": \"qda\",\n",
-    "        \"LGBMClassifier\": \"lightgbm\",\n",
-    "        \"AdaBoostClassifier\": \"ada\",\n",
-    "        \"RandomForestClassifier\": \"rf\",\n",
-    "        \"ExtraTreesClassifier\": \"et\",\n",
-    "        \"GaussianNB\": \"nb\",\n",
-    "        \"DecisionTreeClassifier\": \"dt\",\n",
-    "        \"KNeighborsClassifier\": \"knn\",\n",
-    "        \"SGDClassifier\": \"svm\",\n",
-    "        \"CatBoostClassifier\": \"catboost\",\n",
-    "        \"SVC\": \"rbfsvm\",\n",
-    "        \"GaussianProcessClassifier\": \"gpc\",\n",
-    "        \"MLPClassifier\": \"mlp\",\n",
-    "        \"XGBClassifier\": \"xgboost\",\n",
-    "    }\n",
-    "\n",
-    "    metrics_list = [\"Accuracy\", \"AUC\", \"Recall\", \"Precision\", \"F1\", \"Kappa\", \"MCC\"]\n",
-    "\n",
-    "    # Private class attributes\n",
-    "    _filter_metric = {\n",
-    "        \"Accuracy\": 0.5,\n",
-    "        \"AUC\": 0.5,\n",
-    "        \"Recall\": 0.6,\n",
-    "        \"Precision\": 0.6,\n",
-    "        \"F1\": 0.6,\n",
-    "        \"Kappa\": 0.1,\n",
-    "        \"MCC\": 0.1,\n",
-    "    }\n",
-    "\n",
-    "    _setup_kwargs = dict(\n",
-    "        preprocess=True,\n",
-    "        train_size=0.75,\n",
-    "        # test_data=test_data,\n",
-    "        session_id=123,\n",
-    "        normalize=True,\n",
-    "        transformation=True,\n",
-    "        ignore_low_variance=True,\n",
-    "        remove_multicollinearity=False,\n",
-    "        multicollinearity_threshold=0.4,\n",
-    "        n_jobs=-1,\n",
-    "        use_gpu=False,\n",
-    "        profile=False,\n",
-    "        ignore_features=None,\n",
-    "        fold_strategy=\"timeseries\",\n",
-    "        remove_perfect_collinearity=True,\n",
-    "        create_clusters=False,\n",
-    "        fold=4,\n",
-    "        feature_selection=False,\n",
-    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
-    "        feature_selection_threshold=0.4,\n",
-    "        combine_rare_levels=False,\n",
-    "        rare_level_threshold=0.02,\n",
-    "        pca=False,\n",
-    "        pca_method=\"kernel\",\n",
-    "        pca_components=30,\n",
-    "        polynomial_features=False,\n",
-    "        polynomial_degree=2,\n",
-    "        polynomial_threshold=0.01,\n",
-    "        trigonometry_features=False,\n",
-    "        remove_outliers=False,\n",
-    "        outliers_threshold=0.01,\n",
-    "        feature_ratio=False,\n",
-    "        feature_interaction=False,\n",
-    "        # Makes everything slow AF. use to find out possibly interesting features\n",
-    "        interaction_threshold=0.01,\n",
-    "        fix_imbalance=False,\n",
-    "        log_experiment=False,\n",
-    "        verbose=False,\n",
-    "        silent=True,\n",
-    "        experiment_name=\"lagstest\",\n",
-    "    )\n",
-    "\n",
-    "    _numerics = [\"int16\", \"int32\", \"int64\", \"float16\", \"float32\", \"float64\", \"int\", \"float\"]\n",
-    "\n",
-    "    # Init values\n",
-    "    ## Feature selection parameters\n",
-    "    target = param.String(\"goal_2.5\")\n",
-    "    number_features = param.Number(0.5, bounds=(0, None), inclusive_bounds=(False, True))\n",
-    "    target_features = param.Number(0.3, bounds=(0, None), inclusive_bounds=(False, True))\n",
-    "    feature_division = param.Number(3, bounds=(1, 100))\n",
-    "    ## Metric parameters\n",
-    "    filter_metrics = param.Dict(_filter_metric)\n",
-    "    ## Model setup and model optimization parameters\n",
-    "    numerics = param.List(_numerics)\n",
-    "    ignore_features = param.List(default=None, allow_None=True)\n",
-    "    setup_kwargs = param.Dict(_setup_kwargs)\n",
-    "    include = param.List(default=None, item_type=str, allow_None=True)\n",
-    "    exclude = param.List([\"qda\", \"knn\", \"nb\"], item_type=str)\n",
-    "    sort = param.String(\"AUC\")\n",
-    "    number_models = param.Integer(10, bounds=(2, 13))\n",
-    "    top_models = param.List(default=None, allow_None=True)\n",
-    "    optimize = param.Boolean(False)\n",
-    "    opt_list = param.List([\"Accuracy\", \"Precision\", \"Recall\", \"F1\", \"AUC\"], item_type=str)\n",
-    "    ## Class selectors\n",
-    "    dataset = param.ClassSelector(class_=pd.DataFrame)\n",
-    "    dict_models = param.ClassSelector(class_=dict)\n",
-    "    tune_dict_models = param.ClassSelector(class_=dict)\n",
-    "    x_train = param.ClassSelector(class_=pd.DataFrame)\n",
-    "    model_df = param.ClassSelector(class_=pd.DataFrame)\n",
-    "    model_tuned_df = param.ClassSelector(class_=pd.DataFrame)\n",
-    "    features_df = param.ClassSelector(class_=pd.DataFrame)\n",
-    "\n",
-    "    def __init__(self, dataset: pd.DataFrame, **kwargs):\n",
-    "        # Compute the upper bound of number_features and target_features\n",
-    "        total_features = dataset.shape[1]\n",
-    "        self.param.number_features.bounds = (0, total_features)\n",
-    "        self.param.target_features.bounds = (0, total_features)\n",
-    "        # Call super\n",
-    "        super(FeatureSelection, self).__init__(dataset=dataset, **kwargs)\n",
-    "        # Get the features of the dataframe\n",
-    "        self.feature_list = self.dataset.columns.tolist()\n",
-    "        self.feature_list.remove(self.target)  # target column should not be counted\n",
-    "        # Compute target features\n",
-    "        self.target_features = self.calculate_number_features(\n",
-    "            number_features=self.target_features, features=self.feature_list\n",
-    "        )\n",
-    "\n",
-    "    def _compute_numeric_features(self, df: pd.DataFrame):\n",
-    "        \"\"\"Return those columns from the given dataset whose data type is numeric.\"\"\"\n",
-    "        return df.select_dtypes(include=self.numerics).columns.tolist()\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def calculate_number_features(\n",
-    "        number_features: Union[int, float], features: Union[pd.DataFrame, List]\n",
-    "    ) -> int:\n",
-    "        n_features = (\n",
-    "            int(number_features)\n",
-    "            if (number_features > 1)\n",
-    "            else int(number_features * len(features))\n",
-    "        )\n",
-    "        return n_features\n",
-    "\n",
-    "    def train_model(self):\n",
-    "        \"\"\"Preprocess the data and select self.number_models top models.\"\"\"\n",
-    "        # Selected dataset\n",
-    "        selected_cols = self.feature_list + [self.target]\n",
-    "        train_data = self.dataset[selected_cols]\n",
-    "        # Numeric features\n",
-    "        numeric_features = self._compute_numeric_features(\n",
-    "            df=train_data.drop(columns=[self.target])\n",
-    "        )\n",
-    "        self.setup_kwargs[\"numeric_features\"] = numeric_features\n",
-    "        # Ignore features\n",
-    "        self.setup_kwargs[\"ignore_features\"] = self.ignore_features\n",
-    "        # Initialize pycaret setup\n",
-    "        run_pycaret_setup(train_data=train_data, target=self.target, **self.setup_kwargs)\n",
-    "        # Get train dataset and best models\n",
-    "        self.x_train = get_config(\"X_train\")\n",
-    "        # Compare models\n",
-    "        compare_dict = (\n",
-    "            {\"exclude\": self.exclude} if self.include is None else {\"include\": self.include}\n",
-    "        )\n",
-    "        self.top_models = compare_models(\n",
-    "            n_select=self.number_models,\n",
-    "            sort=self.sort,\n",
-    "            **compare_dict,\n",
-    "            verbose=False,\n",
-    "        )\n",
-    "\n",
-    "    def create_dict_models(self):\n",
-    "        \"\"\"Create a dictionary whose values are pycaret standard models.\"\"\"\n",
-    "        self.dict_models = {\n",
-    "            str(top_model).split(\"(\")[0]: top_model for top_model in self.top_models\n",
-    "        }\n",
-    "        # Remove bad catboost key\n",
-    "        oldkey = [key for key in self.dict_models.keys() if key.startswith(\"<catboost\")]\n",
-    "        if oldkey:\n",
-    "            self.dict_models[\"CatBoostClassifier\"] = self.dict_models.pop(oldkey[0])\n",
-    "        # Remap\n",
-    "        self.dict_models = {\n",
-    "            self.model_class_to_name[key]: self.dict_models[key] for key in self.dict_models.keys()\n",
-    "        }\n",
-    "\n",
-    "    def create_dict_tuned_models(self):\n",
-    "        \"\"\"Create a dictionary whose keys and values are pycaret tuned models.\"\"\"\n",
-    "        self.tune_dict_models = {}\n",
-    "        for (model_str, py_model), optimize in product(self.dict_models.items(), self.opt_list):\n",
-    "            self.tune_dict_models[f\"{model_str}_tune_{optimize}\"] = tune_model(\n",
-    "                py_model, optimize=optimize, verbose=False\n",
-    "            )\n",
-    "\n",
-    "    def get_metrics_df(self, test_predicted, model, dataframe):\n",
-    "        \"\"\"Compute different metric values for the given model.\"\"\"\n",
-    "        value_dct = dict()\n",
-    "        for metric in self.metrics_list:\n",
-    "            try:\n",
-    "                value_dct[metric] = check_metric(\n",
-    "                    actual=test_predicted[self.target],\n",
-    "                    prediction=test_predicted[\"Label\"],\n",
-    "                    metric=metric,\n",
-    "                )\n",
-    "            except AttributeError:\n",
-    "                value_dct[metric] = np.nan\n",
-    "        for key, val in value_dct.items():\n",
-    "            dataframe.loc[model, key] = val\n",
-    "        return dataframe\n",
-    "\n",
-    "    def remove_bad_models(self, dataframe: pd.DataFrame):\n",
-    "        \"\"\"Filter and remove the models whose metrics do not satisfy the given conditions.\"\"\"\n",
-    "        remove_dict = dict()\n",
-    "        models = dataframe.index.tolist()\n",
-    "        for model, (metric, cond) in product(models, self.filter_metrics.items()):\n",
-    "            if dataframe.loc[model, metric] < cond:\n",
-    "                remove_dict[model] = metric\n",
-    "        remove_models = list(set(remove_dict.keys()))\n",
-    "        dataframe.drop(labels=remove_models, axis=\"index\", inplace=True)\n",
-    "        return dataframe\n",
-    "\n",
-    "    def filter_best_features(self, key_model: str, models_dict: Dict):\n",
-    "        \"\"\"Compute the most relevant features used by the given model.\"\"\"\n",
-    "        py_model = models_dict[key_model]\n",
-    "        cond = any([key_model.startswith(name) for name in [\"lr\", \"lda\", \"ridge\", \"svm\"]])\n",
-    "        score_metric = abs(py_model.coef_[0]) if cond else py_model.feature_importances_\n",
-    "        metrics_dict = {\n",
-    "            \"model_id\": key_model,\n",
-    "            \"model\": key_model.split(\"_\")[0],\n",
-    "            \"feature\": self.x_train.columns,\n",
-    "            \"score\": score_metric,\n",
-    "        }\n",
-    "        df = pd.DataFrame(metrics_dict).sort_values(by=\"score\", ascending=False)\n",
-    "        top_n_features = self.calculate_number_features(\n",
-    "            number_features=self.number_features, features=df,\n",
-    "        )\n",
-    "        return df.iloc[:top_n_features]\n",
-    "\n",
-    "    def extract_features(self, dataframe: pd.DataFrame, dict_models: Dict):\n",
-    "        \"\"\"Update self.features_df with the most relevant features used by the given model.\"\"\"\n",
-    "        models = dataframe.index.tolist()\n",
-    "        for model in models:  # model extracted from dataframe\n",
-    "            # Check\n",
-    "            if model not in dict_models.keys():  # check no errors have been produced during operations\n",
-    "                raise KeyError(\"The selected model is not listed in dict_models.keys()\")\n",
-    "            df_conc = self.filter_best_features(key_model=model, models_dict=dict_models)\n",
-    "            self.features_df = pd.concat([self.features_df, df_conc])\n",
-    "\n",
-    "    def compute_metrics_df(self):\n",
-    "        \"\"\"Update self.features_df with the most relevant features used by the standard models.\"\"\"\n",
-    "        self.model_df = pd.DataFrame(\n",
-    "            data=[], index=self.dict_models.keys(), columns=self.metrics_list\n",
-    "        )\n",
-    "        for model, py_model in self.dict_models.items():\n",
-    "            predict = predict_model(py_model)\n",
-    "            self.model_df = self.get_metrics_df(\n",
-    "                test_predicted=predict,\n",
-    "                model=model,\n",
-    "                dataframe=self.model_df,\n",
-    "            )\n",
-    "        self.model_df = self.remove_bad_models(dataframe=self.model_df)\n",
-    "        self.extract_features(dataframe=self.model_df, dict_models=self.dict_models)\n",
-    "\n",
-    "    def filter_tuned_duplicate(self):\n",
-    "        \"\"\"Remove tuned models with identical metrics.\"\"\"\n",
-    "\n",
-    "        def drop(x):\n",
-    "            x.drop_duplicates(subset=self.metrics_list, keep=\"first\", inplace=True)\n",
-    "            return x\n",
-    "\n",
-    "        df = self.model_tuned_df.groupby(\"model\").apply(drop)\n",
-    "        return df.drop(columns=\"model\").reset_index(level=\"model\")\n",
-    "\n",
-    "    def tune_df(self):\n",
-    "        \"\"\"Update self.features_df with the most relevant features used by tuned models.\"\"\"\n",
-    "        # Tune dataframe\n",
-    "        self.model_tuned_df = pd.DataFrame(\n",
-    "            data=[],\n",
-    "            index=self.tune_dict_models.keys(),\n",
-    "            columns=[\"model\"] + self.metrics_list,\n",
-    "        )\n",
-    "        # Model entry\n",
-    "        for prim_model in self.dict_models.keys():\n",
-    "            ix = [ind.startswith(prim_model) for ind in self.model_tuned_df.index]\n",
-    "            self.model_tuned_df.loc[ix, \"model\"] = prim_model\n",
-    "        # Fill dataframe\n",
-    "        for model, py_model in self.tune_dict_models.items():\n",
-    "            predict = predict_model(py_model)\n",
-    "            self.model_tuned_df = self.get_metrics_df(\n",
-    "                test_predicted=predict, model=model, dataframe=self.model_tuned_df\n",
-    "            )\n",
-    "        # Remove duplicate and filter\n",
-    "        self.model_tuned_df = self.filter_tuned_duplicate()\n",
-    "        self.model_tuned_df = self.remove_bad_models(dataframe=self.model_tuned_df)\n",
-    "        # Get features\n",
-    "        self.extract_features(dataframe=self.model_tuned_df, dict_models=self.tune_dict_models)\n",
-    "\n",
-    "    def run_feature_extraction(self):\n",
-    "        \"\"\"Update self.features_df with the most relevant features used by each model.\"\"\"\n",
-    "        # Initialize feature dataframe and train model\n",
-    "        self.features_df = pd.DataFrame(data=[], columns=[\"model_id\", \"model\", \"feature\", \"score\"])\n",
-    "        self.train_model()\n",
-    "        # Run standard models\n",
-    "        self.create_dict_models()\n",
-    "        self.compute_metrics_df()\n",
-    "        # Run tuned models\n",
-    "        if self.optimize:\n",
-    "            self.create_dict_tuned_models()\n",
-    "            if not bool(self.tune_dict_models):\n",
-    "                raise ValueError(\"The tune dictionary is empty!\")\n",
-    "            self.tune_df()\n",
-    "        # Return the list containing the features and their score\n",
-    "        self.features_df.index.name = \"index_rem\"\n",
-    "        return self.features_df.reset_index().drop(columns=\"index_rem\")\n",
-    "\n",
-    "    def remove_zeros(self):\n",
-    "        \"\"\"Remove non-relevant features (those with a zero score).\"\"\"\n",
-    "        ix = self.features_df[\"score\"] <= 0\n",
-    "        self.features_df.drop(index=self.features_df.loc[ix].index, inplace=True)\n",
-    "        self.features_df.reset_index(drop=True, inplace=True)\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def normalize(dataframe):\n",
-    "        \"\"\"Normalize the pycaret score of each feature.\"\"\"\n",
-    "\n",
-    "        def norm(x):\n",
-    "            x[\"normal\"] = x[\"score\"] / x[\"score\"].max()\n",
-    "            return x\n",
-    "\n",
-    "        return (\n",
-    "            dataframe.groupby(\"model_id\")\n",
-    "            .apply(norm)\n",
-    "            .sort_values([\"model_id\", \"normal\"], ascending=[True, False])\n",
-    "        )\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def feature_score(dataframe):\n",
-    "        \"\"\"Assign the score to the selected features.\"\"\"\n",
-    "        group = dataframe.groupby(\"feature\")\n",
-    "        sorted_data = group.agg(\n",
-    "            counts=pd.NamedAgg(column=\"normal\", aggfunc=\"count\"),\n",
-    "            normal_sum=pd.NamedAgg(column=\"normal\", aggfunc=\"sum\"),\n",
-    "        ).sort_values(by=[\"counts\", \"normal_sum\"], ascending=False)\n",
-    "        sorted_data[\"final_score\"] = sorted_data[\"normal_sum\"] / sorted_data[\"counts\"]\n",
-    "        return sorted_data.sort_values(\"final_score\", ascending=False)\n",
-    "\n",
-    "    def create_feature_list(self):\n",
-    "        \"\"\"Run all necessary methods to extract the list of relevant features.\"\"\"\n",
-    "        # Call creation features dataframe\n",
-    "        self.features_df = self.run_feature_extraction()\n",
-    "        # Remove zeros and normalize\n",
-    "        self.remove_zeros()\n",
-    "        self.features_df = self.normalize(dataframe=self.features_df)\n",
-    "        # Get score\n",
-    "        scoreboard = self.feature_score(dataframe=self.features_df)\n",
-    "        top_n_features = self.calculate_number_features(\n",
-    "            number_features=self.number_features, features=scoreboard,\n",
-    "        )\n",
-    "        filtered = scoreboard.iloc[:top_n_features]\n",
-    "        self.feature_list = filtered.index.tolist()\n",
-    "\n",
-    "    def repeat_pipeline(self):\n",
-    "        \"\"\"Iterate over the process to create the feature list and repeat it self.repeat times.\"\"\"\n",
-    "        while len(self.feature_list) > self.target_features:\n",
-    "            # Call iteration\n",
-    "            self.create_feature_list()\n",
-    "            self.number_features = (\n",
-    "                int(self.number_features / self.feature_division)\n",
-    "                if self.number_features > 1\n",
-    "                else self.number_features\n",
-    "            )\n",
-    "            if len(self.feature_list) <= 1:\n",
-    "                break\n",
-    "        return self.feature_list\n"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1198,28 +802,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 270,
    "id": "5795f3e4-652c-4a26-8c5b-f92ae325e794",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['a', 'b']"
-      ]
-     },
-     "execution_count": 25,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "data.columns.tolist()"
+    "data = predictions.iloc[:500, np.random.randint(0,2000,size=150).tolist()]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 284,
    "id": "cfb1a01d-bad0-474b-98b8-5f168fc7f9df",
    "metadata": {},
    "outputs": [
@@ -1661,13 +1254,13 @@
     "        \"MCC\": 0.1,\n",
     "    }\n",
     "    \n",
+    "    #opt_kwargs = param.Dict({})\n",
     "    # 1st param layer \n",
     "    target = param.String(default='goals_2.5')\n",
     "    # 2nd param layer \n",
     "    ## Feature selection parameters \n",
-    "    number_features = param.Number(0.5, bounds=(0, None), inclusive_bounds=(False, True) )\n",
+    "    number_features = param.Number(0.5, bounds=(0, 1), inclusive_bounds=(False, False) )\n",
     "    target_features = param.Number(0.3, bounds=(0, None), inclusive_bounds=(False, True) )\n",
-    "    feature_division = param.Number(3, bounds=(1, 100) )\n",
     "    ## Metric parameters\n",
     "    acu = param.Number(0.6, bounds=(0, 1), label='Accuracy')\n",
     "    auc = param.Number(0.6, bounds=(0, 1), label='AUC')\n",
@@ -1678,63 +1271,76 @@
     "    mcc = param.Number(0.1, bounds=(0, 1), label='MCC')\n",
     "    ## Model setup and model optimization parameters\n",
     "    ignore_features = param.ListSelector(default=None, objects=[1, 2], allow_None=True )\n",
-    "    include = param.ListSelector(default=None, objects=_available_models + [None], allow_None=True )\n",
-    "    sort = param.ListSelector(default=[\"AUC\"], objects=_filter_metric.keys() )\n",
+    "    sort = param.ObjectSelector(default=\"AUC\", objects=_filter_metric.keys() )\n",
+    "    include = param.ListSelector(default=None, objects= [None] + _available_models, allow_None=True )\n",
     "    number_models = param.Integer(10, bounds=(2, len(_available_models)) )\n",
     "    optimize = param.Boolean(False, label='Tuning process', doc='Call tuning process during feature evaluation.')\n",
+    "    opt_list = param.ListSelector(default=['AUC'], objects=list(_filter_metric.keys())[:-2], precedence=-1, label='Tuning list')\n",
     "    # Hidden layers\n",
     "    dataset = param.ClassSelector(class_=pd.DataFrame, precedence=-1)\n",
-    "    metric_dict = param.Dict(default=_filter_metric, precedence=-1)\n",
-    "    start_run = None#param.Number(bounds=(0,None))\n",
+    "    filter_metrics = param.Dict(default=_filter_metric, precedence=-1)\n",
     "    \n",
     "    def __init__(self, dataset, **kwargs):\n",
     "        upper_bound = dataset.shape[1]\n",
-    "        self.param.number_features.bounds = (0, upper_bound)\n",
     "        self.param.target_features.bounds = (0, upper_bound)\n",
-    "        self.param.ignore_features.objects = dataset.columns.to_list() + [None]\n",
+    "        self.param.ignore_features.objects = [None] + dataset.columns.to_list()\n",
+    "        self.param.target.default = dataset.columns.tolist()[-1]\n",
     "        super(Widget, self).__init__(dataset=dataset, **kwargs)\n",
+    "        self.button = pn.widgets.Button(name='Start process!', button_type=\"success\") \n",
+    "        self.button.on_click(self.run_process)\n",
     "    \n",
+    "    @param.depends('optimize', watch=True)\n",
+    "    def update_precedence(self):\n",
+    "        self.param.opt_list.precedence = 1 if self.optimize else -1\n",
+    "        \n",
     "    @param.depends(\"acu\", \"auc\", \"rec\", \"pre\", \"f1\", \"kappa\", \"mcc\", watch=True)\n",
-    "    def _define_metric_dict(self):\n",
+    "    def _define_filter_metrics(self):\n",
     "        legend = [\"acu\", \"auc\", \"rec\", \"pre\", \"f1\", \"kappa\", \"mcc\"]\n",
-    "        self.metric_dict = {key: getattr(self, key) for key in legend} \n",
-    "        \n",
-    "    @staticmethod\n",
-    "    def _button():\n",
-    "        return pn.widgets.Button(name='Start process!', button_type=\"success\") \n",
+    "        self.filter_metrics = {key: getattr(self, key) for key in legend} \n",
+    "    \n",
+    "    \n",
+    "    def _discrete(self):\n",
+    "        options = np.linspace(0,1,100,endpoint=False)[1:].tolist() + np.linspace(1,self.dataset.shape[1],200, dtype=int).tolist()\n",
+    "        return {'widget_type': pn.widgets.DiscreteSlider,\n",
+    "                'name': 'Target features', \n",
+    "                'options': options, \n",
+    "                'value': 0.3,\n",
+    "                'width': 280}\n",
+    "\n",
     "        \n",
     "    def interactive_panel(self):\n",
-    "        first_col = ['target', 'number_features', 'target_features', 'feature_division', 'ignore_features', 'sort']\n",
-    "        second_col = ['include', 'number_models', 'optimize', 'acu', 'auc', 'rec', 'pre', 'f1', 'kappa', 'mcc']\n",
-    "        button = self._button()\n",
-    "        self.start_run = button.clicks\n",
+    "        first_col = ['target', 'number_features', 'target_features', 'ignore_features', 'sort', 'include', 'number_models']\n",
+    "        second_col = ['acu', 'auc', 'rec', 'pre', 'f1', 'kappa', 'mcc', 'optimize', 'opt_list']\n",
     "        view = pn.Row(\n",
     "            pn.Column(\n",
     "                pn.panel(self.param,\n",
     "                         parameters=[c for c in first_col],\n",
-    "                         name='Feature parameters')\n",
+    "                         name='Feature parameters and models included',\n",
+    "                         widgets={'target_features': self._discrete()}\n",
+    "                        )\n",
     "            ),\n",
     "            pn.Column(\n",
     "                pn.panel(self.param,\n",
     "                         parameters=[c for c in second_col],\n",
-    "                         name='Models included and metric parameters')\n",
+    "                         name='Metric and optimize parameters')\n",
     "            ),\n",
     "            pn.Column(\n",
-    "                pn.panel(button, name=\"Run the library to select relevant features\")\n",
+    "                pn.panel(self.button, name=\"Run the library to select relevant features\")\n",
     "            ),\n",
-    "            pn.Column(self._cosa)\n",
     "        )\n",
     "        return view\n",
     "    \n",
-    "    @param.depends(\"start_run\")\n",
-    "    def _cosa(self):\n",
-    "        self.target = 'goals_4.5' if self.start_run > 4 else 'miau'\n",
-    "        #time.sleep(1)"
+    "    def cosita(self):\n",
+    "        return range(10, **self.opt_kwargs)\n",
+    "\n",
+    "    def run_process(self):\n",
+    "        inst = FeatureSelection(dataset = self.dataset, target=self.target, number_features=self.nu)\n",
+    "        "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 285,
    "id": "14ad638b-b851-4d51-b909-2c2fd909dd75",
    "metadata": {
     "scrolled": false
@@ -1749,18 +1355,18 @@
      "data": {
       "application/vnd.holoviews_exec.v0+json": "",
       "text/html": [
-       "<div id='1264'>\n",
+       "<div id='2266'>\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
-       "  <div class=\"bk-root\" id=\"d57b4d6d-2dc8-482a-8910-fe160b0aac7f\" data-root-id=\"1264\"></div>\n",
+       "  <div class=\"bk-root\" id=\"43bf8ba1-0223-4d56-b328-a9210fb5c081\" data-root-id=\"2266\"></div>\n",
        "</div>\n",
        "<script type=\"application/javascript\">(function(root) {\n",
        "  function embed_document(root) {\n",
-       "    var docs_json = {\"f51a7389-4bb2-4f79-a18d-fd7f9ebf9217\":{\"defs\":[{\"extends\":null,\"module\":null,\"name\":\"ReactiveHTML1\",\"overrides\":[],\"properties\":[]},{\"extends\":null,\"module\":null,\"name\":\"FlexBox1\",\"overrides\":[],\"properties\":[{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_content\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_items\"},{\"default\":\"row\",\"kind\":null,\"name\":\"flex_direction\"},{\"default\":\"wrap\",\"kind\":null,\"name\":\"flex_wrap\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"justify_content\"}]},{\"extends\":null,\"module\":null,\"name\":\"TemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]},{\"extends\":null,\"module\":null,\"name\":\"MaterialTemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]}],\"roots\":{\"references\":[{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Precision\",\"value\":0.6},\"id\":\"1280\",\"type\":\"Slider\"},{\"attributes\":{\"children\":[{\"id\":\"1275\"}],\"margin\":[0,0,0,0],\"name\":\"Column00561\"},\"id\":\"1274\",\"type\":\"Column\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"F-score\",\"value\":0.6},\"id\":\"1281\",\"type\":\"Slider\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Kappa\",\"value\":0.1},\"id\":\"1282\",\"type\":\"Slider\"},{\"attributes\":{\"client_comm_id\":\"0fb6beeb644e4a98b1ee33c402521f16\",\"comm_id\":\"1ce782ad43b14c5cb352f4b6d2bec6d8\",\"plot_id\":\"1264\"},\"id\":\"1292\",\"type\":\"panel.models.comm_manager.CommManager\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"MCC\",\"value\":0.1},\"id\":\"1283\",\"type\":\"Slider\"},{\"attributes\":{\"margin\":[5,10,5,10],\"name\":\"\",\"text\":\"<b>Feature parameters</b>\"},\"id\":\"1267\",\"type\":\"Div\"},{\"attributes\":{\"children\":[{\"id\":\"1265\"},{\"id\":\"1274\"},{\"id\":\"1287\"},{\"id\":\"1289\"}],\"margin\":[0,0,0,0],\"name\":\"Row00569\"},\"id\":\"1264\",\"type\":\"Row\"},{\"attributes\":{\"end\":15,\"margin\":[5,10,5,10],\"start\":2,\"title\":\"Number models\",\"value\":10},\"id\":\"1285\",\"type\":\"Slider\"},{\"attributes\":{\"margin\":[5,10,5,10],\"max_length\":5000,\"title\":\"Target\",\"value\":\"miau\"},\"id\":\"1268\",\"type\":\"TextInput\"},{\"attributes\":{\"margin\":[5,10,5,10],\"options\":[\"a\",\"b\",\"None\"],\"title\":\"Ignore features\"},\"id\":\"1272\",\"type\":\"MultiSelect\"},{\"attributes\":{\"children\":[{\"id\":\"1288\"}],\"margin\":[0,0,0,0],\"name\":\"Column00562\"},\"id\":\"1287\",\"type\":\"Column\"},{\"attributes\":{\"end\":2,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Number features\",\"value\":0.5},\"id\":\"1269\",\"type\":\"Slider\"},{\"attributes\":{\"labels\":[\"Tuning process\"],\"margin\":[5,10,5,10]},\"id\":\"1286\",\"type\":\"CheckboxGroup\"},{\"attributes\":{\"children\":[{\"id\":\"1276\"},{\"id\":\"1277\"},{\"id\":\"1278\"},{\"id\":\"1279\"},{\"id\":\"1280\"},{\"id\":\"1281\"},{\"id\":\"1282\"},{\"id\":\"1283\"},{\"id\":\"1284\"},{\"id\":\"1285\"},{\"id\":\"1286\"}],\"margin\":[5,5,5,5],\"min_width\":300,\"name\":\"Models included and metric parameters\",\"width\":300},\"id\":\"1275\",\"type\":\"Column\"},{\"attributes\":{\"end\":100,\"margin\":[5,10,5,10],\"start\":1,\"step\":0.1,\"title\":\"Feature division\",\"value\":3},\"id\":\"1271\",\"type\":\"Slider\"},{\"attributes\":{\"end\":2,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Target features\",\"value\":0.3},\"id\":\"1270\",\"type\":\"Slider\"},{\"attributes\":{\"margin\":[5,10,5,10],\"name\":\"\",\"text\":\"<b>Models included and metric parameters</b>\"},\"id\":\"1276\",\"type\":\"Div\"},{\"attributes\":{\"button_type\":\"success\",\"icon\":null,\"label\":\"Start process!\",\"margin\":[5,10,5,10],\"subscribed_events\":[\"button_click\"]},\"id\":\"1288\",\"type\":\"Button\"},{\"attributes\":{\"margin\":[5,10,5,10],\"options\":[\"Accuracy\",\"AUC\",\"Recall\",\"Precision\",\"F1\",\"Kappa\",\"MCC\"],\"title\":\"Sort\",\"value\":[\"AUC\"]},\"id\":\"1273\",\"type\":\"MultiSelect\"},{\"attributes\":{\"margin\":[5,5,5,5],\"name\":\"Str00565\",\"text\":\"&lt;pre&gt; &lt;/pre&gt;\"},\"id\":\"1291\",\"type\":\"panel.models.markup.HTML\"},{\"attributes\":{\"margin\":[5,10,5,10],\"options\":[\"lr\",\"dt\",\"svm\",\"rbfsvm\",\"gpc\",\"mlp\",\"ridge\",\"rf\",\"ada\",\"gbc\",\"lda\",\"et\",\"xgboost\",\"lightgbm\",\"catboost\",\"None\"],\"title\":\"Include\"},\"id\":\"1284\",\"type\":\"MultiSelect\"},{\"attributes\":{\"children\":[{\"id\":\"1267\"},{\"id\":\"1268\"},{\"id\":\"1269\"},{\"id\":\"1270\"},{\"id\":\"1271\"},{\"id\":\"1272\"},{\"id\":\"1273\"}],\"margin\":[5,5,5,5],\"min_width\":300,\"name\":\"Feature parameters\",\"width\":300},\"id\":\"1266\",\"type\":\"Column\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Recall\",\"value\":0.6},\"id\":\"1279\",\"type\":\"Slider\"},{\"attributes\":{\"children\":[{\"id\":\"1291\"}],\"margin\":[0,0,0,0],\"name\":\"Row00567\"},\"id\":\"1290\",\"type\":\"Row\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Accuracy\",\"value\":0.6},\"id\":\"1277\",\"type\":\"Slider\"},{\"attributes\":{\"children\":[{\"id\":\"1290\"}],\"margin\":[0,0,0,0],\"name\":\"Column00568\"},\"id\":\"1289\",\"type\":\"Column\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"AUC\",\"value\":0.6},\"id\":\"1278\",\"type\":\"Slider\"},{\"attributes\":{\"children\":[{\"id\":\"1266\"}],\"margin\":[0,0,0,0],\"name\":\"Column00544\"},\"id\":\"1265\",\"type\":\"Column\"}],\"root_ids\":[\"1264\",\"1292\"]},\"title\":\"Bokeh Application\",\"version\":\"2.4.2\"}};\n",
-       "    var render_items = [{\"docid\":\"f51a7389-4bb2-4f79-a18d-fd7f9ebf9217\",\"root_ids\":[\"1264\"],\"roots\":{\"1264\":\"d57b4d6d-2dc8-482a-8910-fe160b0aac7f\"}}];\n",
+       "    var docs_json = {\"0fb65867-6b3e-43ae-a5a0-be2405b34da2\":{\"defs\":[{\"extends\":null,\"module\":null,\"name\":\"ReactiveHTML1\",\"overrides\":[],\"properties\":[]},{\"extends\":null,\"module\":null,\"name\":\"FlexBox1\",\"overrides\":[],\"properties\":[{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_content\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_items\"},{\"default\":\"row\",\"kind\":null,\"name\":\"flex_direction\"},{\"default\":\"wrap\",\"kind\":null,\"name\":\"flex_wrap\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"justify_content\"}]},{\"extends\":null,\"module\":null,\"name\":\"TemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]},{\"extends\":null,\"module\":null,\"name\":\"MaterialTemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]}],\"roots\":{\"references\":[{\"attributes\":{\"margin\":[5,0,0,10],\"name\":\"\",\"style\":{\"white-space\":\"nowrap\"},\"text\":\"Target features: <b>0.3</b>\",\"width\":280},\"id\":\"2273\",\"type\":\"Div\"},{\"attributes\":{\"margin\":[5,10,5,10],\"options\":[\"None\",\"lr\",\"dt\",\"svm\",\"rbfsvm\",\"gpc\",\"mlp\",\"ridge\",\"rf\",\"ada\",\"gbc\",\"lda\",\"et\",\"xgboost\",\"lightgbm\",\"catboost\"],\"title\":\"Include\"},\"id\":\"2277\",\"type\":\"MultiSelect\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Accuracy\",\"value\":0.6},\"id\":\"2282\",\"type\":\"Slider\"},{\"attributes\":{\"margin\":[5,10,5,10],\"name\":\"\",\"text\":\"<b>Metric and optimize parameters</b>\"},\"id\":\"2281\",\"type\":\"Div\"},{\"attributes\":{\"client_comm_id\":\"9b6421de7b744e4797636d05bb659b09\",\"comm_id\":\"197aa0799f144e418bd90a5a77f08447\",\"plot_id\":\"2266\"},\"id\":\"2293\",\"type\":\"panel.models.comm_manager.CommManager\"},{\"attributes\":{\"children\":[{\"id\":\"2269\"},{\"id\":\"2270\"},{\"id\":\"2271\"},{\"id\":\"2272\"},{\"id\":\"2275\"},{\"id\":\"2276\"},{\"id\":\"2277\"},{\"id\":\"2278\"}],\"margin\":[5,5,5,5],\"min_width\":300,\"name\":\"Feature parameters and models included\",\"width\":300},\"id\":\"2268\",\"type\":\"Column\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"F-score\",\"value\":0.6},\"id\":\"2286\",\"type\":\"Slider\"},{\"attributes\":{\"args\":{\"bidirectional\":false,\"properties\":{},\"source\":{\"id\":\"2274\"},\"target\":{\"id\":\"2273\"}},\"code\":\"try { \\n    var labels = ['Target features: <b>0.01</b>', 'Target features: <b>0.02</b>', 'Target features: <b>0.03</b>', 'Target features: <b>0.04</b>', 'Target features: <b>0.05</b>', 'Target features: <b>0.06</b>', 'Target features: <b>0.07</b>', 'Target features: <b>0.08</b>', 'Target features: <b>0.09</b>', 'Target features: <b>0.1</b>', 'Target features: <b>0.11</b>', 'Target features: <b>0.12</b>', 'Target features: <b>0.13</b>', 'Target features: <b>0.14</b>', 'Target features: <b>0.15</b>', 'Target features: <b>0.16</b>', 'Target features: <b>0.17</b>', 'Target features: <b>0.18</b>', 'Target features: <b>0.19</b>', 'Target features: <b>0.2</b>', 'Target features: <b>0.21</b>', 'Target features: <b>0.22</b>', 'Target features: <b>0.23</b>', 'Target features: <b>0.24</b>', 'Target features: <b>0.25</b>', 'Target features: <b>0.26</b>', 'Target features: <b>0.27</b>', 'Target features: <b>0.28</b>', 'Target features: <b>0.29</b>', 'Target features: <b>0.3</b>', 'Target features: <b>0.31</b>', 'Target features: <b>0.32</b>', 'Target features: <b>0.33</b>', 'Target features: <b>0.34</b>', 'Target features: <b>0.35</b>', 'Target features: <b>0.36</b>', 'Target features: <b>0.37</b>', 'Target features: <b>0.38</b>', 'Target features: <b>0.39</b>', 'Target features: <b>0.4</b>', 'Target features: <b>0.41</b>', 'Target features: <b>0.42</b>', 'Target features: <b>0.43</b>', 'Target features: <b>0.44</b>', 'Target features: <b>0.45</b>', 'Target features: <b>0.46</b>', 'Target features: <b>0.47</b>', 'Target features: <b>0.48</b>', 'Target features: <b>0.49</b>', 'Target features: <b>0.5</b>', 'Target features: <b>0.51</b>', 'Target features: <b>0.52</b>', 'Target features: <b>0.53</b>', 'Target features: <b>0.54</b>', 'Target features: <b>0.55</b>', 'Target features: <b>0.56</b>', 'Target features: <b>0.57</b>', 'Target features: <b>0.58</b>', 'Target features: <b>0.59</b>', 'Target features: <b>0.6</b>', 'Target features: <b>0.61</b>', 'Target features: <b>0.62</b>', 'Target features: <b>0.63</b>', 'Target features: <b>0.64</b>', 'Target features: <b>0.65</b>', 'Target features: <b>0.66</b>', 'Target features: <b>0.67</b>', 'Target features: <b>0.68</b>', 'Target features: <b>0.69</b>', 'Target features: <b>0.7</b>', 'Target features: <b>0.71</b>', 'Target features: <b>0.72</b>', 'Target features: <b>0.73</b>', 'Target features: <b>0.74</b>', 'Target features: <b>0.75</b>', 'Target features: <b>0.76</b>', 'Target features: <b>0.77</b>', 'Target features: <b>0.78</b>', 'Target features: <b>0.79</b>', 'Target features: <b>0.8</b>', 'Target features: <b>0.81</b>', 'Target features: <b>0.82</b>', 'Target features: <b>0.83</b>', 'Target features: <b>0.84</b>', 'Target features: <b>0.85</b>', 'Target features: <b>0.86</b>', 'Target features: <b>0.87</b>', 'Target features: <b>0.88</b>', 'Target features: <b>0.89</b>', 'Target features: <b>0.9</b>', 'Target features: <b>0.91</b>', 'Target features: <b>0.92</b>', 'Target features: <b>0.93</b>', 'Target features: <b>0.94</b>', 'Target features: <b>0.95</b>', 'Target features: <b>0.96</b>', 'Target features: <b>0.97</b>', 'Target features: <b>0.98</b>', 'Target features: <b>0.99</b>', 'Target features: <b>1</b>', 'Target features: <b>11</b>', 'Target features: <b>21</b>', 'Target features: <b>32</b>', 'Target features: <b>42</b>', 'Target features: <b>52</b>', 'Target features: <b>63</b>', 'Target features: <b>73</b>', 'Target features: <b>83</b>', 'Target features: <b>94</b>', 'Target features: <b>104</b>', 'Target features: <b>114</b>', 'Target features: <b>125</b>', 'Target features: <b>135</b>', 'Target features: <b>146</b>', 'Target features: <b>156</b>', 'Target features: <b>166</b>', 'Target features: <b>177</b>', 'Target features: <b>187</b>', 'Target features: <b>197</b>', 'Target features: <b>208</b>', 'Target features: <b>218</b>', 'Target features: <b>228</b>', 'Target features: <b>239</b>', 'Target features: <b>249</b>', 'Target features: <b>260</b>', 'Target features: <b>270</b>', 'Target features: <b>280</b>', 'Target features: <b>291</b>', 'Target features: <b>301</b>', 'Target features: <b>311</b>', 'Target features: <b>322</b>', 'Target features: <b>332</b>', 'Target features: <b>342</b>', 'Target features: <b>353</b>', 'Target features: <b>363</b>', 'Target features: <b>374</b>', 'Target features: <b>384</b>', 'Target features: <b>394</b>', 'Target features: <b>405</b>', 'Target features: <b>415</b>', 'Target features: <b>425</b>', 'Target features: <b>436</b>', 'Target features: <b>446</b>', 'Target features: <b>456</b>', 'Target features: <b>467</b>', 'Target features: <b>477</b>', 'Target features: <b>488</b>', 'Target features: <b>498</b>', 'Target features: <b>508</b>', 'Target features: <b>519</b>', 'Target features: <b>529</b>', 'Target features: <b>539</b>', 'Target features: <b>550</b>', 'Target features: <b>560</b>', 'Target features: <b>570</b>', 'Target features: <b>581</b>', 'Target features: <b>591</b>', 'Target features: <b>601</b>', 'Target features: <b>612</b>', 'Target features: <b>622</b>', 'Target features: <b>633</b>', 'Target features: <b>643</b>', 'Target features: <b>653</b>', 'Target features: <b>664</b>', 'Target features: <b>674</b>', 'Target features: <b>684</b>', 'Target features: <b>695</b>', 'Target features: <b>705</b>', 'Target features: <b>715</b>', 'Target features: <b>726</b>', 'Target features: <b>736</b>', 'Target features: <b>747</b>', 'Target features: <b>757</b>', 'Target features: <b>767</b>', 'Target features: <b>778</b>', 'Target features: <b>788</b>', 'Target features: <b>798</b>', 'Target features: <b>809</b>', 'Target features: <b>819</b>', 'Target features: <b>829</b>', 'Target features: <b>840</b>', 'Target features: <b>850</b>', 'Target features: <b>861</b>', 'Target features: <b>871</b>', 'Target features: <b>881</b>', 'Target features: <b>892</b>', 'Target features: <b>902</b>', 'Target features: <b>912</b>', 'Target features: <b>923</b>', 'Target features: <b>933</b>', 'Target features: <b>943</b>', 'Target features: <b>954</b>', 'Target features: <b>964</b>', 'Target features: <b>975</b>', 'Target features: <b>985</b>', 'Target features: <b>995</b>', 'Target features: <b>1.01e+03</b>', 'Target features: <b>1.02e+03</b>', 'Target features: <b>1.03e+03</b>', 'Target features: <b>1.04e+03</b>', 'Target features: <b>1.05e+03</b>', 'Target features: <b>1.06e+03</b>', 'Target features: <b>1.07e+03</b>', 'Target features: <b>1.08e+03</b>', 'Target features: <b>1.09e+03</b>', 'Target features: <b>1.1e+03</b>', 'Target features: <b>1.11e+03</b>', 'Target features: <b>1.12e+03</b>', 'Target features: <b>1.13e+03</b>', 'Target features: <b>1.14e+03</b>', 'Target features: <b>1.15e+03</b>', 'Target features: <b>1.16e+03</b>', 'Target features: <b>1.17e+03</b>', 'Target features: <b>1.18e+03</b>', 'Target features: <b>1.19e+03</b>', 'Target features: <b>1.2e+03</b>', 'Target features: <b>1.21e+03</b>', 'Target features: <b>1.22e+03</b>', 'Target features: <b>1.23e+03</b>', 'Target features: <b>1.24e+03</b>', 'Target features: <b>1.25e+03</b>', 'Target features: <b>1.26e+03</b>', 'Target features: <b>1.28e+03</b>', 'Target features: <b>1.28e+03</b>', 'Target features: <b>1.3e+03</b>', 'Target features: <b>1.31e+03</b>', 'Target features: <b>1.32e+03</b>', 'Target features: <b>1.33e+03</b>', 'Target features: <b>1.34e+03</b>', 'Target features: <b>1.35e+03</b>', 'Target features: <b>1.36e+03</b>', 'Target features: <b>1.37e+03</b>', 'Target features: <b>1.38e+03</b>', 'Target features: <b>1.39e+03</b>', 'Target features: <b>1.4e+03</b>', 'Target features: <b>1.41e+03</b>', 'Target features: <b>1.42e+03</b>', 'Target features: <b>1.43e+03</b>', 'Target features: <b>1.44e+03</b>', 'Target features: <b>1.45e+03</b>', 'Target features: <b>1.46e+03</b>', 'Target features: <b>1.47e+03</b>', 'Target features: <b>1.48e+03</b>', 'Target features: <b>1.49e+03</b>', 'Target features: <b>1.5e+03</b>', 'Target features: <b>1.51e+03</b>', 'Target features: <b>1.52e+03</b>', 'Target features: <b>1.53e+03</b>', 'Target features: <b>1.54e+03</b>', 'Target features: <b>1.56e+03</b>', 'Target features: <b>1.56e+03</b>', 'Target features: <b>1.58e+03</b>', 'Target features: <b>1.59e+03</b>', 'Target features: <b>1.6e+03</b>', 'Target features: <b>1.61e+03</b>', 'Target features: <b>1.62e+03</b>', 'Target features: <b>1.63e+03</b>', 'Target features: <b>1.64e+03</b>', 'Target features: <b>1.65e+03</b>', 'Target features: <b>1.66e+03</b>', 'Target features: <b>1.67e+03</b>', 'Target features: <b>1.68e+03</b>', 'Target features: <b>1.69e+03</b>', 'Target features: <b>1.7e+03</b>', 'Target features: <b>1.71e+03</b>', 'Target features: <b>1.72e+03</b>', 'Target features: <b>1.73e+03</b>', 'Target features: <b>1.74e+03</b>', 'Target features: <b>1.75e+03</b>', 'Target features: <b>1.76e+03</b>', 'Target features: <b>1.77e+03</b>', 'Target features: <b>1.78e+03</b>', 'Target features: <b>1.79e+03</b>', 'Target features: <b>1.8e+03</b>', 'Target features: <b>1.81e+03</b>', 'Target features: <b>1.82e+03</b>', 'Target features: <b>1.84e+03</b>', 'Target features: <b>1.84e+03</b>', 'Target features: <b>1.86e+03</b>', 'Target features: <b>1.87e+03</b>', 'Target features: <b>1.88e+03</b>', 'Target features: <b>1.89e+03</b>', 'Target features: <b>1.9e+03</b>', 'Target features: <b>1.91e+03</b>', 'Target features: <b>1.92e+03</b>', 'Target features: <b>1.93e+03</b>', 'Target features: <b>1.94e+03</b>', 'Target features: <b>1.95e+03</b>', 'Target features: <b>1.96e+03</b>', 'Target features: <b>1.97e+03</b>', 'Target features: <b>1.98e+03</b>', 'Target features: <b>1.99e+03</b>', 'Target features: <b>2e+03</b>', 'Target features: <b>2.01e+03</b>', 'Target features: <b>2.02e+03</b>', 'Target features: <b>2.03e+03</b>', 'Target features: <b>2.04e+03</b>', 'Target features: <b>2.05e+03</b>', 'Target features: <b>2.06e+03</b>']\\n    target.text = labels[source.value]\\n     } catch(err) { console.log(err) }\",\"tags\":[4756277856]},\"id\":\"2292\",\"type\":\"CustomJS\"},{\"attributes\":{\"margin\":[5,10,5,10],\"options\":[[\"Accuracy\",\"Accuracy\"],[\"AUC\",\"AUC\"],[\"Recall\",\"Recall\"],[\"Precision\",\"Precision\"],[\"F1\",\"F1\"],[\"Kappa\",\"Kappa\"],[\"MCC\",\"MCC\"]],\"title\":\"Sort\",\"value\":\"AUC\"},\"id\":\"2276\",\"type\":\"Select\"},{\"attributes\":{\"button_type\":\"success\",\"icon\":null,\"label\":\"Start process!\",\"margin\":[5,10,5,10],\"subscribed_events\":[\"button_click\"]},\"id\":\"2291\",\"type\":\"Button\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Number features\",\"value\":0.5},\"id\":\"2271\",\"type\":\"Slider\"},{\"attributes\":{\"end\":15,\"margin\":[5,10,5,10],\"start\":2,\"title\":\"Number models\",\"value\":10},\"id\":\"2278\",\"type\":\"Slider\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Precision\",\"value\":0.6},\"id\":\"2285\",\"type\":\"Slider\"},{\"attributes\":{\"end\":298,\"js_property_callbacks\":{\"change:value\":[{\"id\":\"2292\"}]},\"margin\":[0,10,5,10],\"min_width\":280,\"show_value\":false,\"start\":0,\"tooltips\":false,\"value\":29,\"width\":280},\"id\":\"2274\",\"type\":\"Slider\"},{\"attributes\":{\"children\":[{\"id\":\"2267\"},{\"id\":\"2279\"},{\"id\":\"2290\"}],\"margin\":[0,0,0,0],\"name\":\"Row02212\"},\"id\":\"2266\",\"type\":\"Row\"},{\"attributes\":{\"margin\":[5,10,5,10],\"name\":\"\",\"text\":\"<b>Feature parameters and models included</b>\"},\"id\":\"2269\",\"type\":\"Div\"},{\"attributes\":{\"children\":[{\"id\":\"2291\"}],\"margin\":[0,0,0,0],\"name\":\"Column02211\"},\"id\":\"2290\",\"type\":\"Column\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"MCC\",\"value\":0.1},\"id\":\"2288\",\"type\":\"Slider\"},{\"attributes\":{\"children\":[{\"id\":\"2280\"}],\"margin\":[0,0,0,0],\"name\":\"Column02210\"},\"id\":\"2279\",\"type\":\"Column\"},{\"attributes\":{\"children\":[{\"id\":\"2268\"}],\"margin\":[0,0,0,0],\"name\":\"Column02196\"},\"id\":\"2267\",\"type\":\"Column\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"AUC\",\"value\":0.6},\"id\":\"2283\",\"type\":\"Slider\"},{\"attributes\":{\"labels\":[\"Tuning process\"],\"margin\":[5,10,5,10]},\"id\":\"2289\",\"type\":\"CheckboxGroup\"},{\"attributes\":{\"children\":[{\"id\":\"2273\"},{\"id\":\"2274\"}],\"margin\":[0,0,0,0],\"min_width\":300,\"name\":\"Column02185\",\"width\":300},\"id\":\"2272\",\"type\":\"Column\"},{\"attributes\":{\"margin\":[5,10,5,10],\"max_length\":5000,\"title\":\"Target\",\"value\":\"Score\"},\"id\":\"2270\",\"type\":\"TextInput\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Recall\",\"value\":0.6},\"id\":\"2284\",\"type\":\"Slider\"},{\"attributes\":{\"margin\":[5,10,5,10],\"options\":[\"None\",\"match_id\",\"win_pct_h\",\"draw_pct_h\",\"loss_pct_h\",\"win_streak_h\",\"loss_streak_h\",\"draw_streak_h\",\"points_h\",\"ranking_h\",\"ATT_h\",\"DEF_h\",\"MED_h\",\"acc_corners_intobox_pct_mean_h\",\"att_pen_miss_mean_h\",\"att_rf_goal_mean_h\",\"att_goal_high_centre_mean_h\",\"att_sv_high_left_mean_h\",\"att_obox_blocked_mean_h\",\"att_goal_high_left_mean_h\",\"shots_mul_goals_mean_h\",\"first_yellow_card_1t_mean_h\",\"goals_mean_h\",\"att_post_high_mean_h\",\"att_hd_target_mean_h\",\"shots_div_passes_right_mean_h\",\"att_freekick_miss_mean_h\",\"corners_ratio_mean_h\",\"att_freekick_goal_mean_h\",\"att_miss_high_mean_h\",\"fifty_fifty_mean_h\",\"put_through_mean_h\",\"poss_lost_ctrl_mean_h\",\"att_pen_post_mean_h\",\"leftside_pass_mean_h\",\"total_corners_intobox_mean_h\",\"final_third_entries_mean_h\",\"total_through_ball_mean_h\",\"total_offside_mean_h\",\"total_layoffs_mean_h\",\"crosses_18yardplus_mean_h\",\"winner_mean_h\",\"att_obox_own_goal_mean_h\",\"accurate_long_balls_mean_h\",\"goal_assist_openplay_mean_h\",\"passes_right_div_blocked_pass_mean_h\",\"saved_ibox_mean_h\",\"red_card_2t_mean_h\",\"goals_1t_mean_h\",\"no_foot_goals_ratio_mean_h\",\"att_obxd_left_mean_h\",\"att_obp_goal_mean_h\",\"total_cross_nocorner_mean_h\",\"touches_mean_h\",\"foul_throw_in_mean_h\",\"corner_taken_mean_h\",\"duel_won_pct_mean_h\",\"error_lead_to_shot_mean_h\",\"goals_2t_pct_mean_h\",\"goal_assist_setplay_mean_h\",\"first_yellow_card_2t_mean_h\",\"att_goal_low_left_mean_h\",\"total_yel_card_mean_h\",\"accurate_through_ball_mean_h\",\"post_scoring_att_mean_h\",\"ontarget_att_assist_mean_h\",\"open_play_pass_mean_h\",\"accurate_throws_mean_h\",\"right_to_left_goals_mean_h\",\"total_launches_mean_h\",\"att_miss_right_mean_h\",\"attempted_tackle_foul_mean_h\",\"total_final_third_passes_mean_h\",\"head_clearance_mean_h\",\"att_sv_low_centre_mean_h\",\"poss_won_att_3rd_mean_h\",\"att_bx_centre_mean_h\",\"lineup_mean_h\",\"att_one_on_one_mean_h\",\"total_chipped_pass_mean_h\",\"attempts_conceded_obox_mean_h\",\"big_chance_created_mean_h\",\"accurate_corners_intobox_mean_h\",\"ontarget_scoring_att_mean_h\",\"diving_save_mean_h\",\"saves_mean_h\",\"high_to_low_goals_mean_h\",\"accurate_layoffs_mean_h\",\"goals_2t_mean_h\",\"total_keeper_sweeper_mean_h\",\"ranking_mean_h\",\"first_yellow_card_mean_h\",\"att_obx_right_mean_h\",\"att_obox_miss_mean_h\",\"bc_miss_div_scored_mean_h\",\"six_yard_block_mean_h\",\"att_rf_target_mean_h\",\"bc_scored_div_created_mean_h\",\"att_rf_total_mean_h\",\"outfielder_block_mean_h\",\"saved_obox_mean_h\",\"won_corners_ratio_mean_h\",\"rightside_pass_div_leftside_pass_mean_h\",\"good_high_claim_mean_h\",\"accurate_back_zone_pass_div_backward_pass_mean_h\",\"attempts_ibox_mean_h\",\"att_hd_goal_mean_h\",\"rightside_pass_div_accurate_pass_mean_h\",\"long_passes_mean_h\",\"second_yellow_card_2t_mean_h\",\"challenge_lost_mean_h\",\"att_ibox_goal_mean_h\",\"points_mean_h\",\"total_contest_mean_h\",\"bc_miss_div_created_mean_h\",\"goal_assist_deadball_mean_h\",\"att_obxd_right_mean_h\",\"fwd_pass_mean_h\",\"att_lg_centre_mean_h\",\"last_man_tackle_mean_h\",\"goal_kicks_per_shot_mean_h\",\"fwd_pass_div_long_pass_own_to_opp_success_mean_h\",\"att_goal_low_centre_mean_h\",\"att_openplay_mean_h\",\"pen_area_entries_mean_h\",\"effective_clearance_mean_h\",\"long_pass_own_to_opp_mean_h\",\"unsuccessful_touch_mean_h\",\"accurate_flick_on_mean_h\",\"att_sv_high_right_mean_h\",\"overrun_mean_h\",\"touches_in_opp_box_mean_h\",\"total_pull_back_mean_h\",\"att_cmiss_right_mean_h\",\"goal_fastbreak_mean_h\",\"blocked_cross_mean_h\",\"big_chance_scored_mean_h\",\"attempts_obox_mean_h\",\"lost_corners_mean_h\",\"att_goal_low_right_mean_h\",\"passes_right_mean_h\",\"passes_left_div_blocked_pass_mean_h\",\"att_cmiss_high_mean_h\",\"long_pass_own_to_opp_success_mean_h\",\"accurate_cross_nocorner_mean_h\",\"total_red_card_mean_h\",\"fk_foul_lost_mean_h\",\"accurate_fwd_zone_pass_mean_h\",\"second_yellow_card_1t_mean_h\",\"failed_to_block_mean_h\",\"accurate_pass_mean_h\",\"leftside_pass_div_accurate_pass_mean_h\",\"red_card_1t_mean_h\",\"dispossessed_mean_h\",\"pts_gained_losing_pos_mean_h\",\"rightside_pass_mean_h\",\"shield_ball_oop_mean_h\",\"midfielder_goals_mean_h\",\"interceptions_in_box_mean_h\",\"att_lg_right_mean_h\",\"won_corners_mean_h\",\"att_pen_goal_mean_h\",\"forward_goals_mean_h\",\"accurate_cross_mean_h\",\"acc_cross_nocorner_pct_mean_h\",\"att_miss_high_left_mean_h\",\"total_att_assist_mean_h\",\"att_lf_target_mean_h\",\"att_cmiss_high_right_mean_h\",\"punches_mean_h\",\"contentious_decision_mean_h\",\"att_freekick_post_mean_h\",\"interception_mean_h\",\"total_throws_mean_h\",\"crosses_18yard_mean_h\",\"accurate_pull_back_mean_h\",\"att_obox_goal_mean_h\",\"fk_foul_won_mean_h\",\"accurate_launches_mean_h\",\"goal_assist_mean_h\",\"att_lf_total_mean_h\",\"rescinded_red_card_mean_h\",\"backward_pass_mean_h\",\"accurate_keeper_throws_mean_h\",\"att_ibox_own_goal_mean_h\",\"accurate_back_zone_pass_mean_h\",\"hit_woodwork_mean_h\",\"att_freekick_total_mean_h\",\"accurate_goal_kicks_mean_h\",\"won_tackle_mean_h\",\"corners_mean_h\",\"own_goals_mean_h\",\"blocked_scoring_att_mean_h\",\"poss_lost_all_mean_h\",\"att_hd_miss_mean_h\",\"shots_mean_h\",\"att_ibox_target_mean_h\",\"duel_won_mean_h\",\"crosses_mean_h\",\"att_ibox_post_mean_h\",\"total_clearance_mean_h\",\"attempts_conceded_ibox_mean_h\",\"big_chance_missed_mean_h\",\"goal_assist_intent_norm_mean_h\",\"att_miss_high_right_mean_h\",\"red_card_mean_h\",\"passes_left_mean_h\",\"left_div_right_foot_goals_mean_h\",\"successful_fifty_fifty_mean_h\",\"penalty_save_mean_h\",\"att_obox_post_mean_h\",\"att_lf_goal_mean_h\",\"penalty_conceded_mean_h\",\"freekick_cross_mean_h\",\"error_lead_to_goal_mean_h\",\"duel_lost_mean_h\",\"att_bx_left_mean_h\",\"goals_openplay_mean_h\",\"shots_div_passes_left_mean_h\",\"att_goal_high_right_mean_h\",\"total_fwd_zone_pass_mean_h\",\"corners_2t_mean_h\",\"goal_kicks_mean_h\",\"penalty_won_mean_h\",\"goal_kicks_div_long_passes_mean_h\",\"goals_conceded_obox_mean_h\",\"DEF_mean_h\",\"att_obx_centre_mean_h\",\"att_assist_openplay_mean_h\",\"total_flick_on_mean_h\",\"att_bx_right_mean_h\",\"subs_made_mean_h\",\"goals_conceded_ibox_mean_h\",\"att_obx_left_mean_h\",\"pen_goals_conceded_mean_h\",\"won_contest_mean_h\",\"att_obox_target_mean_h\",\"shot_off_target_mean_h\",\"total_high_claim_mean_h\",\"att_post_right_mean_h\",\"penalty_faced_mean_h\",\"att_cmiss_left_mean_h\",\"att_hd_total_mean_h\",\"keeper_throws_mean_h\",\"total_fwd_zone_pass_div_long_pass_own_to_opp_success_mean_h\",\"accurate_freekick_cross_mean_h\",\"aerial_lost_mean_h\",\"successful_final_third_passes_mean_h\",\"att_sv_low_right_mean_h\",\"goal_assist_intentional_mean_h\",\"ATT_mean_h\",\"second_yellow_card_mean_h\",\"total_tackle_mean_h\",\"effective_head_clearance_mean_h\",\"att_fastbreak_mean_h\",\"successful_put_through_mean_h\",\"total_back_zone_pass_mean_h\",\"blocked_pass_mean_h\",\"ball_recovery_mean_h\",\"att_corner_ratio_mean_h\",\"att_freekick_target_mean_h\",\"att_ibox_miss_mean_h\",\"second_yellow_mean_h\",\"MED_mean_h\",\"offtarget_att_assist_mean_h\",\"att_pen_target_mean_h\",\"successful_open_play_pass_mean_h\",\"att_setpiece_mean_h\",\"att_ibox_blocked_mean_h\",\"att_sv_low_left_mean_h\",\"total_fastbreak_mean_h\",\"percentage_goal_shot_mean_h\",\"aerial_won_mean_h\",\"goals_conceded_mean_h\",\"att_miss_left_mean_h\",\"att_cmiss_high_left_mean_h\",\"clearance_off_line_mean_h\",\"accurate_keeper_sweeper_mean_h\",\"pts_dropped_winning_pos_mean_h\",\"fouled_final_third_mean_h\",\"att_lg_left_mean_h\",\"defender_goals_mean_h\",\"poss_won_def_3rd_mean_h\",\"att_corner_mean_h\",\"clean_sheet_mean_h\",\"att_sv_high_centre_mean_h\",\"hand_ball_mean_h\",\"att_assist_setplay_mean_h\",\"att_post_left_mean_h\",\"att_hd_post_mean_h\",\"total_pass_mean_h\",\"accurate_chipped_pass_mean_h\",\"poss_won_mid_3rd_mean_h\",\"acc_corners_intobox_pct_std_h\",\"att_pen_miss_std_h\",\"att_rf_goal_std_h\",\"att_goal_high_centre_std_h\",\"att_sv_high_left_std_h\",\"att_obox_blocked_std_h\",\"att_goal_high_left_std_h\",\"shots_mul_goals_std_h\",\"first_yellow_card_1t_std_h\",\"goals_std_h\",\"att_post_high_std_h\",\"att_hd_target_std_h\",\"shots_div_passes_right_std_h\",\"att_freekick_miss_std_h\",\"corners_ratio_std_h\",\"att_freekick_goal_std_h\",\"att_miss_high_std_h\",\"fifty_fifty_std_h\",\"put_through_std_h\",\"poss_lost_ctrl_std_h\",\"att_pen_post_std_h\",\"leftside_pass_std_h\",\"total_corners_intobox_std_h\",\"final_third_entries_std_h\",\"total_through_ball_std_h\",\"total_offside_std_h\",\"total_layoffs_std_h\",\"crosses_18yardplus_std_h\",\"winner_std_h\",\"att_obox_own_goal_std_h\",\"accurate_long_balls_std_h\",\"goal_assist_openplay_std_h\",\"passes_right_div_blocked_pass_std_h\",\"saved_ibox_std_h\",\"red_card_2t_std_h\",\"goals_1t_std_h\",\"no_foot_goals_ratio_std_h\",\"att_obxd_left_std_h\",\"att_obp_goal_std_h\",\"total_cross_nocorner_std_h\",\"touches_std_h\",\"foul_throw_in_std_h\",\"corner_taken_std_h\",\"duel_won_pct_std_h\",\"error_lead_to_shot_std_h\",\"goals_2t_pct_std_h\",\"goal_assist_setplay_std_h\",\"first_yellow_card_2t_std_h\",\"att_goal_low_left_std_h\",\"total_yel_card_std_h\",\"accurate_through_ball_std_h\",\"post_scoring_att_std_h\",\"ontarget_att_assist_std_h\",\"open_play_pass_std_h\",\"accurate_throws_std_h\",\"right_to_left_goals_std_h\",\"total_launches_std_h\",\"att_miss_right_std_h\",\"attempted_tackle_foul_std_h\",\"total_final_third_passes_std_h\",\"head_clearance_std_h\",\"att_sv_low_centre_std_h\",\"poss_won_att_3rd_std_h\",\"att_bx_centre_std_h\",\"lineup_std_h\",\"att_one_on_one_std_h\",\"total_chipped_pass_std_h\",\"attempts_conceded_obox_std_h\",\"big_chance_created_std_h\",\"accurate_corners_intobox_std_h\",\"ontarget_scoring_att_std_h\",\"diving_save_std_h\",\"saves_std_h\",\"high_to_low_goals_std_h\",\"accurate_layoffs_std_h\",\"goals_2t_std_h\",\"total_keeper_sweeper_std_h\",\"ranking_std_h\",\"first_yellow_card_std_h\",\"att_obx_right_std_h\",\"att_obox_miss_std_h\",\"bc_miss_div_scored_std_h\",\"six_yard_block_std_h\",\"att_rf_target_std_h\",\"bc_scored_div_created_std_h\",\"att_rf_total_std_h\",\"outfielder_block_std_h\",\"saved_obox_std_h\",\"won_corners_ratio_std_h\",\"rightside_pass_div_leftside_pass_std_h\",\"good_high_claim_std_h\",\"accurate_back_zone_pass_div_backward_pass_std_h\",\"attempts_ibox_std_h\",\"att_hd_goal_std_h\",\"rightside_pass_div_accurate_pass_std_h\",\"long_passes_std_h\",\"second_yellow_card_2t_std_h\",\"challenge_lost_std_h\",\"att_ibox_goal_std_h\",\"points_std_h\",\"total_contest_std_h\",\"bc_miss_div_created_std_h\",\"goal_assist_deadball_std_h\",\"att_obxd_right_std_h\",\"fwd_pass_std_h\",\"att_lg_centre_std_h\",\"last_man_tackle_std_h\",\"goal_kicks_per_shot_std_h\",\"fwd_pass_div_long_pass_own_to_opp_success_std_h\",\"att_goal_low_centre_std_h\",\"att_openplay_std_h\",\"pen_area_entries_std_h\",\"effective_clearance_std_h\",\"long_pass_own_to_opp_std_h\",\"unsuccessful_touch_std_h\",\"accurate_flick_on_std_h\",\"att_sv_high_right_std_h\",\"overrun_std_h\",\"touches_in_opp_box_std_h\",\"total_pull_back_std_h\",\"att_cmiss_right_std_h\",\"goal_fastbreak_std_h\",\"blocked_cross_std_h\",\"big_chance_scored_std_h\",\"attempts_obox_std_h\",\"lost_corners_std_h\",\"att_goal_low_right_std_h\",\"passes_right_std_h\",\"passes_left_div_blocked_pass_std_h\",\"att_cmiss_high_std_h\",\"long_pass_own_to_opp_success_std_h\",\"accurate_cross_nocorner_std_h\",\"total_red_card_std_h\",\"fk_foul_lost_std_h\",\"accurate_fwd_zone_pass_std_h\",\"second_yellow_card_1t_std_h\",\"failed_to_block_std_h\",\"accurate_pass_std_h\",\"leftside_pass_div_accurate_pass_std_h\",\"red_card_1t_std_h\",\"dispossessed_std_h\",\"pts_gained_losing_pos_std_h\",\"rightside_pass_std_h\",\"shield_ball_oop_std_h\",\"midfielder_goals_std_h\",\"interceptions_in_box_std_h\",\"att_lg_right_std_h\",\"won_corners_std_h\",\"att_pen_goal_std_h\",\"forward_goals_std_h\",\"accurate_cross_std_h\",\"acc_cross_nocorner_pct_std_h\",\"att_miss_high_left_std_h\",\"total_att_assist_std_h\",\"att_lf_target_std_h\",\"att_cmiss_high_right_std_h\",\"punches_std_h\",\"contentious_decision_std_h\",\"att_freekick_post_std_h\",\"interception_std_h\",\"total_throws_std_h\",\"crosses_18yard_std_h\",\"accurate_pull_back_std_h\",\"att_obox_goal_std_h\",\"fk_foul_won_std_h\",\"accurate_launches_std_h\",\"goal_assist_std_h\",\"att_lf_total_std_h\",\"rescinded_red_card_std_h\",\"backward_pass_std_h\",\"accurate_keeper_throws_std_h\",\"att_ibox_own_goal_std_h\",\"accurate_back_zone_pass_std_h\",\"hit_woodwork_std_h\",\"att_freekick_total_std_h\",\"accurate_goal_kicks_std_h\",\"won_tackle_std_h\",\"corners_std_h\",\"own_goals_std_h\",\"blocked_scoring_att_std_h\",\"poss_lost_all_std_h\",\"att_hd_miss_std_h\",\"shots_std_h\",\"att_ibox_target_std_h\",\"duel_won_std_h\",\"crosses_std_h\",\"att_ibox_post_std_h\",\"total_clearance_std_h\",\"attempts_conceded_ibox_std_h\",\"big_chance_missed_std_h\",\"goal_assist_intent_norm_std_h\",\"att_miss_high_right_std_h\",\"red_card_std_h\",\"passes_left_std_h\",\"left_div_right_foot_goals_std_h\",\"successful_fifty_fifty_std_h\",\"penalty_save_std_h\",\"att_obox_post_std_h\",\"att_lf_goal_std_h\",\"penalty_conceded_std_h\",\"freekick_cross_std_h\",\"error_lead_to_goal_std_h\",\"duel_lost_std_h\",\"att_bx_left_std_h\",\"goals_openplay_std_h\",\"shots_div_passes_left_std_h\",\"att_goal_high_right_std_h\",\"total_fwd_zone_pass_std_h\",\"corners_2t_std_h\",\"goal_kicks_std_h\",\"penalty_won_std_h\",\"goal_kicks_div_long_passes_std_h\",\"goals_conceded_obox_std_h\",\"DEF_std_h\",\"att_obx_centre_std_h\",\"att_assist_openplay_std_h\",\"total_flick_on_std_h\",\"att_bx_right_std_h\",\"subs_made_std_h\",\"goals_conceded_ibox_std_h\",\"att_obx_left_std_h\",\"pen_goals_conceded_std_h\",\"won_contest_std_h\",\"att_obox_target_std_h\",\"shot_off_target_std_h\",\"total_high_claim_std_h\",\"att_post_right_std_h\",\"penalty_faced_std_h\",\"att_cmiss_left_std_h\",\"att_hd_total_std_h\",\"keeper_throws_std_h\",\"total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_h\",\"accurate_freekick_cross_std_h\",\"aerial_lost_std_h\",\"successful_final_third_passes_std_h\",\"att_sv_low_right_std_h\",\"goal_assist_intentional_std_h\",\"ATT_std_h\",\"second_yellow_card_std_h\",\"total_tackle_std_h\",\"effective_head_clearance_std_h\",\"att_fastbreak_std_h\",\"successful_put_through_std_h\",\"total_back_zone_pass_std_h\",\"blocked_pass_std_h\",\"ball_recovery_std_h\",\"att_corner_ratio_std_h\",\"att_freekick_target_std_h\",\"att_ibox_miss_std_h\",\"second_yellow_std_h\",\"MED_std_h\",\"offtarget_att_assist_std_h\",\"att_pen_target_std_h\",\"successful_open_play_pass_std_h\",\"att_setpiece_std_h\",\"att_ibox_blocked_std_h\",\"att_sv_low_left_std_h\",\"total_fastbreak_std_h\",\"percentage_goal_shot_std_h\",\"aerial_won_std_h\",\"goals_conceded_std_h\",\"att_miss_left_std_h\",\"att_cmiss_high_left_std_h\",\"clearance_off_line_std_h\",\"accurate_keeper_sweeper_std_h\",\"pts_dropped_winning_pos_std_h\",\"fouled_final_third_std_h\",\"att_lg_left_std_h\",\"defender_goals_std_h\",\"poss_won_def_3rd_std_h\",\"att_corner_std_h\",\"clean_sheet_std_h\",\"att_sv_high_centre_std_h\",\"hand_ball_std_h\",\"att_assist_setplay_std_h\",\"att_post_left_std_h\",\"att_hd_post_std_h\",\"total_pass_std_h\",\"accurate_chipped_pass_std_h\",\"poss_won_mid_3rd_std_h\",\"win_pct_home_over_both_score_h\",\"odds_home_over_both_score_h\",\"win_pct_away_over_both_score_h\",\"odds_away_over_both_score_h\",\"total_bets_over_both_score_h\",\"total_win_pct_over_both_score_h\",\"odds_over_both_score_h\",\"imp_prob_over_both_score_h\",\"ratio_over_both_score_h\",\"win_pct_home_under_both_score_h\",\"odds_home_under_both_score_h\",\"win_pct_away_under_both_score_h\",\"odds_away_under_both_score_h\",\"total_bets_under_both_score_h\",\"total_win_pct_under_both_score_h\",\"odds_under_both_score_h\",\"imp_prob_under_both_score_h\",\"ratio_under_both_score_h\",\"win_pct_home_over_goals_1.5_h\",\"odds_home_over_goals_1.5_h\",\"win_pct_away_over_goals_1.5_h\",\"odds_away_over_goals_1.5_h\",\"total_bets_over_goals_1.5_h\",\"total_win_pct_over_goals_1.5_h\",\"odds_over_goals_1.5_h\",\"imp_prob_over_goals_1.5_h\",\"ratio_over_goals_1.5_h\",\"win_pct_home_under_goals_1.5_h\",\"odds_home_under_goals_1.5_h\",\"win_pct_away_under_goals_1.5_h\",\"odds_away_under_goals_1.5_h\",\"total_bets_under_goals_1.5_h\",\"total_win_pct_under_goals_1.5_h\",\"odds_under_goals_1.5_h\",\"imp_prob_under_goals_1.5_h\",\"ratio_under_goals_1.5_h\",\"win_pct_home_over_goals_2.5_h\",\"odds_home_over_goals_2.5_h\",\"win_pct_away_over_goals_2.5_h\",\"odds_away_over_goals_2.5_h\",\"total_bets_over_goals_2.5_h\",\"total_win_pct_over_goals_2.5_h\",\"odds_over_goals_2.5_h\",\"imp_prob_over_goals_2.5_h\",\"ratio_over_goals_2.5_h\",\"win_pct_home_under_goals_2.5_h\",\"odds_home_under_goals_2.5_h\",\"win_pct_away_under_goals_2.5_h\",\"odds_away_under_goals_2.5_h\",\"total_bets_under_goals_2.5_h\",\"total_win_pct_under_goals_2.5_h\",\"odds_under_goals_2.5_h\",\"imp_prob_under_goals_2.5_h\",\"ratio_under_goals_2.5_h\",\"win_pct_home_over_goals_3.5_h\",\"odds_home_over_goals_3.5_h\",\"win_pct_away_over_goals_3.5_h\",\"odds_away_over_goals_3.5_h\",\"total_bets_over_goals_3.5_h\",\"total_win_pct_over_goals_3.5_h\",\"odds_over_goals_3.5_h\",\"imp_prob_over_goals_3.5_h\",\"ratio_over_goals_3.5_h\",\"win_pct_home_under_goals_3.5_h\",\"odds_home_under_goals_3.5_h\",\"win_pct_away_under_goals_3.5_h\",\"odds_away_under_goals_3.5_h\",\"total_bets_under_goals_3.5_h\",\"total_win_pct_under_goals_3.5_h\",\"odds_under_goals_3.5_h\",\"imp_prob_under_goals_3.5_h\",\"ratio_under_goals_3.5_h\",\"win_pct_home_over_goals_4.5_h\",\"odds_home_over_goals_4.5_h\",\"win_pct_away_over_goals_4.5_h\",\"odds_away_over_goals_4.5_h\",\"total_bets_over_goals_4.5_h\",\"total_win_pct_over_goals_4.5_h\",\"odds_over_goals_4.5_h\",\"imp_prob_over_goals_4.5_h\",\"ratio_over_goals_4.5_h\",\"win_pct_home_under_goals_4.5_h\",\"odds_home_under_goals_4.5_h\",\"win_pct_away_under_goals_4.5_h\",\"odds_away_under_goals_4.5_h\",\"total_bets_under_goals_4.5_h\",\"total_win_pct_under_goals_4.5_h\",\"odds_under_goals_4.5_h\",\"imp_prob_under_goals_4.5_h\",\"ratio_under_goals_4.5_h\",\"win_pct_a\",\"draw_pct_a\",\"loss_pct_a\",\"win_streak_a\",\"loss_streak_a\",\"draw_streak_a\",\"points_a\",\"ranking_a\",\"ATT_a\",\"DEF_a\",\"MED_a\",\"acc_corners_intobox_pct_mean_a\",\"att_pen_miss_mean_a\",\"att_rf_goal_mean_a\",\"att_goal_high_centre_mean_a\",\"att_sv_high_left_mean_a\",\"att_obox_blocked_mean_a\",\"att_goal_high_left_mean_a\",\"shots_mul_goals_mean_a\",\"first_yellow_card_1t_mean_a\",\"goals_mean_a\",\"att_post_high_mean_a\",\"att_hd_target_mean_a\",\"shots_div_passes_right_mean_a\",\"att_freekick_miss_mean_a\",\"corners_ratio_mean_a\",\"att_freekick_goal_mean_a\",\"att_miss_high_mean_a\",\"fifty_fifty_mean_a\",\"put_through_mean_a\",\"poss_lost_ctrl_mean_a\",\"att_pen_post_mean_a\",\"leftside_pass_mean_a\",\"total_corners_intobox_mean_a\",\"final_third_entries_mean_a\",\"total_through_ball_mean_a\",\"total_offside_mean_a\",\"total_layoffs_mean_a\",\"crosses_18yardplus_mean_a\",\"winner_mean_a\",\"att_obox_own_goal_mean_a\",\"accurate_long_balls_mean_a\",\"goal_assist_openplay_mean_a\",\"passes_right_div_blocked_pass_mean_a\",\"saved_ibox_mean_a\",\"red_card_2t_mean_a\",\"goals_1t_mean_a\",\"no_foot_goals_ratio_mean_a\",\"att_obxd_left_mean_a\",\"att_obp_goal_mean_a\",\"total_cross_nocorner_mean_a\",\"touches_mean_a\",\"foul_throw_in_mean_a\",\"corner_taken_mean_a\",\"duel_won_pct_mean_a\",\"error_lead_to_shot_mean_a\",\"goals_2t_pct_mean_a\",\"goal_assist_setplay_mean_a\",\"first_yellow_card_2t_mean_a\",\"att_goal_low_left_mean_a\",\"total_yel_card_mean_a\",\"accurate_through_ball_mean_a\",\"post_scoring_att_mean_a\",\"ontarget_att_assist_mean_a\",\"open_play_pass_mean_a\",\"accurate_throws_mean_a\",\"right_to_left_goals_mean_a\",\"total_launches_mean_a\",\"att_miss_right_mean_a\",\"attempted_tackle_foul_mean_a\",\"total_final_third_passes_mean_a\",\"head_clearance_mean_a\",\"att_sv_low_centre_mean_a\",\"poss_won_att_3rd_mean_a\",\"att_bx_centre_mean_a\",\"lineup_mean_a\",\"att_one_on_one_mean_a\",\"total_chipped_pass_mean_a\",\"attempts_conceded_obox_mean_a\",\"big_chance_created_mean_a\",\"accurate_corners_intobox_mean_a\",\"ontarget_scoring_att_mean_a\",\"diving_save_mean_a\",\"saves_mean_a\",\"high_to_low_goals_mean_a\",\"accurate_layoffs_mean_a\",\"goals_2t_mean_a\",\"total_keeper_sweeper_mean_a\",\"ranking_mean_a\",\"first_yellow_card_mean_a\",\"att_obx_right_mean_a\",\"att_obox_miss_mean_a\",\"bc_miss_div_scored_mean_a\",\"six_yard_block_mean_a\",\"att_rf_target_mean_a\",\"bc_scored_div_created_mean_a\",\"att_rf_total_mean_a\",\"outfielder_block_mean_a\",\"saved_obox_mean_a\",\"won_corners_ratio_mean_a\",\"rightside_pass_div_leftside_pass_mean_a\",\"good_high_claim_mean_a\",\"accurate_back_zone_pass_div_backward_pass_mean_a\",\"attempts_ibox_mean_a\",\"att_hd_goal_mean_a\",\"rightside_pass_div_accurate_pass_mean_a\",\"long_passes_mean_a\",\"second_yellow_card_2t_mean_a\",\"challenge_lost_mean_a\",\"att_ibox_goal_mean_a\",\"points_mean_a\",\"total_contest_mean_a\",\"bc_miss_div_created_mean_a\",\"goal_assist_deadball_mean_a\",\"att_obxd_right_mean_a\",\"fwd_pass_mean_a\",\"att_lg_centre_mean_a\",\"last_man_tackle_mean_a\",\"goal_kicks_per_shot_mean_a\",\"fwd_pass_div_long_pass_own_to_opp_success_mean_a\",\"att_goal_low_centre_mean_a\",\"att_openplay_mean_a\",\"pen_area_entries_mean_a\",\"effective_clearance_mean_a\",\"long_pass_own_to_opp_mean_a\",\"unsuccessful_touch_mean_a\",\"accurate_flick_on_mean_a\",\"att_sv_high_right_mean_a\",\"overrun_mean_a\",\"touches_in_opp_box_mean_a\",\"total_pull_back_mean_a\",\"att_cmiss_right_mean_a\",\"goal_fastbreak_mean_a\",\"blocked_cross_mean_a\",\"big_chance_scored_mean_a\",\"attempts_obox_mean_a\",\"lost_corners_mean_a\",\"att_goal_low_right_mean_a\",\"passes_right_mean_a\",\"passes_left_div_blocked_pass_mean_a\",\"att_cmiss_high_mean_a\",\"long_pass_own_to_opp_success_mean_a\",\"accurate_cross_nocorner_mean_a\",\"total_red_card_mean_a\",\"fk_foul_lost_mean_a\",\"accurate_fwd_zone_pass_mean_a\",\"second_yellow_card_1t_mean_a\",\"failed_to_block_mean_a\",\"accurate_pass_mean_a\",\"leftside_pass_div_accurate_pass_mean_a\",\"red_card_1t_mean_a\",\"dispossessed_mean_a\",\"pts_gained_losing_pos_mean_a\",\"rightside_pass_mean_a\",\"shield_ball_oop_mean_a\",\"midfielder_goals_mean_a\",\"interceptions_in_box_mean_a\",\"att_lg_right_mean_a\",\"won_corners_mean_a\",\"att_pen_goal_mean_a\",\"forward_goals_mean_a\",\"accurate_cross_mean_a\",\"acc_cross_nocorner_pct_mean_a\",\"att_miss_high_left_mean_a\",\"total_att_assist_mean_a\",\"att_lf_target_mean_a\",\"att_cmiss_high_right_mean_a\",\"punches_mean_a\",\"contentious_decision_mean_a\",\"att_freekick_post_mean_a\",\"interception_mean_a\",\"total_throws_mean_a\",\"crosses_18yard_mean_a\",\"accurate_pull_back_mean_a\",\"att_obox_goal_mean_a\",\"fk_foul_won_mean_a\",\"accurate_launches_mean_a\",\"goal_assist_mean_a\",\"att_lf_total_mean_a\",\"rescinded_red_card_mean_a\",\"backward_pass_mean_a\",\"accurate_keeper_throws_mean_a\",\"att_ibox_own_goal_mean_a\",\"accurate_back_zone_pass_mean_a\",\"hit_woodwork_mean_a\",\"att_freekick_total_mean_a\",\"accurate_goal_kicks_mean_a\",\"won_tackle_mean_a\",\"corners_mean_a\",\"own_goals_mean_a\",\"blocked_scoring_att_mean_a\",\"poss_lost_all_mean_a\",\"att_hd_miss_mean_a\",\"shots_mean_a\",\"att_ibox_target_mean_a\",\"duel_won_mean_a\",\"crosses_mean_a\",\"att_ibox_post_mean_a\",\"total_clearance_mean_a\",\"attempts_conceded_ibox_mean_a\",\"big_chance_missed_mean_a\",\"goal_assist_intent_norm_mean_a\",\"att_miss_high_right_mean_a\",\"red_card_mean_a\",\"passes_left_mean_a\",\"left_div_right_foot_goals_mean_a\",\"successful_fifty_fifty_mean_a\",\"penalty_save_mean_a\",\"att_obox_post_mean_a\",\"att_lf_goal_mean_a\",\"penalty_conceded_mean_a\",\"freekick_cross_mean_a\",\"error_lead_to_goal_mean_a\",\"duel_lost_mean_a\",\"att_bx_left_mean_a\",\"goals_openplay_mean_a\",\"shots_div_passes_left_mean_a\",\"att_goal_high_right_mean_a\",\"total_fwd_zone_pass_mean_a\",\"corners_2t_mean_a\",\"goal_kicks_mean_a\",\"penalty_won_mean_a\",\"goal_kicks_div_long_passes_mean_a\",\"goals_conceded_obox_mean_a\",\"DEF_mean_a\",\"att_obx_centre_mean_a\",\"att_assist_openplay_mean_a\",\"total_flick_on_mean_a\",\"att_bx_right_mean_a\",\"subs_made_mean_a\",\"goals_conceded_ibox_mean_a\",\"att_obx_left_mean_a\",\"pen_goals_conceded_mean_a\",\"won_contest_mean_a\",\"att_obox_target_mean_a\",\"shot_off_target_mean_a\",\"total_high_claim_mean_a\",\"att_post_right_mean_a\",\"penalty_faced_mean_a\",\"att_cmiss_left_mean_a\",\"att_hd_total_mean_a\",\"keeper_throws_mean_a\",\"total_fwd_zone_pass_div_long_pass_own_to_opp_success_mean_a\",\"accurate_freekick_cross_mean_a\",\"aerial_lost_mean_a\",\"successful_final_third_passes_mean_a\",\"att_sv_low_right_mean_a\",\"goal_assist_intentional_mean_a\",\"ATT_mean_a\",\"second_yellow_card_mean_a\",\"total_tackle_mean_a\",\"effective_head_clearance_mean_a\",\"att_fastbreak_mean_a\",\"successful_put_through_mean_a\",\"total_back_zone_pass_mean_a\",\"blocked_pass_mean_a\",\"ball_recovery_mean_a\",\"att_corner_ratio_mean_a\",\"att_freekick_target_mean_a\",\"att_ibox_miss_mean_a\",\"second_yellow_mean_a\",\"MED_mean_a\",\"offtarget_att_assist_mean_a\",\"att_pen_target_mean_a\",\"successful_open_play_pass_mean_a\",\"att_setpiece_mean_a\",\"att_ibox_blocked_mean_a\",\"att_sv_low_left_mean_a\",\"total_fastbreak_mean_a\",\"percentage_goal_shot_mean_a\",\"aerial_won_mean_a\",\"goals_conceded_mean_a\",\"att_miss_left_mean_a\",\"att_cmiss_high_left_mean_a\",\"clearance_off_line_mean_a\",\"accurate_keeper_sweeper_mean_a\",\"pts_dropped_winning_pos_mean_a\",\"fouled_final_third_mean_a\",\"att_lg_left_mean_a\",\"defender_goals_mean_a\",\"poss_won_def_3rd_mean_a\",\"att_corner_mean_a\",\"clean_sheet_mean_a\",\"att_sv_high_centre_mean_a\",\"hand_ball_mean_a\",\"att_assist_setplay_mean_a\",\"att_post_left_mean_a\",\"att_hd_post_mean_a\",\"total_pass_mean_a\",\"accurate_chipped_pass_mean_a\",\"poss_won_mid_3rd_mean_a\",\"acc_corners_intobox_pct_std_a\",\"att_pen_miss_std_a\",\"att_rf_goal_std_a\",\"att_goal_high_centre_std_a\",\"att_sv_high_left_std_a\",\"att_obox_blocked_std_a\",\"att_goal_high_left_std_a\",\"shots_mul_goals_std_a\",\"first_yellow_card_1t_std_a\",\"goals_std_a\",\"att_post_high_std_a\",\"att_hd_target_std_a\",\"shots_div_passes_right_std_a\",\"att_freekick_miss_std_a\",\"corners_ratio_std_a\",\"att_freekick_goal_std_a\",\"att_miss_high_std_a\",\"fifty_fifty_std_a\",\"put_through_std_a\",\"poss_lost_ctrl_std_a\",\"att_pen_post_std_a\",\"leftside_pass_std_a\",\"total_corners_intobox_std_a\",\"final_third_entries_std_a\",\"total_through_ball_std_a\",\"total_offside_std_a\",\"total_layoffs_std_a\",\"crosses_18yardplus_std_a\",\"winner_std_a\",\"att_obox_own_goal_std_a\",\"accurate_long_balls_std_a\",\"goal_assist_openplay_std_a\",\"passes_right_div_blocked_pass_std_a\",\"saved_ibox_std_a\",\"red_card_2t_std_a\",\"goals_1t_std_a\",\"no_foot_goals_ratio_std_a\",\"att_obxd_left_std_a\",\"att_obp_goal_std_a\",\"total_cross_nocorner_std_a\",\"touches_std_a\",\"foul_throw_in_std_a\",\"corner_taken_std_a\",\"duel_won_pct_std_a\",\"error_lead_to_shot_std_a\",\"goals_2t_pct_std_a\",\"goal_assist_setplay_std_a\",\"first_yellow_card_2t_std_a\",\"att_goal_low_left_std_a\",\"total_yel_card_std_a\",\"accurate_through_ball_std_a\",\"post_scoring_att_std_a\",\"ontarget_att_assist_std_a\",\"open_play_pass_std_a\",\"accurate_throws_std_a\",\"right_to_left_goals_std_a\",\"total_launches_std_a\",\"att_miss_right_std_a\",\"attempted_tackle_foul_std_a\",\"total_final_third_passes_std_a\",\"head_clearance_std_a\",\"att_sv_low_centre_std_a\",\"poss_won_att_3rd_std_a\",\"att_bx_centre_std_a\",\"lineup_std_a\",\"att_one_on_one_std_a\",\"total_chipped_pass_std_a\",\"attempts_conceded_obox_std_a\",\"big_chance_created_std_a\",\"accurate_corners_intobox_std_a\",\"ontarget_scoring_att_std_a\",\"diving_save_std_a\",\"saves_std_a\",\"high_to_low_goals_std_a\",\"accurate_layoffs_std_a\",\"goals_2t_std_a\",\"total_keeper_sweeper_std_a\",\"ranking_std_a\",\"first_yellow_card_std_a\",\"att_obx_right_std_a\",\"att_obox_miss_std_a\",\"bc_miss_div_scored_std_a\",\"six_yard_block_std_a\",\"att_rf_target_std_a\",\"bc_scored_div_created_std_a\",\"att_rf_total_std_a\",\"outfielder_block_std_a\",\"saved_obox_std_a\",\"won_corners_ratio_std_a\",\"rightside_pass_div_leftside_pass_std_a\",\"good_high_claim_std_a\",\"accurate_back_zone_pass_div_backward_pass_std_a\",\"attempts_ibox_std_a\",\"att_hd_goal_std_a\",\"rightside_pass_div_accurate_pass_std_a\",\"long_passes_std_a\",\"second_yellow_card_2t_std_a\",\"challenge_lost_std_a\",\"att_ibox_goal_std_a\",\"points_std_a\",\"total_contest_std_a\",\"bc_miss_div_created_std_a\",\"goal_assist_deadball_std_a\",\"att_obxd_right_std_a\",\"fwd_pass_std_a\",\"att_lg_centre_std_a\",\"last_man_tackle_std_a\",\"goal_kicks_per_shot_std_a\",\"fwd_pass_div_long_pass_own_to_opp_success_std_a\",\"att_goal_low_centre_std_a\",\"att_openplay_std_a\",\"pen_area_entries_std_a\",\"effective_clearance_std_a\",\"long_pass_own_to_opp_std_a\",\"unsuccessful_touch_std_a\",\"accurate_flick_on_std_a\",\"att_sv_high_right_std_a\",\"overrun_std_a\",\"touches_in_opp_box_std_a\",\"total_pull_back_std_a\",\"att_cmiss_right_std_a\",\"goal_fastbreak_std_a\",\"blocked_cross_std_a\",\"big_chance_scored_std_a\",\"attempts_obox_std_a\",\"lost_corners_std_a\",\"att_goal_low_right_std_a\",\"passes_right_std_a\",\"passes_left_div_blocked_pass_std_a\",\"att_cmiss_high_std_a\",\"long_pass_own_to_opp_success_std_a\",\"accurate_cross_nocorner_std_a\",\"total_red_card_std_a\",\"fk_foul_lost_std_a\",\"accurate_fwd_zone_pass_std_a\",\"second_yellow_card_1t_std_a\",\"failed_to_block_std_a\",\"accurate_pass_std_a\",\"leftside_pass_div_accurate_pass_std_a\",\"red_card_1t_std_a\",\"dispossessed_std_a\",\"pts_gained_losing_pos_std_a\",\"rightside_pass_std_a\",\"shield_ball_oop_std_a\",\"midfielder_goals_std_a\",\"interceptions_in_box_std_a\",\"att_lg_right_std_a\",\"won_corners_std_a\",\"att_pen_goal_std_a\",\"forward_goals_std_a\",\"accurate_cross_std_a\",\"acc_cross_nocorner_pct_std_a\",\"att_miss_high_left_std_a\",\"total_att_assist_std_a\",\"att_lf_target_std_a\",\"att_cmiss_high_right_std_a\",\"punches_std_a\",\"contentious_decision_std_a\",\"att_freekick_post_std_a\",\"interception_std_a\",\"total_throws_std_a\",\"crosses_18yard_std_a\",\"accurate_pull_back_std_a\",\"att_obox_goal_std_a\",\"fk_foul_won_std_a\",\"accurate_launches_std_a\",\"goal_assist_std_a\",\"att_lf_total_std_a\",\"rescinded_red_card_std_a\",\"backward_pass_std_a\",\"accurate_keeper_throws_std_a\",\"att_ibox_own_goal_std_a\",\"accurate_back_zone_pass_std_a\",\"hit_woodwork_std_a\",\"att_freekick_total_std_a\",\"accurate_goal_kicks_std_a\",\"won_tackle_std_a\",\"corners_std_a\",\"own_goals_std_a\",\"blocked_scoring_att_std_a\",\"poss_lost_all_std_a\",\"att_hd_miss_std_a\",\"shots_std_a\",\"att_ibox_target_std_a\",\"duel_won_std_a\",\"crosses_std_a\",\"att_ibox_post_std_a\",\"total_clearance_std_a\",\"attempts_conceded_ibox_std_a\",\"big_chance_missed_std_a\",\"goal_assist_intent_norm_std_a\",\"att_miss_high_right_std_a\",\"red_card_std_a\",\"passes_left_std_a\",\"left_div_right_foot_goals_std_a\",\"successful_fifty_fifty_std_a\",\"penalty_save_std_a\",\"att_obox_post_std_a\",\"att_lf_goal_std_a\",\"penalty_conceded_std_a\",\"freekick_cross_std_a\",\"error_lead_to_goal_std_a\",\"duel_lost_std_a\",\"att_bx_left_std_a\",\"goals_openplay_std_a\",\"shots_div_passes_left_std_a\",\"att_goal_high_right_std_a\",\"total_fwd_zone_pass_std_a\",\"corners_2t_std_a\",\"goal_kicks_std_a\",\"penalty_won_std_a\",\"goal_kicks_div_long_passes_std_a\",\"goals_conceded_obox_std_a\",\"DEF_std_a\",\"att_obx_centre_std_a\",\"att_assist_openplay_std_a\",\"total_flick_on_std_a\",\"att_bx_right_std_a\",\"subs_made_std_a\",\"goals_conceded_ibox_std_a\",\"att_obx_left_std_a\",\"pen_goals_conceded_std_a\",\"won_contest_std_a\",\"att_obox_target_std_a\",\"shot_off_target_std_a\",\"total_high_claim_std_a\",\"att_post_right_std_a\",\"penalty_faced_std_a\",\"att_cmiss_left_std_a\",\"att_hd_total_std_a\",\"keeper_throws_std_a\",\"total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_a\",\"accurate_freekick_cross_std_a\",\"aerial_lost_std_a\",\"successful_final_third_passes_std_a\",\"att_sv_low_right_std_a\",\"goal_assist_intentional_std_a\",\"ATT_std_a\",\"second_yellow_card_std_a\",\"total_tackle_std_a\",\"effective_head_clearance_std_a\",\"att_fastbreak_std_a\",\"successful_put_through_std_a\",\"total_back_zone_pass_std_a\",\"blocked_pass_std_a\",\"ball_recovery_std_a\",\"att_corner_ratio_std_a\",\"att_freekick_target_std_a\",\"att_ibox_miss_std_a\",\"second_yellow_std_a\",\"MED_std_a\",\"offtarget_att_assist_std_a\",\"att_pen_target_std_a\",\"successful_open_play_pass_std_a\",\"att_setpiece_std_a\",\"att_ibox_blocked_std_a\",\"att_sv_low_left_std_a\",\"total_fastbreak_std_a\",\"percentage_goal_shot_std_a\",\"aerial_won_std_a\",\"goals_conceded_std_a\",\"att_miss_left_std_a\",\"att_cmiss_high_left_std_a\",\"clearance_off_line_std_a\",\"accurate_keeper_sweeper_std_a\",\"pts_dropped_winning_pos_std_a\",\"fouled_final_third_std_a\",\"att_lg_left_std_a\",\"defender_goals_std_a\",\"poss_won_def_3rd_std_a\",\"att_corner_std_a\",\"clean_sheet_std_a\",\"att_sv_high_centre_std_a\",\"hand_ball_std_a\",\"att_assist_setplay_std_a\",\"att_post_left_std_a\",\"att_hd_post_std_a\",\"total_pass_std_a\",\"accurate_chipped_pass_std_a\",\"poss_won_mid_3rd_std_a\",\"win_pct_home_over_both_score_a\",\"odds_home_over_both_score_a\",\"win_pct_away_over_both_score_a\",\"odds_away_over_both_score_a\",\"total_bets_over_both_score_a\",\"total_win_pct_over_both_score_a\",\"odds_over_both_score_a\",\"imp_prob_over_both_score_a\",\"ratio_over_both_score_a\",\"win_pct_home_under_both_score_a\",\"odds_home_under_both_score_a\",\"win_pct_away_under_both_score_a\",\"odds_away_under_both_score_a\",\"total_bets_under_both_score_a\",\"total_win_pct_under_both_score_a\",\"odds_under_both_score_a\",\"imp_prob_under_both_score_a\",\"ratio_under_both_score_a\",\"win_pct_home_over_goals_1.5_a\",\"odds_home_over_goals_1.5_a\",\"win_pct_away_over_goals_1.5_a\",\"odds_away_over_goals_1.5_a\",\"total_bets_over_goals_1.5_a\",\"total_win_pct_over_goals_1.5_a\",\"odds_over_goals_1.5_a\",\"imp_prob_over_goals_1.5_a\",\"ratio_over_goals_1.5_a\",\"win_pct_home_under_goals_1.5_a\",\"odds_home_under_goals_1.5_a\",\"win_pct_away_under_goals_1.5_a\",\"odds_away_under_goals_1.5_a\",\"total_bets_under_goals_1.5_a\",\"total_win_pct_under_goals_1.5_a\",\"odds_under_goals_1.5_a\",\"imp_prob_under_goals_1.5_a\",\"ratio_under_goals_1.5_a\",\"win_pct_home_over_goals_2.5_a\",\"odds_home_over_goals_2.5_a\",\"win_pct_away_over_goals_2.5_a\",\"odds_away_over_goals_2.5_a\",\"total_bets_over_goals_2.5_a\",\"total_win_pct_over_goals_2.5_a\",\"odds_over_goals_2.5_a\",\"imp_prob_over_goals_2.5_a\",\"ratio_over_goals_2.5_a\",\"win_pct_home_under_goals_2.5_a\",\"odds_home_under_goals_2.5_a\",\"win_pct_away_under_goals_2.5_a\",\"odds_away_under_goals_2.5_a\",\"total_bets_under_goals_2.5_a\",\"total_win_pct_under_goals_2.5_a\",\"odds_under_goals_2.5_a\",\"imp_prob_under_goals_2.5_a\",\"ratio_under_goals_2.5_a\",\"win_pct_home_over_goals_3.5_a\",\"odds_home_over_goals_3.5_a\",\"win_pct_away_over_goals_3.5_a\",\"odds_away_over_goals_3.5_a\",\"total_bets_over_goals_3.5_a\",\"total_win_pct_over_goals_3.5_a\",\"odds_over_goals_3.5_a\",\"imp_prob_over_goals_3.5_a\",\"ratio_over_goals_3.5_a\",\"win_pct_home_under_goals_3.5_a\",\"odds_home_under_goals_3.5_a\",\"win_pct_away_under_goals_3.5_a\",\"odds_away_under_goals_3.5_a\",\"total_bets_under_goals_3.5_a\",\"total_win_pct_under_goals_3.5_a\",\"odds_under_goals_3.5_a\",\"imp_prob_under_goals_3.5_a\",\"ratio_under_goals_3.5_a\",\"win_pct_home_over_goals_4.5_a\",\"odds_home_over_goals_4.5_a\",\"win_pct_away_over_goals_4.5_a\",\"odds_away_over_goals_4.5_a\",\"total_bets_over_goals_4.5_a\",\"total_win_pct_over_goals_4.5_a\",\"odds_over_goals_4.5_a\",\"imp_prob_over_goals_4.5_a\",\"ratio_over_goals_4.5_a\",\"win_pct_home_under_goals_4.5_a\",\"odds_home_under_goals_4.5_a\",\"win_pct_away_under_goals_4.5_a\",\"odds_away_under_goals_4.5_a\",\"total_bets_under_goals_4.5_a\",\"total_win_pct_under_goals_4.5_a\",\"odds_under_goals_4.5_a\",\"imp_prob_under_goals_4.5_a\",\"ratio_under_goals_4.5_a\",\"win_pct_diff\",\"draw_pct_diff\",\"loss_pct_diff\",\"win_streak_diff\",\"loss_streak_diff\",\"draw_streak_diff\",\"points_diff\",\"ranking_diff\",\"ATT_diff\",\"DEF_diff\",\"MED_diff\",\"acc_corners_intobox_pct_mean_diff\",\"att_pen_miss_mean_diff\",\"att_rf_goal_mean_diff\",\"att_goal_high_centre_mean_diff\",\"att_sv_high_left_mean_diff\",\"att_obox_blocked_mean_diff\",\"att_goal_high_left_mean_diff\",\"shots_mul_goals_mean_diff\",\"first_yellow_card_1t_mean_diff\",\"goals_mean_diff\",\"att_post_high_mean_diff\",\"att_hd_target_mean_diff\",\"shots_div_passes_right_mean_diff\",\"att_freekick_miss_mean_diff\",\"corners_ratio_mean_diff\",\"att_freekick_goal_mean_diff\",\"att_miss_high_mean_diff\",\"fifty_fifty_mean_diff\",\"put_through_mean_diff\",\"poss_lost_ctrl_mean_diff\",\"att_pen_post_mean_diff\",\"leftside_pass_mean_diff\",\"total_corners_intobox_mean_diff\",\"final_third_entries_mean_diff\",\"total_through_ball_mean_diff\",\"total_offside_mean_diff\",\"total_layoffs_mean_diff\",\"crosses_18yardplus_mean_diff\",\"winner_mean_diff\",\"att_obox_own_goal_mean_diff\",\"accurate_long_balls_mean_diff\",\"goal_assist_openplay_mean_diff\",\"passes_right_div_blocked_pass_mean_diff\",\"saved_ibox_mean_diff\",\"red_card_2t_mean_diff\",\"goals_1t_mean_diff\",\"no_foot_goals_ratio_mean_diff\",\"att_obxd_left_mean_diff\",\"att_obp_goal_mean_diff\",\"total_cross_nocorner_mean_diff\",\"touches_mean_diff\",\"foul_throw_in_mean_diff\",\"corner_taken_mean_diff\",\"duel_won_pct_mean_diff\",\"error_lead_to_shot_mean_diff\",\"goals_2t_pct_mean_diff\",\"goal_assist_setplay_mean_diff\",\"first_yellow_card_2t_mean_diff\",\"att_goal_low_left_mean_diff\",\"total_yel_card_mean_diff\",\"accurate_through_ball_mean_diff\",\"post_scoring_att_mean_diff\",\"ontarget_att_assist_mean_diff\",\"open_play_pass_mean_diff\",\"accurate_throws_mean_diff\",\"right_to_left_goals_mean_diff\",\"total_launches_mean_diff\",\"att_miss_right_mean_diff\",\"attempted_tackle_foul_mean_diff\",\"total_final_third_passes_mean_diff\",\"head_clearance_mean_diff\",\"att_sv_low_centre_mean_diff\",\"poss_won_att_3rd_mean_diff\",\"att_bx_centre_mean_diff\",\"lineup_mean_diff\",\"att_one_on_one_mean_diff\",\"total_chipped_pass_mean_diff\",\"attempts_conceded_obox_mean_diff\",\"big_chance_created_mean_diff\",\"accurate_corners_intobox_mean_diff\",\"ontarget_scoring_att_mean_diff\",\"diving_save_mean_diff\",\"saves_mean_diff\",\"high_to_low_goals_mean_diff\",\"accurate_layoffs_mean_diff\",\"goals_2t_mean_diff\",\"total_keeper_sweeper_mean_diff\",\"ranking_mean_diff\",\"first_yellow_card_mean_diff\",\"att_obx_right_mean_diff\",\"att_obox_miss_mean_diff\",\"bc_miss_div_scored_mean_diff\",\"six_yard_block_mean_diff\",\"att_rf_target_mean_diff\",\"bc_scored_div_created_mean_diff\",\"att_rf_total_mean_diff\",\"outfielder_block_mean_diff\",\"saved_obox_mean_diff\",\"won_corners_ratio_mean_diff\",\"rightside_pass_div_leftside_pass_mean_diff\",\"good_high_claim_mean_diff\",\"accurate_back_zone_pass_div_backward_pass_mean_diff\",\"attempts_ibox_mean_diff\",\"att_hd_goal_mean_diff\",\"rightside_pass_div_accurate_pass_mean_diff\",\"long_passes_mean_diff\",\"second_yellow_card_2t_mean_diff\",\"challenge_lost_mean_diff\",\"att_ibox_goal_mean_diff\",\"points_mean_diff\",\"total_contest_mean_diff\",\"bc_miss_div_created_mean_diff\",\"goal_assist_deadball_mean_diff\",\"att_obxd_right_mean_diff\",\"fwd_pass_mean_diff\",\"att_lg_centre_mean_diff\",\"last_man_tackle_mean_diff\",\"goal_kicks_per_shot_mean_diff\",\"fwd_pass_div_long_pass_own_to_opp_success_mean_diff\",\"att_goal_low_centre_mean_diff\",\"att_openplay_mean_diff\",\"pen_area_entries_mean_diff\",\"effective_clearance_mean_diff\",\"long_pass_own_to_opp_mean_diff\",\"unsuccessful_touch_mean_diff\",\"accurate_flick_on_mean_diff\",\"att_sv_high_right_mean_diff\",\"overrun_mean_diff\",\"touches_in_opp_box_mean_diff\",\"total_pull_back_mean_diff\",\"att_cmiss_right_mean_diff\",\"goal_fastbreak_mean_diff\",\"blocked_cross_mean_diff\",\"big_chance_scored_mean_diff\",\"attempts_obox_mean_diff\",\"lost_corners_mean_diff\",\"att_goal_low_right_mean_diff\",\"passes_right_mean_diff\",\"passes_left_div_blocked_pass_mean_diff\",\"att_cmiss_high_mean_diff\",\"long_pass_own_to_opp_success_mean_diff\",\"accurate_cross_nocorner_mean_diff\",\"total_red_card_mean_diff\",\"fk_foul_lost_mean_diff\",\"accurate_fwd_zone_pass_mean_diff\",\"second_yellow_card_1t_mean_diff\",\"failed_to_block_mean_diff\",\"accurate_pass_mean_diff\",\"leftside_pass_div_accurate_pass_mean_diff\",\"red_card_1t_mean_diff\",\"dispossessed_mean_diff\",\"pts_gained_losing_pos_mean_diff\",\"rightside_pass_mean_diff\",\"shield_ball_oop_mean_diff\",\"midfielder_goals_mean_diff\",\"interceptions_in_box_mean_diff\",\"att_lg_right_mean_diff\",\"won_corners_mean_diff\",\"att_pen_goal_mean_diff\",\"forward_goals_mean_diff\",\"accurate_cross_mean_diff\",\"acc_cross_nocorner_pct_mean_diff\",\"att_miss_high_left_mean_diff\",\"total_att_assist_mean_diff\",\"att_lf_target_mean_diff\",\"att_cmiss_high_right_mean_diff\",\"punches_mean_diff\",\"contentious_decision_mean_diff\",\"att_freekick_post_mean_diff\",\"interception_mean_diff\",\"total_throws_mean_diff\",\"crosses_18yard_mean_diff\",\"accurate_pull_back_mean_diff\",\"att_obox_goal_mean_diff\",\"fk_foul_won_mean_diff\",\"accurate_launches_mean_diff\",\"goal_assist_mean_diff\",\"att_lf_total_mean_diff\",\"rescinded_red_card_mean_diff\",\"backward_pass_mean_diff\",\"accurate_keeper_throws_mean_diff\",\"att_ibox_own_goal_mean_diff\",\"accurate_back_zone_pass_mean_diff\",\"hit_woodwork_mean_diff\",\"att_freekick_total_mean_diff\",\"accurate_goal_kicks_mean_diff\",\"won_tackle_mean_diff\",\"corners_mean_diff\",\"own_goals_mean_diff\",\"blocked_scoring_att_mean_diff\",\"poss_lost_all_mean_diff\",\"att_hd_miss_mean_diff\",\"shots_mean_diff\",\"att_ibox_target_mean_diff\",\"duel_won_mean_diff\",\"crosses_mean_diff\",\"att_ibox_post_mean_diff\",\"total_clearance_mean_diff\",\"attempts_conceded_ibox_mean_diff\",\"big_chance_missed_mean_diff\",\"goal_assist_intent_norm_mean_diff\",\"att_miss_high_right_mean_diff\",\"red_card_mean_diff\",\"passes_left_mean_diff\",\"left_div_right_foot_goals_mean_diff\",\"successful_fifty_fifty_mean_diff\",\"penalty_save_mean_diff\",\"att_obox_post_mean_diff\",\"att_lf_goal_mean_diff\",\"penalty_conceded_mean_diff\",\"freekick_cross_mean_diff\",\"error_lead_to_goal_mean_diff\",\"duel_lost_mean_diff\",\"att_bx_left_mean_diff\",\"goals_openplay_mean_diff\",\"shots_div_passes_left_mean_diff\",\"att_goal_high_right_mean_diff\",\"total_fwd_zone_pass_mean_diff\",\"corners_2t_mean_diff\",\"goal_kicks_mean_diff\",\"penalty_won_mean_diff\",\"goal_kicks_div_long_passes_mean_diff\",\"goals_conceded_obox_mean_diff\",\"DEF_mean_diff\",\"att_obx_centre_mean_diff\",\"att_assist_openplay_mean_diff\",\"total_flick_on_mean_diff\",\"att_bx_right_mean_diff\",\"subs_made_mean_diff\",\"goals_conceded_ibox_mean_diff\",\"att_obx_left_mean_diff\",\"pen_goals_conceded_mean_diff\",\"won_contest_mean_diff\",\"att_obox_target_mean_diff\",\"shot_off_target_mean_diff\",\"total_high_claim_mean_diff\",\"att_post_right_mean_diff\",\"penalty_faced_mean_diff\",\"att_cmiss_left_mean_diff\",\"att_hd_total_mean_diff\",\"keeper_throws_mean_diff\",\"total_fwd_zone_pass_div_long_pass_own_to_opp_success_mean_diff\",\"accurate_freekick_cross_mean_diff\",\"aerial_lost_mean_diff\",\"successful_final_third_passes_mean_diff\",\"att_sv_low_right_mean_diff\",\"goal_assist_intentional_mean_diff\",\"ATT_mean_diff\",\"second_yellow_card_mean_diff\",\"total_tackle_mean_diff\",\"effective_head_clearance_mean_diff\",\"att_fastbreak_mean_diff\",\"successful_put_through_mean_diff\",\"total_back_zone_pass_mean_diff\",\"blocked_pass_mean_diff\",\"ball_recovery_mean_diff\",\"att_corner_ratio_mean_diff\",\"att_freekick_target_mean_diff\",\"att_ibox_miss_mean_diff\",\"second_yellow_mean_diff\",\"MED_mean_diff\",\"offtarget_att_assist_mean_diff\",\"att_pen_target_mean_diff\",\"successful_open_play_pass_mean_diff\",\"att_setpiece_mean_diff\",\"att_ibox_blocked_mean_diff\",\"att_sv_low_left_mean_diff\",\"total_fastbreak_mean_diff\",\"percentage_goal_shot_mean_diff\",\"aerial_won_mean_diff\",\"goals_conceded_mean_diff\",\"att_miss_left_mean_diff\",\"att_cmiss_high_left_mean_diff\",\"clearance_off_line_mean_diff\",\"accurate_keeper_sweeper_mean_diff\",\"pts_dropped_winning_pos_mean_diff\",\"fouled_final_third_mean_diff\",\"att_lg_left_mean_diff\",\"defender_goals_mean_diff\",\"poss_won_def_3rd_mean_diff\",\"att_corner_mean_diff\",\"clean_sheet_mean_diff\",\"att_sv_high_centre_mean_diff\",\"hand_ball_mean_diff\",\"att_assist_setplay_mean_diff\",\"att_post_left_mean_diff\",\"att_hd_post_mean_diff\",\"total_pass_mean_diff\",\"accurate_chipped_pass_mean_diff\",\"poss_won_mid_3rd_mean_diff\",\"acc_corners_intobox_pct_std_diff\",\"att_pen_miss_std_diff\",\"att_rf_goal_std_diff\",\"att_goal_high_centre_std_diff\",\"att_sv_high_left_std_diff\",\"att_obox_blocked_std_diff\",\"att_goal_high_left_std_diff\",\"shots_mul_goals_std_diff\",\"first_yellow_card_1t_std_diff\",\"goals_std_diff\",\"att_post_high_std_diff\",\"att_hd_target_std_diff\",\"shots_div_passes_right_std_diff\",\"att_freekick_miss_std_diff\",\"corners_ratio_std_diff\",\"att_freekick_goal_std_diff\",\"att_miss_high_std_diff\",\"fifty_fifty_std_diff\",\"put_through_std_diff\",\"poss_lost_ctrl_std_diff\",\"att_pen_post_std_diff\",\"leftside_pass_std_diff\",\"total_corners_intobox_std_diff\",\"final_third_entries_std_diff\",\"total_through_ball_std_diff\",\"total_offside_std_diff\",\"total_layoffs_std_diff\",\"crosses_18yardplus_std_diff\",\"winner_std_diff\",\"att_obox_own_goal_std_diff\",\"accurate_long_balls_std_diff\",\"goal_assist_openplay_std_diff\",\"passes_right_div_blocked_pass_std_diff\",\"saved_ibox_std_diff\",\"red_card_2t_std_diff\",\"goals_1t_std_diff\",\"no_foot_goals_ratio_std_diff\",\"att_obxd_left_std_diff\",\"att_obp_goal_std_diff\",\"total_cross_nocorner_std_diff\",\"touches_std_diff\",\"foul_throw_in_std_diff\",\"corner_taken_std_diff\",\"duel_won_pct_std_diff\",\"error_lead_to_shot_std_diff\",\"goals_2t_pct_std_diff\",\"goal_assist_setplay_std_diff\",\"first_yellow_card_2t_std_diff\",\"att_goal_low_left_std_diff\",\"total_yel_card_std_diff\",\"accurate_through_ball_std_diff\",\"post_scoring_att_std_diff\",\"ontarget_att_assist_std_diff\",\"open_play_pass_std_diff\",\"accurate_throws_std_diff\",\"right_to_left_goals_std_diff\",\"total_launches_std_diff\",\"att_miss_right_std_diff\",\"attempted_tackle_foul_std_diff\",\"total_final_third_passes_std_diff\",\"head_clearance_std_diff\",\"att_sv_low_centre_std_diff\",\"poss_won_att_3rd_std_diff\",\"att_bx_centre_std_diff\",\"lineup_std_diff\",\"att_one_on_one_std_diff\",\"total_chipped_pass_std_diff\",\"attempts_conceded_obox_std_diff\",\"big_chance_created_std_diff\",\"accurate_corners_intobox_std_diff\",\"ontarget_scoring_att_std_diff\",\"diving_save_std_diff\",\"saves_std_diff\",\"high_to_low_goals_std_diff\",\"accurate_layoffs_std_diff\",\"goals_2t_std_diff\",\"total_keeper_sweeper_std_diff\",\"ranking_std_diff\",\"first_yellow_card_std_diff\",\"att_obx_right_std_diff\",\"att_obox_miss_std_diff\",\"bc_miss_div_scored_std_diff\",\"six_yard_block_std_diff\",\"att_rf_target_std_diff\",\"bc_scored_div_created_std_diff\",\"att_rf_total_std_diff\",\"outfielder_block_std_diff\",\"saved_obox_std_diff\",\"won_corners_ratio_std_diff\",\"rightside_pass_div_leftside_pass_std_diff\",\"good_high_claim_std_diff\",\"accurate_back_zone_pass_div_backward_pass_std_diff\",\"attempts_ibox_std_diff\",\"att_hd_goal_std_diff\",\"rightside_pass_div_accurate_pass_std_diff\",\"long_passes_std_diff\",\"second_yellow_card_2t_std_diff\",\"challenge_lost_std_diff\",\"att_ibox_goal_std_diff\",\"points_std_diff\",\"total_contest_std_diff\",\"bc_miss_div_created_std_diff\",\"goal_assist_deadball_std_diff\",\"att_obxd_right_std_diff\",\"fwd_pass_std_diff\",\"att_lg_centre_std_diff\",\"last_man_tackle_std_diff\",\"goal_kicks_per_shot_std_diff\",\"fwd_pass_div_long_pass_own_to_opp_success_std_diff\",\"att_goal_low_centre_std_diff\",\"att_openplay_std_diff\",\"pen_area_entries_std_diff\",\"effective_clearance_std_diff\",\"long_pass_own_to_opp_std_diff\",\"unsuccessful_touch_std_diff\",\"accurate_flick_on_std_diff\",\"att_sv_high_right_std_diff\",\"overrun_std_diff\",\"touches_in_opp_box_std_diff\",\"total_pull_back_std_diff\",\"att_cmiss_right_std_diff\",\"goal_fastbreak_std_diff\",\"blocked_cross_std_diff\",\"big_chance_scored_std_diff\",\"attempts_obox_std_diff\",\"lost_corners_std_diff\",\"att_goal_low_right_std_diff\",\"passes_right_std_diff\",\"passes_left_div_blocked_pass_std_diff\",\"att_cmiss_high_std_diff\",\"long_pass_own_to_opp_success_std_diff\",\"accurate_cross_nocorner_std_diff\",\"total_red_card_std_diff\",\"fk_foul_lost_std_diff\",\"accurate_fwd_zone_pass_std_diff\",\"second_yellow_card_1t_std_diff\",\"failed_to_block_std_diff\",\"accurate_pass_std_diff\",\"leftside_pass_div_accurate_pass_std_diff\",\"red_card_1t_std_diff\",\"dispossessed_std_diff\",\"pts_gained_losing_pos_std_diff\",\"rightside_pass_std_diff\",\"shield_ball_oop_std_diff\",\"midfielder_goals_std_diff\",\"interceptions_in_box_std_diff\",\"att_lg_right_std_diff\",\"won_corners_std_diff\",\"att_pen_goal_std_diff\",\"forward_goals_std_diff\",\"accurate_cross_std_diff\",\"acc_cross_nocorner_pct_std_diff\",\"att_miss_high_left_std_diff\",\"total_att_assist_std_diff\",\"att_lf_target_std_diff\",\"att_cmiss_high_right_std_diff\",\"punches_std_diff\",\"contentious_decision_std_diff\",\"att_freekick_post_std_diff\",\"interception_std_diff\",\"total_throws_std_diff\",\"crosses_18yard_std_diff\",\"accurate_pull_back_std_diff\",\"att_obox_goal_std_diff\",\"fk_foul_won_std_diff\",\"accurate_launches_std_diff\",\"goal_assist_std_diff\",\"att_lf_total_std_diff\",\"rescinded_red_card_std_diff\",\"backward_pass_std_diff\",\"accurate_keeper_throws_std_diff\",\"att_ibox_own_goal_std_diff\",\"accurate_back_zone_pass_std_diff\",\"hit_woodwork_std_diff\",\"att_freekick_total_std_diff\",\"accurate_goal_kicks_std_diff\",\"won_tackle_std_diff\",\"corners_std_diff\",\"own_goals_std_diff\",\"blocked_scoring_att_std_diff\",\"poss_lost_all_std_diff\",\"att_hd_miss_std_diff\",\"shots_std_diff\",\"att_ibox_target_std_diff\",\"duel_won_std_diff\",\"crosses_std_diff\",\"att_ibox_post_std_diff\",\"total_clearance_std_diff\",\"attempts_conceded_ibox_std_diff\",\"big_chance_missed_std_diff\",\"goal_assist_intent_norm_std_diff\",\"att_miss_high_right_std_diff\",\"red_card_std_diff\",\"passes_left_std_diff\",\"left_div_right_foot_goals_std_diff\",\"successful_fifty_fifty_std_diff\",\"penalty_save_std_diff\",\"att_obox_post_std_diff\",\"att_lf_goal_std_diff\",\"penalty_conceded_std_diff\",\"freekick_cross_std_diff\",\"error_lead_to_goal_std_diff\",\"duel_lost_std_diff\",\"att_bx_left_std_diff\",\"goals_openplay_std_diff\",\"shots_div_passes_left_std_diff\",\"att_goal_high_right_std_diff\",\"total_fwd_zone_pass_std_diff\",\"corners_2t_std_diff\",\"goal_kicks_std_diff\",\"penalty_won_std_diff\",\"goal_kicks_div_long_passes_std_diff\",\"goals_conceded_obox_std_diff\",\"DEF_std_diff\",\"att_obx_centre_std_diff\",\"att_assist_openplay_std_diff\",\"total_flick_on_std_diff\",\"att_bx_right_std_diff\",\"subs_made_std_diff\",\"goals_conceded_ibox_std_diff\",\"att_obx_left_std_diff\",\"pen_goals_conceded_std_diff\",\"won_contest_std_diff\",\"att_obox_target_std_diff\",\"shot_off_target_std_diff\",\"total_high_claim_std_diff\",\"att_post_right_std_diff\",\"penalty_faced_std_diff\",\"att_cmiss_left_std_diff\",\"att_hd_total_std_diff\",\"keeper_throws_std_diff\",\"total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_diff\",\"accurate_freekick_cross_std_diff\",\"aerial_lost_std_diff\",\"successful_final_third_passes_std_diff\",\"att_sv_low_right_std_diff\",\"goal_assist_intentional_std_diff\",\"ATT_std_diff\",\"second_yellow_card_std_diff\",\"total_tackle_std_diff\",\"effective_head_clearance_std_diff\",\"att_fastbreak_std_diff\",\"successful_put_through_std_diff\",\"total_back_zone_pass_std_diff\",\"blocked_pass_std_diff\",\"ball_recovery_std_diff\",\"att_corner_ratio_std_diff\",\"att_freekick_target_std_diff\",\"att_ibox_miss_std_diff\",\"second_yellow_std_diff\",\"MED_std_diff\",\"offtarget_att_assist_std_diff\",\"att_pen_target_std_diff\",\"successful_open_play_pass_std_diff\",\"att_setpiece_std_diff\",\"att_ibox_blocked_std_diff\",\"att_sv_low_left_std_diff\",\"total_fastbreak_std_diff\",\"percentage_goal_shot_std_diff\",\"aerial_won_std_diff\",\"goals_conceded_std_diff\",\"att_miss_left_std_diff\",\"att_cmiss_high_left_std_diff\",\"clearance_off_line_std_diff\",\"accurate_keeper_sweeper_std_diff\",\"pts_dropped_winning_pos_std_diff\",\"fouled_final_third_std_diff\",\"att_lg_left_std_diff\",\"defender_goals_std_diff\",\"poss_won_def_3rd_std_diff\",\"att_corner_std_diff\",\"clean_sheet_std_diff\",\"att_sv_high_centre_std_diff\",\"hand_ball_std_diff\",\"att_assist_setplay_std_diff\",\"att_post_left_std_diff\",\"att_hd_post_std_diff\",\"total_pass_std_diff\",\"accurate_chipped_pass_std_diff\",\"poss_won_mid_3rd_std_diff\",\"win_pct_home_over_both_score_diff\",\"odds_home_over_both_score_diff\",\"win_pct_away_over_both_score_diff\",\"odds_away_over_both_score_diff\",\"total_bets_over_both_score_diff\",\"total_win_pct_over_both_score_diff\",\"odds_over_both_score_diff\",\"imp_prob_over_both_score_diff\",\"ratio_over_both_score_diff\",\"win_pct_home_under_both_score_diff\",\"odds_home_under_both_score_diff\",\"win_pct_away_under_both_score_diff\",\"odds_away_under_both_score_diff\",\"total_bets_under_both_score_diff\",\"total_win_pct_under_both_score_diff\",\"odds_under_both_score_diff\",\"imp_prob_under_both_score_diff\",\"ratio_under_both_score_diff\",\"win_pct_home_over_goals_1.5_diff\",\"odds_home_over_goals_1.5_diff\",\"win_pct_away_over_goals_1.5_diff\",\"odds_away_over_goals_1.5_diff\",\"total_bets_over_goals_1.5_diff\",\"total_win_pct_over_goals_1.5_diff\",\"odds_over_goals_1.5_diff\",\"imp_prob_over_goals_1.5_diff\",\"ratio_over_goals_1.5_diff\",\"win_pct_home_under_goals_1.5_diff\",\"odds_home_under_goals_1.5_diff\",\"win_pct_away_under_goals_1.5_diff\",\"odds_away_under_goals_1.5_diff\",\"total_bets_under_goals_1.5_diff\",\"total_win_pct_under_goals_1.5_diff\",\"odds_under_goals_1.5_diff\",\"imp_prob_under_goals_1.5_diff\",\"ratio_under_goals_1.5_diff\",\"win_pct_home_over_goals_2.5_diff\",\"odds_home_over_goals_2.5_diff\",\"win_pct_away_over_goals_2.5_diff\",\"odds_away_over_goals_2.5_diff\",\"total_bets_over_goals_2.5_diff\",\"total_win_pct_over_goals_2.5_diff\",\"odds_over_goals_2.5_diff\",\"imp_prob_over_goals_2.5_diff\",\"ratio_over_goals_2.5_diff\",\"win_pct_home_under_goals_2.5_diff\",\"odds_home_under_goals_2.5_diff\",\"win_pct_away_under_goals_2.5_diff\",\"odds_away_under_goals_2.5_diff\",\"total_bets_under_goals_2.5_diff\",\"total_win_pct_under_goals_2.5_diff\",\"odds_under_goals_2.5_diff\",\"imp_prob_under_goals_2.5_diff\",\"ratio_under_goals_2.5_diff\",\"win_pct_home_over_goals_3.5_diff\",\"odds_home_over_goals_3.5_diff\",\"win_pct_away_over_goals_3.5_diff\",\"odds_away_over_goals_3.5_diff\",\"total_bets_over_goals_3.5_diff\",\"total_win_pct_over_goals_3.5_diff\",\"odds_over_goals_3.5_diff\",\"imp_prob_over_goals_3.5_diff\",\"ratio_over_goals_3.5_diff\",\"win_pct_home_under_goals_3.5_diff\",\"odds_home_under_goals_3.5_diff\",\"win_pct_away_under_goals_3.5_diff\",\"odds_away_under_goals_3.5_diff\",\"total_bets_under_goals_3.5_diff\",\"total_win_pct_under_goals_3.5_diff\",\"odds_under_goals_3.5_diff\",\"imp_prob_under_goals_3.5_diff\",\"ratio_under_goals_3.5_diff\",\"win_pct_home_over_goals_4.5_diff\",\"odds_home_over_goals_4.5_diff\",\"win_pct_away_over_goals_4.5_diff\",\"odds_away_over_goals_4.5_diff\",\"total_bets_over_goals_4.5_diff\",\"total_win_pct_over_goals_4.5_diff\",\"odds_over_goals_4.5_diff\",\"imp_prob_over_goals_4.5_diff\",\"ratio_over_goals_4.5_diff\",\"win_pct_home_under_goals_4.5_diff\",\"odds_home_under_goals_4.5_diff\",\"win_pct_away_under_goals_4.5_diff\",\"odds_away_under_goals_4.5_diff\",\"total_bets_under_goals_4.5_diff\",\"total_win_pct_under_goals_4.5_diff\",\"odds_under_goals_4.5_diff\",\"imp_prob_under_goals_4.5_diff\",\"ratio_under_goals_4.5_diff\",\"shots_mean_h_mul_shots_mean_a\",\"corner_taken_std_mul_att_corner_ratio_mean_diffs\",\"big_chance_scored_mean_diff_mul_total_fwd_zone_pass_mean_diff\",\"ATT_h_mul_shots_mul_goals_mean_diff\",\"total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_diff\",\"DEF_h_mul_shots_mul_goals_mean_diff\",\"is_weekend\",\"hour_rank\",\"hour_before_16\",\"derby\",\"prob_under_both_score\",\"raw_prob_under_both_score\",\"odd_ratio_under_both_score\",\"prob_squared_under_both_score\",\"prob_over_both_score\",\"raw_prob_over_both_score\",\"odd_ratio_over_both_score\",\"prob_squared_over_both_score\",\"prob_under_goals_1.5\",\"raw_prob_under_goals_1.5\",\"odd_ratio_under_goals_1.5\",\"prob_squared_under_goals_1.5\",\"prob_over_goals_1.5\",\"raw_prob_over_goals_1.5\",\"odd_ratio_over_goals_1.5\",\"prob_squared_over_goals_1.5\",\"prob_under_goals_2.5\",\"raw_prob_under_goals_2.5\",\"odd_ratio_under_goals_2.5\",\"prob_squared_under_goals_2.5\",\"prob_over_goals_2.5\",\"raw_prob_over_goals_2.5\",\"odd_ratio_over_goals_2.5\",\"prob_squared_over_goals_2.5\",\"prob_under_goals_3.5\",\"raw_prob_under_goals_3.5\",\"odd_ratio_under_goals_3.5\",\"prob_squared_under_goals_3.5\",\"prob_over_goals_3.5\",\"raw_prob_over_goals_3.5\",\"odd_ratio_over_goals_3.5\",\"prob_squared_over_goals_3.5\",\"prob_under_goals_4.5\",\"raw_prob_under_goals_4.5\",\"odd_ratio_under_goals_4.5\",\"prob_squared_under_goals_4.5\",\"prob_over_goals_4.5\",\"raw_prob_over_goals_4.5\",\"odd_ratio_over_goals_4.5\",\"prob_squared_over_goals_4.5\",\"goals_2.5\",\"red_card_home_ref\",\"red_card_away_ref\",\"total_red_card_1t_ref\",\"total_red_card_2t_ref\",\"total_red_card_match_ref\",\"total_yel_card_home_ref\",\"total_yel_card_away_ref\",\"total_yel_card_match_ref\",\"total_yel_card_1t_ref\",\"total_yel_card_2t_ref\",\"total_first_yel_card_ref\",\"total_first_yel_card_home_ref\",\"total_first_yel_card_away_ref\",\"total_second_yel_card_ref\",\"total_second_yel_card_home_ref\",\"total_second_yel_card_away_ref\",\"total_expulsions_ref\",\"total_tackle_ref\",\"total_fk_foul_ref\",\"total_penalty_match_ref\",\"total_hand_ball_ref\",\"total_fouls_home_ref\",\"total_fouls_away_ref\",\"total_fouls_match_ref\",\"foul_per_tackle_ref\",\"fk_per_tackle_ref\",\"Label\",\"Score\"],\"title\":\"Ignore features\"},\"id\":\"2275\",\"type\":\"MultiSelect\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Kappa\",\"value\":0.1},\"id\":\"2287\",\"type\":\"Slider\"},{\"attributes\":{\"children\":[{\"id\":\"2281\"},{\"id\":\"2282\"},{\"id\":\"2283\"},{\"id\":\"2284\"},{\"id\":\"2285\"},{\"id\":\"2286\"},{\"id\":\"2287\"},{\"id\":\"2288\"},{\"id\":\"2289\"}],\"margin\":[5,5,5,5],\"min_width\":300,\"name\":\"Metric and optimize parameters\",\"width\":300},\"id\":\"2280\",\"type\":\"Column\"}],\"root_ids\":[\"2266\",\"2293\"]},\"title\":\"Bokeh Application\",\"version\":\"2.4.2\"}};\n",
+       "    var render_items = [{\"docid\":\"0fb65867-6b3e-43ae-a5a0-be2405b34da2\",\"root_ids\":[\"2266\"],\"roots\":{\"2266\":\"43bf8ba1-0223-4d56-b328-a9210fb5c081\"}}];\n",
        "    root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
        "  }\n",
        "  if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
@@ -1785,17 +1391,18 @@
       "text/plain": [
        "Row\n",
        "    [0] Column\n",
-       "        [0] Column(margin=5, name='Feature parameters', width=300)\n",
-       "            [0] StaticText(value='<b>Feature parameters</b>...)\n",
-       "            [1] TextInput(name='Target', value='miau')\n",
-       "            [2] FloatSlider(end=2, name='Number features', value=0.5)\n",
-       "            [3] FloatSlider(end=2, name='Target features', value=0.3)\n",
-       "            [4] FloatSlider(end=100, name='Feature division', start=1, value=3)\n",
-       "            [5] MultiSelect(name='Ignore features', options=OrderedDict([('a', ...]))\n",
-       "            [6] MultiSelect(name='Sort', options=OrderedDict([('Accuracy', ...]), value=['AUC'])\n",
+       "        [0] Column(margin=5, name='Feature parameters a..., width=300)\n",
+       "            [0] StaticText(value='<b>Feature parameters a...)\n",
+       "            [1] TextInput(name='Target', value='Score')\n",
+       "            [2] FloatSlider(name='Number features', value=0.5)\n",
+       "            [3] DiscreteSlider(name='Target features', options=[0.01, 0.02, 0.03, ...], value=0.3, width=280)\n",
+       "            [4] MultiSelect(name='Ignore features', options=OrderedDict([('None', ...]))\n",
+       "            [5] Select(name='Sort', options=OrderedDict([('Accuracy', ...]), value='AUC')\n",
+       "            [6] MultiSelect(name='Include', options=OrderedDict([('None', ...]))\n",
+       "            [7] IntSlider(end=15, name='Number models', start=2, value=10)\n",
        "    [1] Column\n",
-       "        [0] Column(margin=5, name='Models included a..., width=300)\n",
-       "            [0] StaticText(value='<b>Models included a...)\n",
+       "        [0] Column(margin=5, name='Metric and optimize p..., width=300)\n",
+       "            [0] StaticText(value='<b>Metric and o...)\n",
        "            [1] FloatSlider(name='Accuracy', value=0.6)\n",
        "            [2] FloatSlider(name='AUC', value=0.6)\n",
        "            [3] FloatSlider(name='Recall', value=0.6)\n",
@@ -1803,102 +1410,558 @@
        "            [5] FloatSlider(name='F-score', value=0.6)\n",
        "            [6] FloatSlider(name='Kappa', value=0.1)\n",
        "            [7] FloatSlider(name='MCC', value=0.1)\n",
-       "            [8] MultiSelect(name='Include', options=OrderedDict([('lr', ...]))\n",
-       "            [9] IntSlider(end=15, name='Number models', start=2, value=10)\n",
-       "            [10] Checkbox(name='Tuning process')\n",
+       "            [8] Checkbox(name='Tuning process')\n",
        "    [2] Column\n",
-       "        [0] Button(button_type='success', name='Start process!')\n",
-       "    [3] Column\n",
-       "        [0] ParamMethod(method)"
+       "        [0] Button(button_type='success', name='Start process!')"
       ]
      },
-     "execution_count": 26,
+     "execution_count": 285,
      "metadata": {
       "application/vnd.holoviews_exec.v0+json": {
-       "id": "1264"
+       "id": "2266"
       }
      },
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "a = Widget(dataset=pd.DataFrame({'a': [1,23,4,4], 'b': [1,2,2,2]}))\n",
+    "a = Widget(dataset=predictions)#pd.DataFrame({'a': [1,23,4,4], 'b': [1,2,2,2]}))\n",
     "a.interactive_panel()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
-   "id": "acf267b1",
+   "execution_count": 287,
+   "id": "0b748748",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['rf']"
+       "{'acu': 0.6,\n",
+       " 'auc': 0.6,\n",
+       " 'rec': 0.6,\n",
+       " 'pre': 0.6,\n",
+       " 'f1': 0.9,\n",
+       " 'kappa': 0.1,\n",
+       " 'mcc': 0.1}"
       ]
      },
-     "execution_count": 29,
+     "execution_count": 287,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "a.include"
+    "a.filter_metrics"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "id": "78d9264c",
+   "execution_count": 264,
+   "id": "07451211",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ll\\ \n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "if not a.include:\n",
+    "    print('ll\\ ')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 296,
+   "id": "54995c89",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "translate = {'acu': \"Accuracy\",\n",
+    "  'auc': \"AUC\",\n",
+    "  'rec': \"Recall\",\n",
+    "  'pre': \"Precision\",\n",
+    "  'f1': \"F1\",\n",
+    "  'kappa': \"Kappa\", \n",
+    "  'mcc': \"MCC\"}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 298,
+   "id": "a28037ad",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "'pepis'"
+       "['name',\n",
+       " 'acu',\n",
+       " 'auc',\n",
+       " 'rec',\n",
+       " 'pre',\n",
+       " 'f1',\n",
+       " 'kappa',\n",
+       " 'mcc',\n",
+       " 'dataset',\n",
+       " 'include']"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 298,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "a.target"
+    "not_list = ['name', \"acu\", \"auc\", \"rec\", \"pre\", \"f1\", \"kappa\", \"mcc\", 'dataset']\n",
+    "if (not a.include or a.include==[None]):\n",
+    "    not_list = not_list + ['include']\n",
+    "kwargs = {c: getattr(a, c) for c in a.param if c not in not_list}\n",
+    "#FeatureSelection(**{c: getattr(a, c) for c in a.param if c not in not_list})\n",
+    "not_list"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 62,
-   "id": "d1dbb192",
+   "execution_count": 291,
+   "id": "59015732",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "0.9"
+       "{'target': 'Score',\n",
+       " 'number_features': 0.5,\n",
+       " 'target_features': 0.3,\n",
+       " 'ignore_features': None,\n",
+       " 'sort': 'AUC',\n",
+       " 'number_models': 10,\n",
+       " 'optimize': False,\n",
+       " 'opt_list': ['AUC'],\n",
+       " 'filter_metrics': {'acu': 0.6,\n",
+       "  'auc': 0.6,\n",
+       "  'rec': 0.6,\n",
+       "  'pre': 0.6,\n",
+       "  'f1': 0.9,\n",
+       "  'kappa': 0.1,\n",
+       "  'mcc': 0.1}}"
       ]
      },
-     "execution_count": 62,
+     "execution_count": 291,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "a.pre"
+    "kwargs['filter_metrics'] = {}"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "id": "b248cf01-fa47-466f-8f45-4b43cac3aa94",
+   "execution_count": 292,
+   "id": "bc53e928",
    "metadata": {},
    "outputs": [
     {
-     "data": {},
-     "metadata": {},
-     "output_type": "display_data"
+     "data": {
+      "text/plain": [
+       "FeatureSelection(dataset=                           match_id  win_pct_h  draw_pct_h  loss_pct_h  \\\n",
+       "0           2220927_mexican_primera        0.2         0.2         0.6   \n",
+       "1           2219375_spanish_la_liga        0.6         0.0         0.4   \n",
+       "2           2229005_italian_serie_a        0.2         0.2         0.6   \n",
+       "3           2219371_spanish_la_liga        0.0         0.4         0.6   \n",
+       "4    2210370_english_premier_league        0.6         0.2         0.2   \n",
+       "..                              ...        ...         ...         ...   \n",
+       "495       2215358_german_bundesliga        0.2         0.4         0.4   \n",
+       "496       2215359_german_bundesliga        0.4         0.4         0.2   \n",
+       "497         2229119_italian_serie_a        0.4         0.2         0.4   \n",
+       "498         2229114_italian_serie_a        0.6         0.2         0.2   \n",
+       "499          2215697_french_ligue_1        0.6         0.2         0.2   \n",
+       "\n",
+       "     win_streak_h  loss_streak_h  draw_streak_h  points_h  ranking_h  ATT_h  \\\n",
+       "0             0.0            2.0            0.0      40.0       12.0   72.0   \n",
+       "1             1.0            0.0            0.0      19.0       23.0   70.0   \n",
+       "2             0.0            2.0            0.0      12.0       10.0   75.0   \n",
+       "3             0.0            0.0            1.0       6.0       15.0   78.0   \n",
+       "4             0.0            0.0            1.0      13.0        7.0   78.0   \n",
+       "..            ...            ...            ...       ...        ...    ...   \n",
+       "495           0.0            1.0            0.0      18.0       15.0   75.0   \n",
+       "496           0.0            0.0            1.0      17.0       25.0   72.0   \n",
+       "497           0.0            2.0            0.0      32.0        6.0   80.0   \n",
+       "498           1.0            0.0            0.0      41.0        3.0   82.0   \n",
+       "499           1.0            0.0            0.0      36.0        5.0   78.0   \n",
+       "\n",
+       "     ...  total_fk_foul_ref  total_penalty_match_ref  total_hand_ball_ref  \\\n",
+       "0    ...          22.222222                 0.444444             1.666667   \n",
+       "1    ...          26.000000                 0.875000             1.625000   \n",
+       "2    ...          24.800000                 0.000000             0.600000   \n",
+       "3    ...          26.142857                 0.142857             1.285714   \n",
+       "4    ...          14.444444                 0.333333             0.777778   \n",
+       "..   ...                ...                      ...                  ...   \n",
+       "495  ...          19.400000                 0.200000             0.400000   \n",
+       "496  ...          24.333333                 0.666667             1.000000   \n",
+       "497  ...          27.000000                 1.000000             0.000000   \n",
+       "498  ...          24.800000                 0.000000             0.600000   \n",
+       "499  ...          22.125000                 0.375000             0.625000   \n",
+       "\n",
+       "     total_fouls_home_ref  total_fouls_away_ref  total_fouls_match_ref  \\\n",
+       "0               12.111111             11.777778              23.888889   \n",
+       "1               14.375000             13.250000              27.625000   \n",
+       "2               13.600000             11.800000              25.400000   \n",
+       "3               13.000000             14.428571              27.428571   \n",
+       "4                7.444444              7.777778              15.222222   \n",
+       "..                    ...                   ...                    ...   \n",
+       "495              9.400000             10.400000              19.800000   \n",
+       "496             10.666667             14.666667              25.333333   \n",
+       "497             18.000000              9.000000              27.000000   \n",
+       "498             13.600000             11.800000              25.400000   \n",
+       "499             14.000000              8.750000              22.750000   \n",
+       "\n",
+       "     foul_per_tackle_ref  fk_per_tackle_ref  Label   Score  \n",
+       "0               0.802636           0.747847      0  0.6280  \n",
+       "1               1.184427           1.118378      0  0.6345  \n",
+       "2               0.878321           0.853986      0  0.5524  \n",
+       "3               0.915621           0.877201      0  0.4604  \n",
+       "4               0.504600           0.478298      0  0.7605  \n",
+       "..                   ...                ...    ...     ...  \n",
+       "495             0.611544           0.601018      0  0.4456  \n",
+       "496             0.692162           0.663957      0  0.4098  \n",
+       "497             0.658537           0.658537      0  0.5200  \n",
+       "498             0.878321           0.853986      0  0.4291  \n",
+       "499             0.563708           0.545911      0  0.6225  \n",
+       "\n",
+       "[500 rows x 2063 columns], dict_models=None, exclude=['qda', 'knn', 'nb'], features_df=None, filter_metrics={'Accuracy': 0.5, 'AUC': 0.5, 'Recall': 0.6, 'Precision': 0.6, 'F1': 0.6, 'Kappa': 0.1, 'MCC': 0.1}, ignore_features=None, include=None, model_df=None, model_tuned_df=None, name='FeatureSelection02215', number_features=0.5, number_models=10, numerics=['int16', 'int32', 'int64', 'float16', 'float32', 'float64', 'int', 'float'], opt_list=['AUC'], optimize=False, setup_kwargs={'preprocess': True, 'train_size': 0.75, 'session_id': 123, 'normalize': True, 'transformation': True, 'ignore_low_variance': True, 'remove_multicollinearity': False, 'multicollinearity_threshold': 0.4, 'n_jobs': -1, 'use_gpu': False, 'profile': False, 'ignore_features': None, 'fold_strategy': 'timeseries', 'remove_perfect_collinearity': True, 'create_clusters': False, 'fold': 4, 'feature_selection': False, 'feature_selection_threshold': 0.4, 'combine_rare_levels': False, 'rare_level_threshold': 0.02, 'pca': False, 'pca_method': 'kernel', 'pca_components': 30, 'polynomial_features': False, 'polynomial_degree': 2, 'polynomial_threshold': 0.01, 'trigonometry_features': False, 'remove_outliers': False, 'outliers_threshold': 0.01, 'feature_ratio': False, 'feature_interaction': False, 'interaction_threshold': 0.01, 'fix_imbalance': False, 'log_experiment': False, 'verbose': False, 'silent': True, 'experiment_name': 'lagstest', 'html': False}, sort='AUC', target='Score', target_features=618, top_models=None, tune_dict_models=None, x_df=Empty DataFrame\n",
+       "Columns: []\n",
+       "Index: [], x_train=None)"
+      ]
+     },
+     "execution_count": 292,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "FeatureSelection(predictions, **{'target': 'Score',\n",
+    " 'number_features': 0.5,\n",
+    " 'target_features': 0.3,\n",
+    " 'ignore_features': None,\n",
+    " 'sort': 'AUC',\n",
+    " 'number_models': 10,\n",
+    " 'optimize': False,\n",
+    " 'opt_list': ['AUC'],\n",
+    " 'filter_metrics': {'Accuracy': 0.5,\n",
+    "  'AUC': 0.5,\n",
+    "  'Recall': 0.6,\n",
+    "  'Precision': 0.6,\n",
+    "  'F1': 0.6,\n",
+    "  'Kappa': 0.1,\n",
+    "  'MCC': 0.1}})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 239,
+   "id": "c41ba064",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[]"
+      ]
+     },
+     "execution_count": 239,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pepe.ignore_features"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "52bb605d",
+   "metadata": {},
+   "source": [
+    "list(a._filter_metric.keys())[:-2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 72,
+   "id": "fac0f7df",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pn.panel(a.param.number_features).width = 500\n",
+    "#np.linspace(1,a.dataset.shape[1],200, dtype=int)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "id": "271b2aec",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {},
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.holoviews_exec.v0+json": "",
+      "text/html": [
+       "<div id='1234'>\n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "  <div class=\"bk-root\" id=\"4d76135e-0f3e-428d-b2d8-500df1800b2c\" data-root-id=\"1234\"></div>\n",
+       "</div>\n",
+       "<script type=\"application/javascript\">(function(root) {\n",
+       "  function embed_document(root) {\n",
+       "    var docs_json = {\"6ccdbcc7-fb08-40e8-8b48-d2f7b111bf18\":{\"defs\":[{\"extends\":null,\"module\":null,\"name\":\"ReactiveHTML1\",\"overrides\":[],\"properties\":[]},{\"extends\":null,\"module\":null,\"name\":\"FlexBox1\",\"overrides\":[],\"properties\":[{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_content\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_items\"},{\"default\":\"row\",\"kind\":null,\"name\":\"flex_direction\"},{\"default\":\"wrap\",\"kind\":null,\"name\":\"flex_wrap\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"justify_content\"}]},{\"extends\":null,\"module\":null,\"name\":\"TemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]},{\"extends\":null,\"module\":null,\"name\":\"MaterialTemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]}],\"roots\":{\"references\":[{\"attributes\":{\"client_comm_id\":\"64815ab98e1d48be82e3440d155fc87c\",\"comm_id\":\"ebf41c9a8f754095b7fe4f62c73c5564\",\"plot_id\":\"1234\"},\"id\":\"1235\",\"type\":\"panel.models.comm_manager.CommManager\"},{\"attributes\":{\"end\":1,\"margin\":[5,10,5,10],\"start\":0,\"step\":0.1,\"title\":\"Number features\",\"value\":0.5},\"id\":\"1234\",\"type\":\"Slider\"}],\"root_ids\":[\"1234\",\"1235\"]},\"title\":\"Bokeh Application\",\"version\":\"2.4.2\"}};\n",
+       "    var render_items = [{\"docid\":\"6ccdbcc7-fb08-40e8-8b48-d2f7b111bf18\",\"root_ids\":[\"1234\"],\"roots\":{\"1234\":\"4d76135e-0f3e-428d-b2d8-500df1800b2c\"}}];\n",
+       "    root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+       "  }\n",
+       "  if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
+       "    embed_document(root);\n",
+       "  } else {\n",
+       "    var attempts = 0;\n",
+       "    var timer = setInterval(function(root) {\n",
+       "      if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
+       "        clearInterval(timer);\n",
+       "        embed_document(root);\n",
+       "      } else if (document.readyState == \"complete\") {\n",
+       "        attempts++;\n",
+       "        if (attempts > 200) {\n",
+       "          clearInterval(timer);\n",
+       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+       "        }\n",
+       "      }\n",
+       "    }, 25, root)\n",
+       "  }\n",
+       "})(window);</script>"
+      ],
+      "text/plain": [
+       "FloatSlider(name='Number features', value=0.5)"
+      ]
+     },
+     "execution_count": 62,
+     "metadata": {
+      "application/vnd.holoviews_exec.v0+json": {
+       "id": "1234"
+      }
+     },
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pn.panel(a.param.number_features)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "id": "3cbb4bfe",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {},
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.holoviews_exec.v0+json": "",
+      "text/html": [
+       "<div id='1225'>\n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "  <div class=\"bk-root\" id=\"010af84a-902b-48ab-bf73-f1c954048b6a\" data-root-id=\"1225\"></div>\n",
+       "</div>\n",
+       "<script type=\"application/javascript\">(function(root) {\n",
+       "  function embed_document(root) {\n",
+       "    var docs_json = {\"46033f1f-d2aa-4bd8-9bfc-ab3d40536290\":{\"defs\":[{\"extends\":null,\"module\":null,\"name\":\"ReactiveHTML1\",\"overrides\":[],\"properties\":[]},{\"extends\":null,\"module\":null,\"name\":\"FlexBox1\",\"overrides\":[],\"properties\":[{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_content\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_items\"},{\"default\":\"row\",\"kind\":null,\"name\":\"flex_direction\"},{\"default\":\"wrap\",\"kind\":null,\"name\":\"flex_wrap\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"justify_content\"}]},{\"extends\":null,\"module\":null,\"name\":\"TemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]},{\"extends\":null,\"module\":null,\"name\":\"MaterialTemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]}],\"roots\":{\"references\":[{\"attributes\":{\"margin\":[5,0,0,10],\"name\":\"\",\"style\":{\"white-space\":\"nowrap\"},\"text\":\"Discrete Slider: <b>32</b>\"},\"id\":\"1226\",\"type\":\"Div\"},{\"attributes\":{\"args\":{\"bidirectional\":false,\"properties\":{},\"source\":{\"id\":\"1227\"},\"target\":{\"id\":\"1226\"}},\"code\":\"try { \\n    var labels = ['Discrete Slider: <b>2</b>', 'Discrete Slider: <b>4</b>', 'Discrete Slider: <b>8</b>', 'Discrete Slider: <b>16</b>', 'Discrete Slider: <b>32</b>', 'Discrete Slider: <b>64</b>', 'Discrete Slider: <b>128</b>']\\n    target.text = labels[source.value]\\n     } catch(err) { console.log(err) }\",\"tags\":[4694347536]},\"id\":\"1228\",\"type\":\"CustomJS\"},{\"attributes\":{\"children\":[{\"id\":\"1226\"},{\"id\":\"1227\"}],\"margin\":[0,0,0,0],\"name\":\"Column00446\"},\"id\":\"1225\",\"type\":\"Column\"},{\"attributes\":{\"end\":6,\"js_property_callbacks\":{\"change:value\":[{\"id\":\"1228\"}]},\"margin\":[0,10,5,10],\"show_value\":false,\"start\":0,\"tooltips\":false,\"value\":4},\"id\":\"1227\",\"type\":\"Slider\"},{\"attributes\":{\"client_comm_id\":\"038b13945c724531baaa717dbd7c7084\",\"comm_id\":\"c04e391e314148f3b33594cbdd2ade63\",\"plot_id\":\"1225\"},\"id\":\"1229\",\"type\":\"panel.models.comm_manager.CommManager\"}],\"root_ids\":[\"1225\",\"1229\"]},\"title\":\"Bokeh Application\",\"version\":\"2.4.2\"}};\n",
+       "    var render_items = [{\"docid\":\"46033f1f-d2aa-4bd8-9bfc-ab3d40536290\",\"root_ids\":[\"1225\"],\"roots\":{\"1225\":\"010af84a-902b-48ab-bf73-f1c954048b6a\"}}];\n",
+       "    root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+       "  }\n",
+       "  if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
+       "    embed_document(root);\n",
+       "  } else {\n",
+       "    var attempts = 0;\n",
+       "    var timer = setInterval(function(root) {\n",
+       "      if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
+       "        clearInterval(timer);\n",
+       "        embed_document(root);\n",
+       "      } else if (document.readyState == \"complete\") {\n",
+       "        attempts++;\n",
+       "        if (attempts > 200) {\n",
+       "          clearInterval(timer);\n",
+       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+       "        }\n",
+       "      }\n",
+       "    }, 25, root)\n",
+       "  }\n",
+       "})(window);</script>"
+      ],
+      "text/plain": [
+       "DiscreteSlider(formatter='%d', name='Discrete Slider', options=[2, 4, 8, 16, ...], value=32)"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {
+      "application/vnd.holoviews_exec.v0+json": {
+       "id": "1225"
+      }
+     },
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pn.widgets.DiscreteSlider(name='Discrete Slider', options=[2, 4, 8, 16, 32, 64, 128], value=32, tooltips=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "acf267b1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class mem: \n",
+    "    def __init__(self):\n",
+    "        self.button = pn.widgets.Button(name='Click me', button_type='primary')\n",
+    "        self.text = pn.widgets.TextInput(value='Ready')\n",
+    "        self.button.on_click(self.b)\n",
+    "        \n",
+    "    def b(self, event):\n",
+    "        self.text.value = 'Clicked {0} times'.format(self.button.clicks)\n",
+    "    \n",
+    "    def view(self):\n",
+    "        return pn.Row(self.button, self.text)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "78d9264c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cosita = mem()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "9e226490",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {},
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.holoviews_exec.v0+json": "",
+      "text/html": [
+       "<div id='1043'>\n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "\n",
+       "  <div class=\"bk-root\" id=\"526c409b-9559-4156-a7d3-fc31f7a1ea1d\" data-root-id=\"1043\"></div>\n",
+       "</div>\n",
+       "<script type=\"application/javascript\">(function(root) {\n",
+       "  function embed_document(root) {\n",
+       "    var docs_json = {\"69e390af-0037-4a09-b949-f3bf89f57f2e\":{\"defs\":[{\"extends\":null,\"module\":null,\"name\":\"ReactiveHTML1\",\"overrides\":[],\"properties\":[]},{\"extends\":null,\"module\":null,\"name\":\"FlexBox1\",\"overrides\":[],\"properties\":[{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_content\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"align_items\"},{\"default\":\"row\",\"kind\":null,\"name\":\"flex_direction\"},{\"default\":\"wrap\",\"kind\":null,\"name\":\"flex_wrap\"},{\"default\":\"flex-start\",\"kind\":null,\"name\":\"justify_content\"}]},{\"extends\":null,\"module\":null,\"name\":\"TemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]},{\"extends\":null,\"module\":null,\"name\":\"MaterialTemplateActions1\",\"overrides\":[],\"properties\":[{\"default\":0,\"kind\":null,\"name\":\"open_modal\"},{\"default\":0,\"kind\":null,\"name\":\"close_modal\"}]}],\"roots\":{\"references\":[{\"attributes\":{\"client_comm_id\":\"c96ddf7a2e4d4f22ae8c6165845feeaa\",\"comm_id\":\"0c3989f5faf44155acbbd5ff05474ebc\",\"plot_id\":\"1043\"},\"id\":\"1046\",\"type\":\"panel.models.comm_manager.CommManager\"},{\"attributes\":{\"margin\":[5,10,5,10],\"max_length\":5000,\"value\":\"Ready\"},\"id\":\"1045\",\"type\":\"TextInput\"},{\"attributes\":{\"children\":[{\"id\":\"1044\"},{\"id\":\"1045\"}],\"margin\":[0,0,0,0],\"name\":\"Row00156\"},\"id\":\"1043\",\"type\":\"Row\"},{\"attributes\":{\"button_type\":\"primary\",\"icon\":null,\"label\":\"Click me\",\"margin\":[5,10,5,10],\"subscribed_events\":[\"button_click\"]},\"id\":\"1044\",\"type\":\"Button\"}],\"root_ids\":[\"1043\",\"1046\"]},\"title\":\"Bokeh Application\",\"version\":\"2.4.2\"}};\n",
+       "    var render_items = [{\"docid\":\"69e390af-0037-4a09-b949-f3bf89f57f2e\",\"root_ids\":[\"1043\"],\"roots\":{\"1043\":\"526c409b-9559-4156-a7d3-fc31f7a1ea1d\"}}];\n",
+       "    root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+       "  }\n",
+       "  if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
+       "    embed_document(root);\n",
+       "  } else {\n",
+       "    var attempts = 0;\n",
+       "    var timer = setInterval(function(root) {\n",
+       "      if (root.Bokeh !== undefined && root.Bokeh.Panel !== undefined) {\n",
+       "        clearInterval(timer);\n",
+       "        embed_document(root);\n",
+       "      } else if (document.readyState == \"complete\") {\n",
+       "        attempts++;\n",
+       "        if (attempts > 200) {\n",
+       "          clearInterval(timer);\n",
+       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+       "        }\n",
+       "      }\n",
+       "    }, 25, root)\n",
+       "  }\n",
+       "})(window);</script>"
+      ],
+      "text/plain": [
+       "Row\n",
+       "    [0] Button(button_type='primary', name='Click me')\n",
+       "    [1] TextInput(value='Ready')"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {
+      "application/vnd.holoviews_exec.v0+json": {
+       "id": "1043"
+      }
+     },
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cosita.view()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "d1dbb192",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "10"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cosita.button.clicks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "b248cf01-fa47-466f-8f45-4b43cac3aa94",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {},
+     "metadata": {},
+     "output_type": "display_data"
     },
     {
      "data": {
@@ -2046,32 +2109,1044 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 28,
+   "id": "26ee5d57",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list(range(1,a.dataset.shape[1]))[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
    "id": "bc36256d-2cfd-48b6-a47c-2ffc1d9b1a1f",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['feature_performance.ipynb',\n",
-       " 'Untitled.ipynb',\n",
-       " 'logs.log',\n",
-       " '.ipynb_checkpoints']"
+       "[0.01,\n",
+       " 0.02,\n",
+       " 0.03,\n",
+       " 0.04,\n",
+       " 0.05,\n",
+       " 0.06,\n",
+       " 0.07,\n",
+       " 0.08,\n",
+       " 0.09,\n",
+       " 0.1,\n",
+       " 0.11,\n",
+       " 0.12,\n",
+       " 0.13,\n",
+       " 0.14,\n",
+       " 0.15,\n",
+       " 0.16,\n",
+       " 0.17,\n",
+       " 0.18,\n",
+       " 0.19,\n",
+       " 0.2,\n",
+       " 0.21,\n",
+       " 0.22,\n",
+       " 0.23,\n",
+       " 0.24,\n",
+       " 0.25,\n",
+       " 0.26,\n",
+       " 0.27,\n",
+       " 0.28,\n",
+       " 0.29,\n",
+       " 0.3,\n",
+       " 0.31,\n",
+       " 0.32,\n",
+       " 0.33,\n",
+       " 0.34,\n",
+       " 0.35000000000000003,\n",
+       " 0.36,\n",
+       " 0.37,\n",
+       " 0.38,\n",
+       " 0.39,\n",
+       " 0.4,\n",
+       " 0.41000000000000003,\n",
+       " 0.42,\n",
+       " 0.43,\n",
+       " 0.44,\n",
+       " 0.45,\n",
+       " 0.46,\n",
+       " 0.47000000000000003,\n",
+       " 0.48,\n",
+       " 0.49,\n",
+       " 0.5,\n",
+       " 0.51,\n",
+       " 0.52,\n",
+       " 0.53,\n",
+       " 0.54,\n",
+       " 0.55,\n",
+       " 0.56,\n",
+       " 0.5700000000000001,\n",
+       " 0.58,\n",
+       " 0.59,\n",
+       " 0.6,\n",
+       " 0.61,\n",
+       " 0.62,\n",
+       " 0.63,\n",
+       " 0.64,\n",
+       " 0.65,\n",
+       " 0.66,\n",
+       " 0.67,\n",
+       " 0.68,\n",
+       " 0.6900000000000001,\n",
+       " 0.7000000000000001,\n",
+       " 0.71,\n",
+       " 0.72,\n",
+       " 0.73,\n",
+       " 0.74,\n",
+       " 0.75,\n",
+       " 0.76,\n",
+       " 0.77,\n",
+       " 0.78,\n",
+       " 0.79,\n",
+       " 0.8,\n",
+       " 0.81,\n",
+       " 0.8200000000000001,\n",
+       " 0.8300000000000001,\n",
+       " 0.84,\n",
+       " 0.85,\n",
+       " 0.86,\n",
+       " 0.87,\n",
+       " 0.88,\n",
+       " 0.89,\n",
+       " 0.9,\n",
+       " 0.91,\n",
+       " 0.92,\n",
+       " 0.93,\n",
+       " 0.9400000000000001,\n",
+       " 0.9500000000000001,\n",
+       " 0.96,\n",
+       " 0.97,\n",
+       " 0.98,\n",
+       " 0.99,\n",
+       " 1,\n",
+       " 2,\n",
+       " 3,\n",
+       " 4,\n",
+       " 5,\n",
+       " 6,\n",
+       " 7,\n",
+       " 8,\n",
+       " 9,\n",
+       " 10,\n",
+       " 11,\n",
+       " 12,\n",
+       " 13,\n",
+       " 14,\n",
+       " 15,\n",
+       " 16,\n",
+       " 17,\n",
+       " 18,\n",
+       " 19,\n",
+       " 20,\n",
+       " 21,\n",
+       " 22,\n",
+       " 23,\n",
+       " 24,\n",
+       " 25,\n",
+       " 26,\n",
+       " 27,\n",
+       " 28,\n",
+       " 29,\n",
+       " 30,\n",
+       " 31,\n",
+       " 32,\n",
+       " 33,\n",
+       " 34,\n",
+       " 35,\n",
+       " 36,\n",
+       " 37,\n",
+       " 38,\n",
+       " 39,\n",
+       " 40,\n",
+       " 41,\n",
+       " 42,\n",
+       " 43,\n",
+       " 44,\n",
+       " 45,\n",
+       " 46,\n",
+       " 47,\n",
+       " 48,\n",
+       " 49,\n",
+       " 50,\n",
+       " 51,\n",
+       " 52,\n",
+       " 53,\n",
+       " 54,\n",
+       " 55,\n",
+       " 56,\n",
+       " 57,\n",
+       " 58,\n",
+       " 59,\n",
+       " 60,\n",
+       " 61,\n",
+       " 62,\n",
+       " 63,\n",
+       " 64,\n",
+       " 65,\n",
+       " 66,\n",
+       " 67,\n",
+       " 68,\n",
+       " 69,\n",
+       " 70,\n",
+       " 71,\n",
+       " 72,\n",
+       " 73,\n",
+       " 74,\n",
+       " 75,\n",
+       " 76,\n",
+       " 77,\n",
+       " 78,\n",
+       " 79,\n",
+       " 80,\n",
+       " 81,\n",
+       " 82,\n",
+       " 83,\n",
+       " 84,\n",
+       " 85,\n",
+       " 86,\n",
+       " 87,\n",
+       " 88,\n",
+       " 89,\n",
+       " 90,\n",
+       " 91,\n",
+       " 92,\n",
+       " 93,\n",
+       " 94,\n",
+       " 95,\n",
+       " 96,\n",
+       " 97,\n",
+       " 98,\n",
+       " 99,\n",
+       " 100,\n",
+       " 101,\n",
+       " 102,\n",
+       " 103,\n",
+       " 104,\n",
+       " 105,\n",
+       " 106,\n",
+       " 107,\n",
+       " 108,\n",
+       " 109,\n",
+       " 110,\n",
+       " 111,\n",
+       " 112,\n",
+       " 113,\n",
+       " 114,\n",
+       " 115,\n",
+       " 116,\n",
+       " 117,\n",
+       " 118,\n",
+       " 119,\n",
+       " 120,\n",
+       " 121,\n",
+       " 122,\n",
+       " 123,\n",
+       " 124,\n",
+       " 125,\n",
+       " 126,\n",
+       " 127,\n",
+       " 128,\n",
+       " 129,\n",
+       " 130,\n",
+       " 131,\n",
+       " 132,\n",
+       " 133,\n",
+       " 134,\n",
+       " 135,\n",
+       " 136,\n",
+       " 137,\n",
+       " 138,\n",
+       " 139,\n",
+       " 140,\n",
+       " 141,\n",
+       " 142,\n",
+       " 143,\n",
+       " 144,\n",
+       " 145,\n",
+       " 146,\n",
+       " 147,\n",
+       " 148,\n",
+       " 149,\n",
+       " 150,\n",
+       " 151,\n",
+       " 152,\n",
+       " 153,\n",
+       " 154,\n",
+       " 155,\n",
+       " 156,\n",
+       " 157,\n",
+       " 158,\n",
+       " 159,\n",
+       " 160,\n",
+       " 161,\n",
+       " 162,\n",
+       " 163,\n",
+       " 164,\n",
+       " 165,\n",
+       " 166,\n",
+       " 167,\n",
+       " 168,\n",
+       " 169,\n",
+       " 170,\n",
+       " 171,\n",
+       " 172,\n",
+       " 173,\n",
+       " 174,\n",
+       " 175,\n",
+       " 176,\n",
+       " 177,\n",
+       " 178,\n",
+       " 179,\n",
+       " 180,\n",
+       " 181,\n",
+       " 182,\n",
+       " 183,\n",
+       " 184,\n",
+       " 185,\n",
+       " 186,\n",
+       " 187,\n",
+       " 188,\n",
+       " 189,\n",
+       " 190,\n",
+       " 191,\n",
+       " 192,\n",
+       " 193,\n",
+       " 194,\n",
+       " 195,\n",
+       " 196,\n",
+       " 197,\n",
+       " 198,\n",
+       " 199,\n",
+       " 200,\n",
+       " 201,\n",
+       " 202,\n",
+       " 203,\n",
+       " 204,\n",
+       " 205,\n",
+       " 206,\n",
+       " 207,\n",
+       " 208,\n",
+       " 209,\n",
+       " 210,\n",
+       " 211,\n",
+       " 212,\n",
+       " 213,\n",
+       " 214,\n",
+       " 215,\n",
+       " 216,\n",
+       " 217,\n",
+       " 218,\n",
+       " 219,\n",
+       " 220,\n",
+       " 221,\n",
+       " 222,\n",
+       " 223,\n",
+       " 224,\n",
+       " 225,\n",
+       " 226,\n",
+       " 227,\n",
+       " 228,\n",
+       " 229,\n",
+       " 230,\n",
+       " 231,\n",
+       " 232,\n",
+       " 233,\n",
+       " 234,\n",
+       " 235,\n",
+       " 236,\n",
+       " 237,\n",
+       " 238,\n",
+       " 239,\n",
+       " 240,\n",
+       " 241,\n",
+       " 242,\n",
+       " 243,\n",
+       " 244,\n",
+       " 245,\n",
+       " 246,\n",
+       " 247,\n",
+       " 248,\n",
+       " 249,\n",
+       " 250,\n",
+       " 251,\n",
+       " 252,\n",
+       " 253,\n",
+       " 254,\n",
+       " 255,\n",
+       " 256,\n",
+       " 257,\n",
+       " 258,\n",
+       " 259,\n",
+       " 260,\n",
+       " 261,\n",
+       " 262,\n",
+       " 263,\n",
+       " 264,\n",
+       " 265,\n",
+       " 266,\n",
+       " 267,\n",
+       " 268,\n",
+       " 269,\n",
+       " 270,\n",
+       " 271,\n",
+       " 272,\n",
+       " 273,\n",
+       " 274,\n",
+       " 275,\n",
+       " 276,\n",
+       " 277,\n",
+       " 278,\n",
+       " 279,\n",
+       " 280,\n",
+       " 281,\n",
+       " 282,\n",
+       " 283,\n",
+       " 284,\n",
+       " 285,\n",
+       " 286,\n",
+       " 287,\n",
+       " 288,\n",
+       " 289,\n",
+       " 290,\n",
+       " 291,\n",
+       " 292,\n",
+       " 293,\n",
+       " 294,\n",
+       " 295,\n",
+       " 296,\n",
+       " 297,\n",
+       " 298,\n",
+       " 299,\n",
+       " 300,\n",
+       " 301,\n",
+       " 302,\n",
+       " 303,\n",
+       " 304,\n",
+       " 305,\n",
+       " 306,\n",
+       " 307,\n",
+       " 308,\n",
+       " 309,\n",
+       " 310,\n",
+       " 311,\n",
+       " 312,\n",
+       " 313,\n",
+       " 314,\n",
+       " 315,\n",
+       " 316,\n",
+       " 317,\n",
+       " 318,\n",
+       " 319,\n",
+       " 320,\n",
+       " 321,\n",
+       " 322,\n",
+       " 323,\n",
+       " 324,\n",
+       " 325,\n",
+       " 326,\n",
+       " 327,\n",
+       " 328,\n",
+       " 329,\n",
+       " 330,\n",
+       " 331,\n",
+       " 332,\n",
+       " 333,\n",
+       " 334,\n",
+       " 335,\n",
+       " 336,\n",
+       " 337,\n",
+       " 338,\n",
+       " 339,\n",
+       " 340,\n",
+       " 341,\n",
+       " 342,\n",
+       " 343,\n",
+       " 344,\n",
+       " 345,\n",
+       " 346,\n",
+       " 347,\n",
+       " 348,\n",
+       " 349,\n",
+       " 350,\n",
+       " 351,\n",
+       " 352,\n",
+       " 353,\n",
+       " 354,\n",
+       " 355,\n",
+       " 356,\n",
+       " 357,\n",
+       " 358,\n",
+       " 359,\n",
+       " 360,\n",
+       " 361,\n",
+       " 362,\n",
+       " 363,\n",
+       " 364,\n",
+       " 365,\n",
+       " 366,\n",
+       " 367,\n",
+       " 368,\n",
+       " 369,\n",
+       " 370,\n",
+       " 371,\n",
+       " 372,\n",
+       " 373,\n",
+       " 374,\n",
+       " 375,\n",
+       " 376,\n",
+       " 377,\n",
+       " 378,\n",
+       " 379,\n",
+       " 380,\n",
+       " 381,\n",
+       " 382,\n",
+       " 383,\n",
+       " 384,\n",
+       " 385,\n",
+       " 386,\n",
+       " 387,\n",
+       " 388,\n",
+       " 389,\n",
+       " 390,\n",
+       " 391,\n",
+       " 392,\n",
+       " 393,\n",
+       " 394,\n",
+       " 395,\n",
+       " 396,\n",
+       " 397,\n",
+       " 398,\n",
+       " 399,\n",
+       " 400,\n",
+       " 401,\n",
+       " 402,\n",
+       " 403,\n",
+       " 404,\n",
+       " 405,\n",
+       " 406,\n",
+       " 407,\n",
+       " 408,\n",
+       " 409,\n",
+       " 410,\n",
+       " 411,\n",
+       " 412,\n",
+       " 413,\n",
+       " 414,\n",
+       " 415,\n",
+       " 416,\n",
+       " 417,\n",
+       " 418,\n",
+       " 419,\n",
+       " 420,\n",
+       " 421,\n",
+       " 422,\n",
+       " 423,\n",
+       " 424,\n",
+       " 425,\n",
+       " 426,\n",
+       " 427,\n",
+       " 428,\n",
+       " 429,\n",
+       " 430,\n",
+       " 431,\n",
+       " 432,\n",
+       " 433,\n",
+       " 434,\n",
+       " 435,\n",
+       " 436,\n",
+       " 437,\n",
+       " 438,\n",
+       " 439,\n",
+       " 440,\n",
+       " 441,\n",
+       " 442,\n",
+       " 443,\n",
+       " 444,\n",
+       " 445,\n",
+       " 446,\n",
+       " 447,\n",
+       " 448,\n",
+       " 449,\n",
+       " 450,\n",
+       " 451,\n",
+       " 452,\n",
+       " 453,\n",
+       " 454,\n",
+       " 455,\n",
+       " 456,\n",
+       " 457,\n",
+       " 458,\n",
+       " 459,\n",
+       " 460,\n",
+       " 461,\n",
+       " 462,\n",
+       " 463,\n",
+       " 464,\n",
+       " 465,\n",
+       " 466,\n",
+       " 467,\n",
+       " 468,\n",
+       " 469,\n",
+       " 470,\n",
+       " 471,\n",
+       " 472,\n",
+       " 473,\n",
+       " 474,\n",
+       " 475,\n",
+       " 476,\n",
+       " 477,\n",
+       " 478,\n",
+       " 479,\n",
+       " 480,\n",
+       " 481,\n",
+       " 482,\n",
+       " 483,\n",
+       " 484,\n",
+       " 485,\n",
+       " 486,\n",
+       " 487,\n",
+       " 488,\n",
+       " 489,\n",
+       " 490,\n",
+       " 491,\n",
+       " 492,\n",
+       " 493,\n",
+       " 494,\n",
+       " 495,\n",
+       " 496,\n",
+       " 497,\n",
+       " 498,\n",
+       " 499,\n",
+       " 500,\n",
+       " 501,\n",
+       " 502,\n",
+       " 503,\n",
+       " 504,\n",
+       " 505,\n",
+       " 506,\n",
+       " 507,\n",
+       " 508,\n",
+       " 509,\n",
+       " 510,\n",
+       " 511,\n",
+       " 512,\n",
+       " 513,\n",
+       " 514,\n",
+       " 515,\n",
+       " 516,\n",
+       " 517,\n",
+       " 518,\n",
+       " 519,\n",
+       " 520,\n",
+       " 521,\n",
+       " 522,\n",
+       " 523,\n",
+       " 524,\n",
+       " 525,\n",
+       " 526,\n",
+       " 527,\n",
+       " 528,\n",
+       " 529,\n",
+       " 530,\n",
+       " 531,\n",
+       " 532,\n",
+       " 533,\n",
+       " 534,\n",
+       " 535,\n",
+       " 536,\n",
+       " 537,\n",
+       " 538,\n",
+       " 539,\n",
+       " 540,\n",
+       " 541,\n",
+       " 542,\n",
+       " 543,\n",
+       " 544,\n",
+       " 545,\n",
+       " 546,\n",
+       " 547,\n",
+       " 548,\n",
+       " 549,\n",
+       " 550,\n",
+       " 551,\n",
+       " 552,\n",
+       " 553,\n",
+       " 554,\n",
+       " 555,\n",
+       " 556,\n",
+       " 557,\n",
+       " 558,\n",
+       " 559,\n",
+       " 560,\n",
+       " 561,\n",
+       " 562,\n",
+       " 563,\n",
+       " 564,\n",
+       " 565,\n",
+       " 566,\n",
+       " 567,\n",
+       " 568,\n",
+       " 569,\n",
+       " 570,\n",
+       " 571,\n",
+       " 572,\n",
+       " 573,\n",
+       " 574,\n",
+       " 575,\n",
+       " 576,\n",
+       " 577,\n",
+       " 578,\n",
+       " 579,\n",
+       " 580,\n",
+       " 581,\n",
+       " 582,\n",
+       " 583,\n",
+       " 584,\n",
+       " 585,\n",
+       " 586,\n",
+       " 587,\n",
+       " 588,\n",
+       " 589,\n",
+       " 590,\n",
+       " 591,\n",
+       " 592,\n",
+       " 593,\n",
+       " 594,\n",
+       " 595,\n",
+       " 596,\n",
+       " 597,\n",
+       " 598,\n",
+       " 599,\n",
+       " 600,\n",
+       " 601,\n",
+       " 602,\n",
+       " 603,\n",
+       " 604,\n",
+       " 605,\n",
+       " 606,\n",
+       " 607,\n",
+       " 608,\n",
+       " 609,\n",
+       " 610,\n",
+       " 611,\n",
+       " 612,\n",
+       " 613,\n",
+       " 614,\n",
+       " 615,\n",
+       " 616,\n",
+       " 617,\n",
+       " 618,\n",
+       " 619,\n",
+       " 620,\n",
+       " 621,\n",
+       " 622,\n",
+       " 623,\n",
+       " 624,\n",
+       " 625,\n",
+       " 626,\n",
+       " 627,\n",
+       " 628,\n",
+       " 629,\n",
+       " 630,\n",
+       " 631,\n",
+       " 632,\n",
+       " 633,\n",
+       " 634,\n",
+       " 635,\n",
+       " 636,\n",
+       " 637,\n",
+       " 638,\n",
+       " 639,\n",
+       " 640,\n",
+       " 641,\n",
+       " 642,\n",
+       " 643,\n",
+       " 644,\n",
+       " 645,\n",
+       " 646,\n",
+       " 647,\n",
+       " 648,\n",
+       " 649,\n",
+       " 650,\n",
+       " 651,\n",
+       " 652,\n",
+       " 653,\n",
+       " 654,\n",
+       " 655,\n",
+       " 656,\n",
+       " 657,\n",
+       " 658,\n",
+       " 659,\n",
+       " 660,\n",
+       " 661,\n",
+       " 662,\n",
+       " 663,\n",
+       " 664,\n",
+       " 665,\n",
+       " 666,\n",
+       " 667,\n",
+       " 668,\n",
+       " 669,\n",
+       " 670,\n",
+       " 671,\n",
+       " 672,\n",
+       " 673,\n",
+       " 674,\n",
+       " 675,\n",
+       " 676,\n",
+       " 677,\n",
+       " 678,\n",
+       " 679,\n",
+       " 680,\n",
+       " 681,\n",
+       " 682,\n",
+       " 683,\n",
+       " 684,\n",
+       " 685,\n",
+       " 686,\n",
+       " 687,\n",
+       " 688,\n",
+       " 689,\n",
+       " 690,\n",
+       " 691,\n",
+       " 692,\n",
+       " 693,\n",
+       " 694,\n",
+       " 695,\n",
+       " 696,\n",
+       " 697,\n",
+       " 698,\n",
+       " 699,\n",
+       " 700,\n",
+       " 701,\n",
+       " 702,\n",
+       " 703,\n",
+       " 704,\n",
+       " 705,\n",
+       " 706,\n",
+       " 707,\n",
+       " 708,\n",
+       " 709,\n",
+       " 710,\n",
+       " 711,\n",
+       " 712,\n",
+       " 713,\n",
+       " 714,\n",
+       " 715,\n",
+       " 716,\n",
+       " 717,\n",
+       " 718,\n",
+       " 719,\n",
+       " 720,\n",
+       " 721,\n",
+       " 722,\n",
+       " 723,\n",
+       " 724,\n",
+       " 725,\n",
+       " 726,\n",
+       " 727,\n",
+       " 728,\n",
+       " 729,\n",
+       " 730,\n",
+       " 731,\n",
+       " 732,\n",
+       " 733,\n",
+       " 734,\n",
+       " 735,\n",
+       " 736,\n",
+       " 737,\n",
+       " 738,\n",
+       " 739,\n",
+       " 740,\n",
+       " 741,\n",
+       " 742,\n",
+       " 743,\n",
+       " 744,\n",
+       " 745,\n",
+       " 746,\n",
+       " 747,\n",
+       " 748,\n",
+       " 749,\n",
+       " 750,\n",
+       " 751,\n",
+       " 752,\n",
+       " 753,\n",
+       " 754,\n",
+       " 755,\n",
+       " 756,\n",
+       " 757,\n",
+       " 758,\n",
+       " 759,\n",
+       " 760,\n",
+       " 761,\n",
+       " 762,\n",
+       " 763,\n",
+       " 764,\n",
+       " 765,\n",
+       " 766,\n",
+       " 767,\n",
+       " 768,\n",
+       " 769,\n",
+       " 770,\n",
+       " 771,\n",
+       " 772,\n",
+       " 773,\n",
+       " 774,\n",
+       " 775,\n",
+       " 776,\n",
+       " 777,\n",
+       " 778,\n",
+       " 779,\n",
+       " 780,\n",
+       " 781,\n",
+       " 782,\n",
+       " 783,\n",
+       " 784,\n",
+       " 785,\n",
+       " 786,\n",
+       " 787,\n",
+       " 788,\n",
+       " 789,\n",
+       " 790,\n",
+       " 791,\n",
+       " 792,\n",
+       " 793,\n",
+       " 794,\n",
+       " 795,\n",
+       " 796,\n",
+       " 797,\n",
+       " 798,\n",
+       " 799,\n",
+       " 800,\n",
+       " 801,\n",
+       " 802,\n",
+       " 803,\n",
+       " 804,\n",
+       " 805,\n",
+       " 806,\n",
+       " 807,\n",
+       " 808,\n",
+       " 809,\n",
+       " 810,\n",
+       " 811,\n",
+       " 812,\n",
+       " 813,\n",
+       " 814,\n",
+       " 815,\n",
+       " 816,\n",
+       " 817,\n",
+       " 818,\n",
+       " 819,\n",
+       " 820,\n",
+       " 821,\n",
+       " 822,\n",
+       " 823,\n",
+       " 824,\n",
+       " 825,\n",
+       " 826,\n",
+       " 827,\n",
+       " 828,\n",
+       " 829,\n",
+       " 830,\n",
+       " 831,\n",
+       " 832,\n",
+       " 833,\n",
+       " 834,\n",
+       " 835,\n",
+       " 836,\n",
+       " 837,\n",
+       " 838,\n",
+       " 839,\n",
+       " 840,\n",
+       " 841,\n",
+       " 842,\n",
+       " 843,\n",
+       " 844,\n",
+       " 845,\n",
+       " 846,\n",
+       " 847,\n",
+       " 848,\n",
+       " 849,\n",
+       " 850,\n",
+       " 851,\n",
+       " 852,\n",
+       " 853,\n",
+       " 854,\n",
+       " 855,\n",
+       " 856,\n",
+       " 857,\n",
+       " 858,\n",
+       " 859,\n",
+       " 860,\n",
+       " 861,\n",
+       " 862,\n",
+       " 863,\n",
+       " 864,\n",
+       " 865,\n",
+       " 866,\n",
+       " 867,\n",
+       " 868,\n",
+       " 869,\n",
+       " 870,\n",
+       " 871,\n",
+       " 872,\n",
+       " 873,\n",
+       " 874,\n",
+       " 875,\n",
+       " 876,\n",
+       " 877,\n",
+       " 878,\n",
+       " 879,\n",
+       " 880,\n",
+       " 881,\n",
+       " 882,\n",
+       " 883,\n",
+       " 884,\n",
+       " 885,\n",
+       " 886,\n",
+       " 887,\n",
+       " 888,\n",
+       " 889,\n",
+       " 890,\n",
+       " 891,\n",
+       " 892,\n",
+       " 893,\n",
+       " 894,\n",
+       " 895,\n",
+       " 896,\n",
+       " 897,\n",
+       " 898,\n",
+       " 899,\n",
+       " 900,\n",
+       " 901,\n",
+       " ...]"
       ]
      },
-     "execution_count": 35,
+     "execution_count": 30,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "from pathlib import Path\n",
-    "import os \n",
-    "\n",
-    "path = !pwd\n",
-    "path = Path(path[0])\n",
-    "path.parent\n",
-    "os.listdir(path)"
+    "np.linspace(0,1,100,endpoint=False)[1:].tolist() + list(range(1,a.dataset.shape[1]))"
    ]
   },
   {
diff --git a/notebooks/feat_sel_test_j1.ipynb b/notebooks/feat_sel_test_j1.ipynb
new file mode 100644
index 0000000..b9ed794
--- /dev/null
+++ b/notebooks/feat_sel_test_j1.ipynb
@@ -0,0 +1,3908 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from itertools import product\n",
+    "from typing import Dict, List, Optional, Union\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import param\n",
+    "from pycaret.classification import (\n",
+    "    compare_models,\n",
+    "    create_model,\n",
+    "    get_config,\n",
+    "    predict_model,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    ")\n",
+    "from pycaret.utils import check_metric\n",
+    "\n",
+    "from feature_selection.run_pycaret_setup import run_pycaret_setup\n",
+    "\n",
+    "\n",
+    "class FeatureSelection(param.Parameterized):\n",
+    "\n",
+    "    # Class attributes\n",
+    "    model_class_to_name = {\n",
+    "        \"RidgeClassifier\": \"ridge\",\n",
+    "        \"LogisticRegression\": \"lr\",\n",
+    "        \"LinearDiscriminantAnalysis\": \"lda\",\n",
+    "        \"GradientBoostingClassifier\": \"gbc\",\n",
+    "        \"QuadraticDiscriminantAnalysis\": \"qda\",\n",
+    "        \"LGBMClassifier\": \"lightgbm\",\n",
+    "        \"AdaBoostClassifier\": \"ada\",\n",
+    "        \"RandomForestClassifier\": \"rf\",\n",
+    "        \"ExtraTreesClassifier\": \"et\",\n",
+    "        \"GaussianNB\": \"nb\",\n",
+    "        \"DecisionTreeClassifier\": \"dt\",\n",
+    "        \"KNeighborsClassifier\": \"knn\",\n",
+    "        \"SGDClassifier\": \"svm\",\n",
+    "        \"CatBoostClassifier\": \"catboost\",\n",
+    "        \"SVC\": \"rbfsvm\",\n",
+    "        \"GaussianProcessClassifier\": \"gpc\",\n",
+    "        \"MLPClassifier\": \"mlp\",\n",
+    "        \"XGBClassifier\": \"xgboost\",\n",
+    "    }\n",
+    "\n",
+    "    metrics_list = [\"Accuracy\", \"AUC\", \"Recall\", \"Precision\", \"F1\", \"Kappa\", \"MCC\"]\n",
+    "\n",
+    "    # Private class attributes\n",
+    "    _filter_metric = {\n",
+    "        \"Accuracy\": 0.5,\n",
+    "        \"AUC\": 0.5,\n",
+    "        \"Recall\": 0.6,\n",
+    "        \"Precision\": 0.6,\n",
+    "        \"F1\": 0.6,\n",
+    "        \"Kappa\": 0.1,\n",
+    "        \"MCC\": 0.1,\n",
+    "    }\n",
+    "\n",
+    "    _setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        train_size=0.75,\n",
+    "        # test_data=test_data,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.4,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        ignore_features=None,\n",
+    "        fold_strategy=\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=4,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.4,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=False,\n",
+    "        pca_method=\"kernel\",\n",
+    "        pca_components=30,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.01,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=False,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.01,\n",
+    "        fix_imbalance=False,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "        html=False,\n",
+    "    )\n",
+    "\n",
+    "    _numerics = [\"int16\", \"int32\", \"int64\", \"float16\", \"float32\", \"float64\", \"int\", \"float\"]\n",
+    "\n",
+    "    # Init values\n",
+    "    ## Feature selection parameters\n",
+    "    target = param.String(\"goal_2.5\")\n",
+    "    number_features = param.Number(\n",
+    "        0.5,\n",
+    "        bounds=(0, 1),\n",
+    "        inclusive_bounds=(False, False),\n",
+    "        doc=\"Number of features (percentage) selected each iteration. Only the first nth \"\n",
+    "        \"features will be kept for the next iteration.\",\n",
+    "    )\n",
+    "    target_features = param.Number(\n",
+    "        0.3,\n",
+    "        bounds=(0, None),\n",
+    "        inclusive_bounds=(False, True),\n",
+    "        doc=\"Final total number of features. The goal of the package is to reduce \"\n",
+    "        \"the incoming columns of the dataset to this 'target_features' number.\",\n",
+    "    )\n",
+    "    ## Metric parameters\n",
+    "    filter_metrics = param.Dict(_filter_metric)\n",
+    "    ## Model setup and model optimization parameters\n",
+    "    numerics = param.List(_numerics)\n",
+    "    ignore_features = param.List(default=[], allow_None=True)\n",
+    "    setup_kwargs = param.Dict(_setup_kwargs)\n",
+    "    include = param.List(default=None, item_type=str, allow_None=True)\n",
+    "    exclude = param.List([\"qda\", \"knn\", \"nb\"], item_type=str)\n",
+    "    sort = param.String(\"AUC\")\n",
+    "    number_models = param.Integer(10, bounds=(2, 13))\n",
+    "    top_models = param.List(default=None, allow_None=True)\n",
+    "    optimize = param.Boolean(False)\n",
+    "    opt_list = param.List([\"Accuracy\", \"Precision\", \"Recall\", \"F1\", \"AUC\"], item_type=str)\n",
+    "    ## Class selectors\n",
+    "    dataset = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    dict_models = param.ClassSelector(class_=dict)\n",
+    "    tune_dict_models = param.ClassSelector(class_=dict)\n",
+    "    x_train = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    x_df = param.DataFrame(pd.DataFrame())\n",
+    "    model_df = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    model_tuned_df = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    features_df = param.ClassSelector(class_=pd.DataFrame)\n",
+    "\n",
+    "    def __init__(self, dataset: pd.DataFrame, **kwargs):\n",
+    "        # Copy of the incoming dataset\n",
+    "        dataset = dataset.copy()\n",
+    "        # Compute the upper bound of number_features, target_features, number_models\n",
+    "        total_features = dataset.shape[1]\n",
+    "        self.param.target_features.bounds = (0, total_features)\n",
+    "        if \"include\" in kwargs:\n",
+    "            self.param.number_models.default = len(kwargs[\"include\"])\n",
+    "            self.param.number_models.bounds = (0, len(kwargs[\"include\"]))\n",
+    "        # Call super\n",
+    "        super(FeatureSelection, self).__init__(dataset=dataset, **kwargs)\n",
+    "        # Get the features of the dataframe\n",
+    "        self.feature_list = self.dataset.columns.tolist()\n",
+    "        self.feature_list.remove(self.target)  # target column should not be counted\n",
+    "        # Compute target features\n",
+    "        self.target_features = self.calculate_number_features(\n",
+    "            number_features=self.target_features, features=self.feature_list\n",
+    "        )\n",
+    "        # Get the evaluator and the arguments. Depends on the \"include\" parameter\n",
+    "        self._training_function, self._args = self._decide_model_eval()\n",
+    "        # Get all the columns whose type is numeric\n",
+    "        self.numeric_features = self._compute_numeric_features(df=self.dataset[self.feature_list])\n",
+    "\n",
+    "    def _compute_numeric_features(self, df: pd.DataFrame):\n",
+    "        \"\"\"Return those columns from the given dataset whose data type is numeric.\"\"\"\n",
+    "        return df.select_dtypes(include=self.numerics).columns.tolist()\n",
+    "\n",
+    "    def _decide_model_eval(self):\n",
+    "        \"\"\"\n",
+    "        Define the pycaret model evaluator depending on the number of included models.\n",
+    "\n",
+    "        If the 'include' list parameter equals 1, the method will return\n",
+    "        the 'create_models' pycaret object.\n",
+    "        If 'include' parameter list is greatear than 1, the method will\n",
+    "        return the 'compare_model' pycaret object and its arguments.\n",
+    "        If 'include' parameter equals None, the method will return the\n",
+    "        'compare_models' pycaret object, where all possible models are\n",
+    "        considered for evaluation, except those included within the 'exclude'\n",
+    "        list.\n",
+    "        \"\"\"\n",
+    "        args = {\"n_select\": self.number_models, \"sort\": self.sort, \"verbose\": False}\n",
+    "        training_function = compare_models\n",
+    "        if not self.include:\n",
+    "            args[\"exclude\"] = self.exclude\n",
+    "        elif len(self.include) == 1:\n",
+    "            training_function = lambda *rgs, **kwargs: [create_model(*rgs, **kwargs)]\n",
+    "            args = {\"estimator\": self.include[0], \"verbose\": False}\n",
+    "        else:\n",
+    "            args[\"include\"] = self.include\n",
+    "        return training_function, args\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def calculate_number_features(\n",
+    "        number_features: Union[int, float], features: Union[pd.DataFrame, List]\n",
+    "    ) -> int:\n",
+    "        n_features = (\n",
+    "            int(number_features)\n",
+    "            if (number_features >= 1)\n",
+    "            else int(number_features * len(features))\n",
+    "        )\n",
+    "        return n_features\n",
+    "\n",
+    "    def train_model(self):\n",
+    "        \"\"\"Preprocess the data and select self.number_models top models.\"\"\"\n",
+    "        # Selected dataset\n",
+    "        selected_cols = self.feature_list + [self.target]\n",
+    "        train_data = self.dataset[selected_cols] if self.x_df.empty else self.x_df[selected_cols]\n",
+    "        # Numeric features\n",
+    "        self.setup_kwargs[\"numeric_features\"] = [\n",
+    "            c for c in self.numeric_features if c in self.feature_list\n",
+    "        ]\n",
+    "        # Ignore features\n",
+    "        self.setup_kwargs[\"ignore_features\"] = [\n",
+    "            c for c in self.ignore_features if c in self.feature_list\n",
+    "        ]\n",
+    "        # Initialize pycaret setup\n",
+    "        setup(data=train_data, target=self.target, **self.setup_kwargs)\n",
+    "        # Get train dataset and preprocessed dataframe\n",
+    "        self.x_train = get_config(\"X_train\")\n",
+    "        if self.x_df.empty:  # TODO change x_df by dataset and add flag?\n",
+    "            self.x_df = pd.concat([get_config(\"X\"), get_config(\"y\")], axis=1)\n",
+    "            self.setup_kwargs[\"preprocess\"] = False  # Turn off preprocessing\n",
+    "        # Compare models\n",
+    "        self.top_models = self._training_function(**self._args)\n",
+    "\n",
+    "    def create_dict_models(self):\n",
+    "        \"\"\"Create a dictionary whose values are pycaret standard models.\"\"\"\n",
+    "        self.dict_models = {\n",
+    "            str(top_model).split(\"(\")[0]: top_model for top_model in self.top_models\n",
+    "        }\n",
+    "        # Remove bad catboost key\n",
+    "        oldkey = [key for key in self.dict_models.keys() if key.startswith(\"<catboost\")]\n",
+    "        if oldkey:\n",
+    "            self.dict_models[\"CatBoostClassifier\"] = self.dict_models.pop(oldkey[0])\n",
+    "        # Remap\n",
+    "        self.dict_models = {\n",
+    "            self.model_class_to_name[key]: self.dict_models[key] for key in self.dict_models.keys()\n",
+    "        }\n",
+    "\n",
+    "    def create_dict_tuned_models(self):\n",
+    "        \"\"\"Create a dictionary whose keys and values are pycaret tuned models.\"\"\"\n",
+    "        self.tune_dict_models = {}\n",
+    "        for (model_str, py_model), optimize in product(self.dict_models.items(), self.opt_list):\n",
+    "            self.tune_dict_models[f\"{model_str}_tune_{optimize}\"] = tune_model(\n",
+    "                py_model,\n",
+    "                optimize=optimize,\n",
+    "                verbose=False,\n",
+    "                n_iter=30,\n",
+    "                choose_better=True,\n",
+    "            )\n",
+    "\n",
+    "    def get_metrics_df(self, test_predicted, model, dataframe):\n",
+    "        \"\"\"Compute different metric values for the given model.\"\"\"\n",
+    "        value_dct = dict()\n",
+    "        for metric in self.metrics_list:\n",
+    "            try:\n",
+    "                value_dct[metric] = check_metric(\n",
+    "                    actual=test_predicted[self.target],\n",
+    "                    prediction=test_predicted[\"Label\"],\n",
+    "                    metric=metric,\n",
+    "                )\n",
+    "            except AttributeError:\n",
+    "                value_dct[metric] = np.nan\n",
+    "        for key, val in value_dct.items():\n",
+    "            dataframe.loc[model, key] = val\n",
+    "        return dataframe\n",
+    "\n",
+    "    def remove_bad_models(self, dataframe: pd.DataFrame):\n",
+    "        \"\"\"Filter and remove the models whose metrics do not satisfy the given conditions.\"\"\"\n",
+    "        remove_dict = dict()\n",
+    "        models = dataframe.index.tolist()\n",
+    "        for model, (metric, cond) in product(models, self.filter_metrics.items()):\n",
+    "            if dataframe.loc[model, metric] < cond:\n",
+    "                remove_dict[model] = metric\n",
+    "        remove_models = list(set(remove_dict.keys()))\n",
+    "        dataframe.drop(labels=remove_models, axis=\"index\", inplace=True)\n",
+    "        return dataframe\n",
+    "\n",
+    "    def filter_best_features(self, key_model: str, models_dict: Dict):\n",
+    "        \"\"\"Compute the most relevant features used by the given model.\"\"\"\n",
+    "        py_model = models_dict[key_model]\n",
+    "        cond = any([key_model.startswith(name) for name in [\"lr\", \"lda\", \"ridge\", \"svm\"]])\n",
+    "        score_metric = abs(py_model.coef_[0]) if cond else py_model.feature_importances_\n",
+    "        metrics_dict = {\n",
+    "            \"model_id\": key_model,\n",
+    "            \"model\": key_model.split(\"_\")[0],\n",
+    "            \"feature\": self.x_train.columns,\n",
+    "            \"score\": score_metric,\n",
+    "        }\n",
+    "        df = pd.DataFrame(metrics_dict).sort_values(by=\"score\", ascending=False)\n",
+    "        top_n_features = self.calculate_number_features(\n",
+    "            number_features=self.number_features,\n",
+    "            features=df,\n",
+    "        )\n",
+    "        return df.iloc[:top_n_features]\n",
+    "\n",
+    "    def extract_features(self, dataframe: pd.DataFrame, dict_models: Dict):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by the given model.\"\"\"\n",
+    "        models = dataframe.index.tolist()\n",
+    "        for model in models:  # model extracted from dataframe\n",
+    "            # Check\n",
+    "            if (\n",
+    "                model not in dict_models.keys()\n",
+    "            ):  # check no errors have been produced during operations\n",
+    "                raise KeyError(f\"The selected model: {model} is not listed in dict_models.keys()\")\n",
+    "            df_conc = self.filter_best_features(key_model=model, models_dict=dict_models)\n",
+    "            self.features_df = pd.concat([self.features_df, df_conc])\n",
+    "\n",
+    "    def compute_metrics_df(self):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by the standard models.\"\"\"\n",
+    "        self.model_df = pd.DataFrame(\n",
+    "            data=[], index=self.dict_models.keys(), columns=self.metrics_list\n",
+    "        )\n",
+    "        for model, py_model in self.dict_models.items():\n",
+    "            predict = predict_model(py_model)\n",
+    "            self.model_df = self.get_metrics_df(\n",
+    "                test_predicted=predict,\n",
+    "                model=model,\n",
+    "                dataframe=self.model_df,\n",
+    "            )\n",
+    "        self.model_df = self.remove_bad_models(dataframe=self.model_df)\n",
+    "        self.extract_features(dataframe=self.model_df, dict_models=self.dict_models)\n",
+    "\n",
+    "    def filter_tuned_duplicate(self):\n",
+    "        \"\"\"Remove tuned models with identical metrics.\"\"\"\n",
+    "\n",
+    "        def drop(x):\n",
+    "            x.drop_duplicates(subset=self.metrics_list, keep=\"first\", inplace=True)\n",
+    "            return x\n",
+    "\n",
+    "        df = self.model_tuned_df.groupby(\"model\").apply(drop)\n",
+    "        df = (\n",
+    "            df.drop(columns=\"model\").reset_index(level=\"model\")\n",
+    "            if isinstance(df.index, pd.MultiIndex)\n",
+    "            else df\n",
+    "        )\n",
+    "        return df\n",
+    "\n",
+    "    def tune_df(self):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by tuned models.\"\"\"\n",
+    "        # Tune dataframe\n",
+    "        self.model_tuned_df = pd.DataFrame(\n",
+    "            data=[],\n",
+    "            index=self.tune_dict_models.keys(),\n",
+    "            columns=[\"model\"] + self.metrics_list,\n",
+    "        )\n",
+    "        # Model entry\n",
+    "        for prim_model in self.dict_models.keys():\n",
+    "            ix = [ind.startswith(prim_model) for ind in self.model_tuned_df.index]\n",
+    "            self.model_tuned_df.loc[ix, \"model\"] = prim_model\n",
+    "        # Fill dataframe\n",
+    "        for model, py_model in self.tune_dict_models.items():\n",
+    "            predict = predict_model(py_model)\n",
+    "            self.model_tuned_df = self.get_metrics_df(\n",
+    "                test_predicted=predict, model=model, dataframe=self.model_tuned_df\n",
+    "            )\n",
+    "        # Remove duplicate and filter\n",
+    "        self.model_tuned_df = self.filter_tuned_duplicate()\n",
+    "        self.model_tuned_df = self.remove_bad_models(dataframe=self.model_tuned_df)\n",
+    "        # Get features\n",
+    "        self.extract_features(dataframe=self.model_tuned_df, dict_models=self.tune_dict_models)\n",
+    "\n",
+    "    def run_feature_extraction(self):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by each model.\"\"\"\n",
+    "        # Initialize feature dataframe and train model\n",
+    "        self.features_df = pd.DataFrame(data=[], columns=[\"model_id\", \"model\", \"feature\", \"score\"])\n",
+    "        self.train_model()\n",
+    "        # Run standard models\n",
+    "        self.create_dict_models()\n",
+    "        self.compute_metrics_df()\n",
+    "        # Run tuned models\n",
+    "        if self.optimize:\n",
+    "            self.create_dict_tuned_models()\n",
+    "            if not bool(self.tune_dict_models):\n",
+    "                raise ValueError(\"The tune dictionary is empty!\")\n",
+    "            self.tune_df()\n",
+    "        # Return the list containing the features and their score\n",
+    "        self.features_df.index.name = \"index_rem\"\n",
+    "        return self.features_df.reset_index().drop(columns=\"index_rem\")\n",
+    "\n",
+    "    def remove_zeros(self):\n",
+    "        \"\"\"Remove non-relevant features (those with a zero score).\"\"\"\n",
+    "        ix = self.features_df[\"score\"] <= 0\n",
+    "        self.features_df.drop(index=self.features_df.loc[ix].index, inplace=True)\n",
+    "        self.features_df.reset_index(drop=True, inplace=True)\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def normalize(dataframe):\n",
+    "        \"\"\"Normalize the pycaret score of each feature.\"\"\"\n",
+    "\n",
+    "        def norm(x):\n",
+    "            x[\"normal\"] = x[\"score\"] / x[\"score\"].max()\n",
+    "            return x\n",
+    "\n",
+    "        return (\n",
+    "            dataframe.groupby(\"model_id\")\n",
+    "            .apply(norm)\n",
+    "            .sort_values([\"model_id\", \"normal\"], ascending=[True, False])\n",
+    "        )\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def feature_score(dataframe):\n",
+    "        \"\"\"Assign the score to the selected features.\"\"\"\n",
+    "        group = dataframe.groupby(\"feature\")\n",
+    "        sorted_data = group.agg(\n",
+    "            counts=pd.NamedAgg(column=\"normal\", aggfunc=\"count\"),\n",
+    "            normal_sum=pd.NamedAgg(column=\"normal\", aggfunc=\"sum\"),\n",
+    "        ).sort_values(by=[\"counts\", \"normal_sum\"], ascending=False)\n",
+    "        sorted_data[\"final_score\"] = sorted_data[\"normal_sum\"] / sorted_data[\"counts\"]\n",
+    "        return sorted_data.sort_values(\"final_score\", ascending=False)\n",
+    "\n",
+    "    def create_feature_list(self):\n",
+    "        \"\"\"Run all necessary methods to extract the list of relevant features.\"\"\"\n",
+    "        # Call creation features dataframe\n",
+    "        self.features_df = self.run_feature_extraction()\n",
+    "        # Remove zeros and normalize\n",
+    "        self.remove_zeros()\n",
+    "        self.features_df = self.normalize(dataframe=self.features_df)\n",
+    "        # Get score\n",
+    "        scoreboard = self.feature_score(dataframe=self.features_df)\n",
+    "        top_n_features = self.calculate_number_features(\n",
+    "            number_features=self.number_features,\n",
+    "            features=scoreboard,\n",
+    "        )\n",
+    "        filtered = scoreboard.iloc[:top_n_features]\n",
+    "        self.feature_list = filtered.index.tolist()\n",
+    "\n",
+    "    def repeat_pipeline(self):\n",
+    "        \"\"\"Iterate over the process to create the feature list and repeat it self.repeat times.\"\"\"\n",
+    "        while len(self.feature_list) > self.target_features:\n",
+    "            # Call iteration\n",
+    "            self.create_feature_list()\n",
+    "            if len(self.feature_list) <= 1:\n",
+    "                break\n",
+    "        return self.feature_list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
+    "from ml_bets.constants import FEATURES_PATH\n",
+    "from ml_bets.features.features import Features\n",
+    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
+    "\n",
+    "\n",
+    "from pycaret.utils import check_metric\n",
+    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
+    "    create_model,\n",
+    "    finalize_model,\n",
+    "    optimize_threshold,              \n",
+    "    save_model,\n",
+    "    compare_models, \n",
+    "    evaluate_model,\n",
+    "    get_config,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    "    predict_model,\n",
+    ")\n",
+    "\n",
+    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
+    "from ml_bets.research.datasets import create_dataset\n",
+    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target = \"goals_2.5\"\n",
+    "test_date = \"1-Dec-2021\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ESP1C.xls\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ING1C.xls\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ITA1C.xls\n",
+      "Excel file c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\MEX1C.xls is empty. Skipping\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\FRA1C.xls\n",
+      "Excel file c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\MLS1C.xls is empty. Skipping\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ALE1C.xls\n"
+     ]
+    }
+   ],
+   "source": [
+    "feats = Features()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def patched_create_dataset(\n",
+    "    test_date: str,\n",
+    "    target: str,\n",
+    "    columns=None,\n",
+    "    odds_features: bool = True,\n",
+    "    summary: bool = True,\n",
+    "    odds_rankings: bool = True,\n",
+    "    include_std: bool = True,\n",
+    "    features: Features = None,\n",
+    "    ignore_features=None,\n",
+    "    drop_future_matches: bool = True,\n",
+    "    test_weeks: int = 4,\n",
+    "):\n",
+    "    features = features or Features()\n",
+    "    examples = features.create(\n",
+    "        columns=columns,\n",
+    "        odds_features=odds_features,\n",
+    "        odds_rankings=odds_rankings,\n",
+    "        referee_features=True,\n",
+    "        include_std=include_std,\n",
+    "        summary=summary,\n",
+    "    )\n",
+    "    examples = examples[[x for x in examples.columns if \"possession\" not in x]]\n",
+    "    if ignore_features is not None:\n",
+    "        examples.drop(columns=ignore_features, inplace=True)\n",
+    "    pds = PipelineDatasets(\n",
+    "        examples=examples,\n",
+    "        features=features,\n",
+    "        target=target,\n",
+    "        drop_future_matches=drop_future_matches,\n",
+    "        test_size=test_date,\n",
+    "        test_weeks=test_weeks,\n",
+    "    )\n",
+    "    return pds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = patched_create_dataset(target=target,\n",
+    "                    test_date=test_date,\n",
+    "                    features=feats,\n",
+    "                    odds_features=True,\n",
+    "                    include_std=True,\n",
+    "                    ignore_features=IGNORE_FEATURES+[\"referee\", \"hour_rank\", \"hour_before_16\", \"is_weekend\"],\n",
+    "                    drop_future_matches=False,\n",
+    "                    \n",
+    "                   )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data = ds.train_data.copy()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        #custom_pipeline=loaded,\n",
+    "        train_size=0.75,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "       # normalize_method=\"robust\",\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.8,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        #ignore_features=ignore_features,\n",
+    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=3,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.5,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=False,\n",
+    "        pca_method=\"kernel\",\n",
+    "        pca_components=50,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.05,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=True,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.05,\n",
+    "        fix_imbalance=True,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Feature selection (without triplet)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5158</td>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5292</td>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.4973</td>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5257</td>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5111</td>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0323</td>\n",
+       "                        <td id=\"T_f8d1d640_82b2_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0323</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7fb80daa7e80>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5103</td>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5309</td>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.4946</td>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5199</td>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5069</td>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0212</td>\n",
+       "                        <td id=\"T_20358cc2_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0212</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7fb80c0ebc10>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5282</td>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5543</td>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5351</td>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5366</td>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5359</td>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0561</td>\n",
+       "                        <td id=\"T_20358cc3_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0561</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7fb8237074c0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5296</td>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.5570</td>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5297</td>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.5385</td>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5341</td>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.0591</td>\n",
+       "                        <td id=\"T_28cd58ce_82b3_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.0591</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7fb81b0d0070>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "metric_param = {\n",
+    "        \"Accuracy\": -0.1,\n",
+    "        \"AUC\": -0.1,\n",
+    "        \"Recall\": -0.1,\n",
+    "        \"Precision\": -0.1,\n",
+    "        \"F1\": -0.1,\n",
+    "        \"Kappa\": -1.0,\n",
+    "        \"MCC\": -1.0,\n",
+    "    }\n",
+    "feat_sel = FeatureSelection(target=target,\n",
+    "                            dataset=train_data.dropna(),#[list(set(new_subset+new_feat+[target]))],\n",
+    "                            target_features=500,\n",
+    "                            filter_metrics=metric_param,\n",
+    "                            include=[\"lr\"],\n",
+    "                            setup_kwargs=setup_kwargs,\n",
+    "                            optimize=True,\n",
+    "                            opt_list=[\"AUC\"],\n",
+    "                           )\n",
+    "selected_features = feat_sel.repeat_pipeline()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['DEF_mean_a',\n",
+       " 'MED_diff',\n",
+       " 'accurate_cross_nocorner_std_diff',\n",
+       " 'accurate_cross_std_h',\n",
+       " 'accurate_fwd_zone_pass_mean_diff',\n",
+       " 'accurate_fwd_zone_pass_std_a',\n",
+       " 'accurate_goal_kicks_std_h',\n",
+       " 'accurate_keeper_sweeper_mean_a',\n",
+       " 'accurate_keeper_sweeper_mean_h',\n",
+       " 'accurate_keeper_sweeper_std_a',\n",
+       " 'accurate_launches_mean_a',\n",
+       " 'accurate_layoffs_mean_a',\n",
+       " 'accurate_layoffs_std_a',\n",
+       " 'accurate_pass_std_diff',\n",
+       " 'accurate_through_ball_mean_diff',\n",
+       " 'accurate_throws_std_diff',\n",
+       " 'accurate_throws_std_h',\n",
+       " 'att_assist_openplay_mean_a',\n",
+       " 'att_corner_ratio_mean_a',\n",
+       " 'att_freekick_goal_std_diff',\n",
+       " 'att_freekick_miss_mean_h',\n",
+       " 'att_freekick_miss_std_a',\n",
+       " 'att_freekick_total_mean_a',\n",
+       " 'att_freekick_total_mean_h',\n",
+       " 'att_goal_high_centre_mean_diff',\n",
+       " 'att_goal_high_centre_std_diff',\n",
+       " 'att_goal_high_left_mean_diff',\n",
+       " 'att_goal_high_left_std_h',\n",
+       " 'att_goal_high_right_std_a',\n",
+       " 'att_goal_high_right_std_diff',\n",
+       " 'att_goal_low_centre_mean_a',\n",
+       " 'att_goal_low_centre_std_a',\n",
+       " 'att_goal_low_centre_std_h',\n",
+       " 'att_goal_low_right_mean_diff',\n",
+       " 'att_hd_goal_mean_a',\n",
+       " 'att_hd_target_mean_a',\n",
+       " 'att_ibox_own_goal_std_h',\n",
+       " 'att_lf_goal_mean_diff',\n",
+       " 'att_lf_goal_mean_h',\n",
+       " 'att_lf_goal_std_h',\n",
+       " 'att_lg_centre_mean_a',\n",
+       " 'att_lg_centre_mean_diff',\n",
+       " 'att_lg_centre_mean_h',\n",
+       " 'att_miss_high_left_std_a',\n",
+       " 'att_miss_right_std_h',\n",
+       " 'att_obox_blocked_mean_h',\n",
+       " 'att_obox_goal_mean_h',\n",
+       " 'att_obox_goal_std_a',\n",
+       " 'att_obox_goal_std_diff',\n",
+       " 'att_obox_miss_mean_a',\n",
+       " 'att_obox_miss_mean_diff',\n",
+       " 'att_obox_post_mean_diff',\n",
+       " 'att_obx_right_mean_diff',\n",
+       " 'att_obxd_right_mean_diff',\n",
+       " 'att_obxd_right_std_diff',\n",
+       " 'att_openplay_mean_h',\n",
+       " 'att_pen_goal_mean_a',\n",
+       " 'att_post_left_mean_h',\n",
+       " 'att_post_left_std_diff',\n",
+       " 'attempts_conceded_ibox_std_diff',\n",
+       " 'attempts_conceded_obox_std_a',\n",
+       " 'attempts_conceded_obox_std_diff',\n",
+       " 'attempts_conceded_obox_std_h',\n",
+       " 'attempts_obox_mean_h',\n",
+       " 'bc_miss_div_scored_std_h',\n",
+       " 'big_chance_created_mean_h',\n",
+       " 'big_chance_created_std_diff',\n",
+       " 'big_chance_missed_std_diff',\n",
+       " 'big_chance_scored_mean_diff',\n",
+       " 'blocked_cross_mean_a',\n",
+       " 'contentious_decision_mean_a',\n",
+       " 'contentious_decision_std_a',\n",
+       " 'corners_ratio_mean_diff',\n",
+       " 'diving_save_std_diff',\n",
+       " 'effective_clearance_mean_diff',\n",
+       " 'effective_head_clearance_mean_h',\n",
+       " 'first_yellow_card_std_a',\n",
+       " 'goal_assist_deadball_mean_diff',\n",
+       " 'goal_assist_deadball_std_diff',\n",
+       " 'goal_assist_deadball_std_h',\n",
+       " 'goal_assist_intent_norm_mean_h',\n",
+       " 'goal_assist_intentional_mean_h',\n",
+       " 'goal_assist_intentional_std_h',\n",
+       " 'goal_assist_mean_a',\n",
+       " 'goal_assist_mean_diff',\n",
+       " 'goal_assist_openplay_std_a',\n",
+       " 'goal_assist_openplay_std_h',\n",
+       " 'goal_assist_setplay_mean_a',\n",
+       " 'goal_assist_std_diff',\n",
+       " 'goal_assist_std_h',\n",
+       " 'goal_fastbreak_mean_diff',\n",
+       " 'goal_kicks_per_shot_mean_a',\n",
+       " 'goals_2t_mean_h',\n",
+       " 'goals_mean_diff',\n",
+       " 'goals_openplay_mean_a',\n",
+       " 'interceptions_in_box_std_a',\n",
+       " 'interceptions_in_box_std_diff',\n",
+       " 'interceptions_in_box_std_h',\n",
+       " 'last_man_tackle_mean_diff',\n",
+       " 'last_man_tackle_std_a',\n",
+       " 'leftside_pass_mean_a',\n",
+       " 'long_pass_own_to_opp_success_std_diff',\n",
+       " 'lost_corners_mean_a',\n",
+       " 'no_foot_goals_ratio_mean_h',\n",
+       " 'no_foot_goals_ratio_std_a',\n",
+       " 'no_foot_goals_ratio_std_h',\n",
+       " 'odd_ratio_under_goals_2.5',\n",
+       " 'odds_away_over_cards_4.5_h',\n",
+       " 'odds_away_over_cards_5.5_a',\n",
+       " 'odds_away_under_cards_3.5_h',\n",
+       " 'odds_home_under_cards_5.5_a',\n",
+       " 'odds_home_under_goals_0.5_diff',\n",
+       " 'ontarget_att_assist_mean_a',\n",
+       " 'own_goals_std_h',\n",
+       " 'passes_left_mean_diff',\n",
+       " 'passes_right_mean_a',\n",
+       " 'passes_right_mean_diff',\n",
+       " 'passes_right_std_a',\n",
+       " 'passes_right_std_diff',\n",
+       " 'passes_right_std_h',\n",
+       " 'pen_goals_conceded_std_diff',\n",
+       " 'penalty_conceded_mean_a',\n",
+       " 'penalty_faced_std_diff',\n",
+       " 'penalty_save_std_h',\n",
+       " 'points_diff',\n",
+       " 'poss_won_att_3rd_std_a',\n",
+       " 'poss_won_att_3rd_std_diff',\n",
+       " 'poss_won_att_3rd_std_h',\n",
+       " 'prob_over_goals_2.5',\n",
+       " 'prob_squared_over_goals_4.5',\n",
+       " 'prob_squared_under_goals_4.5',\n",
+       " 'prob_under_goals_2.5',\n",
+       " 'ratio_over_corners_11.5_diff',\n",
+       " 'ratio_over_corners_9.5_diff',\n",
+       " 'ratio_under_corners_10.5_h',\n",
+       " 'ratio_under_goals_4.5_h',\n",
+       " 'raw_prob_over_goals_1.5',\n",
+       " 'red_card_1t_std_a',\n",
+       " 'red_card_2t_mean_a',\n",
+       " 'red_card_mean_h',\n",
+       " 'red_card_std_a',\n",
+       " 'red_card_std_diff',\n",
+       " 'right_to_left_goals_mean_diff',\n",
+       " 'right_to_left_goals_std_diff',\n",
+       " 'saves_std_h',\n",
+       " 'second_yellow_mean_diff',\n",
+       " 'second_yellow_std_a',\n",
+       " 'shots_std_diff',\n",
+       " 'successful_final_third_passes_std_h',\n",
+       " 'total_bets_under_cards_4.5_a',\n",
+       " 'total_bets_under_corners_9.5_a',\n",
+       " 'total_clearance_mean_a',\n",
+       " 'total_expulsions_ref',\n",
+       " 'total_fastbreak_std_diff',\n",
+       " 'total_fwd_zone_pass_std_a',\n",
+       " 'total_high_claim_mean_diff',\n",
+       " 'total_keeper_sweeper_mean_a',\n",
+       " 'total_keeper_sweeper_mean_diff',\n",
+       " 'total_keeper_sweeper_std_diff',\n",
+       " 'total_launches_mean_a',\n",
+       " 'total_launches_std_h',\n",
+       " 'total_layoffs_mean_diff',\n",
+       " 'total_layoffs_std_diff',\n",
+       " 'total_red_card_match_ref',\n",
+       " 'total_second_yel_card_ref',\n",
+       " 'total_through_ball_mean_a',\n",
+       " 'total_through_ball_std_a',\n",
+       " 'total_throws_std_a',\n",
+       " 'total_throws_std_diff',\n",
+       " 'total_win_pct_over_corners_9.5_a',\n",
+       " 'total_win_pct_over_goals_2.5_h',\n",
+       " 'total_win_pct_under_goals_1.5_a',\n",
+       " 'total_win_pct_under_goals_3.5_h',\n",
+       " 'total_yel_card_std_a',\n",
+       " 'win_pct_away_over_goals_2.5_diff',\n",
+       " 'win_pct_home_over_cards_6.5_a',\n",
+       " 'win_pct_home_over_goals_2.5_h',\n",
+       " 'win_pct_home_under_cards_4.5_diff',\n",
+       " 'won_corners_ratio_mean_a',\n",
+       " 'won_corners_ratio_mean_h',\n",
+       " 'won_corners_ratio_std_h']"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sorted(selected_features)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### new imports "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
+    "from ml_bets.constants import FEATURES_PATH\n",
+    "from ml_bets.features.features import Features\n",
+    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
+    "#from featsel.feature_selection import FeatureSelection\n",
+    "\n",
+    "from pycaret.utils import check_metric\n",
+    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
+    "    create_model,\n",
+    "    finalize_model,\n",
+    "    optimize_threshold,              \n",
+    "    save_model,\n",
+    "    compare_models, \n",
+    "    evaluate_model,\n",
+    "    get_config,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    "    predict_model,\n",
+    ")\n",
+    "\n",
+    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
+    "from ml_bets.research.datasets import create_dataset\n",
+    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ml_bets.research.tips import calibrate_tips, tips_from_model, combine_tips, predict_dataset, compose_tips, get_tip_probs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ix = np.logical_and(feats.matches[\"date\"].dt.month > 10,\n",
+    "                    feats.matches[\"competition\"].isin({\"mexican_primera\", 'us_major_league_soccer'}))\n",
+    "index = feats.matches[~ix].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def setup_dataset(test_date):\n",
+    "    ds = create_dataset(target=target,\n",
+    "                        test_date=test_date,\n",
+    "                        features=feats,\n",
+    "                        odds_features=True,\n",
+    "                        include_std=True,\n",
+    "                        test_weeks=4,\n",
+    "                        ignore_features=IGNORE_FEATURES,\n",
+    "                        drop_future_matches=False,\n",
+    "                       )\n",
+    "    train_data = ds.train_data.copy()#[ds.train_data.index.map(lambda x: \"us_major_league_soccer\" not in x and \"mexican\" not in x)]\n",
+    "    train_data.drop(columns=[\"hour_rank\", \"hour_before_16\", \"is_weekend\"], inplace=True)\n",
+    "    train_data = train_data[train_data.index.isin(index)][list(set(selected_features)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    test_data = ds.test_set[ds.test_set.index.isin(index)][list(set(selected_features)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    val_data = ds.val_set[ds.val_set.index.isin(index)].copy()\n",
+    "    setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        test_data=test_data[train_data.columns.tolist()],#.dropna(),\n",
+    "        #numeric_features=[x for x in train_data.columns.tolist() if x != target],\n",
+    "        #custom_pipeline=loaded,\n",
+    "        #train_size=0.75,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "        normalize_method=\"robust\",\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.8,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        #ignore_features=ignore_features,\n",
+    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=4,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.5,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=True,\n",
+    "        pca_method=\"kernel\",\n",
+    "        pca_components=50,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.05,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=True,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.05,\n",
+    "        fix_imbalance=True,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "    )\n",
+    "    _ = setup(data=train_data, target=target, **setup_kwargs)\n",
+    "    return train_data, test_data, val_data, ds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data, test_data, val_data, ds = setup_dataset(\"4-Dec-2021\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
+    "def train_ensemble():\n",
+    "    top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"et\", \"rf\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )\n",
+    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
+    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
+    "    blend = blend_models(cali)\n",
+    "    opti = tune_model(blend, optimize=\"Precision\", choose_better=True, n_iter=50, search_library=\"optuna\")\n",
+    "    return opti, cali, tuned_models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "    #T_21ce2d37_829b_11ec_86da_31d766f3ddbd th {\n",
+       "          text-align: left;\n",
+       "    }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col7{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col5{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col8,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col8,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  lightgrey;\n",
+       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "            background-color:  lightgrey;\n",
+       "        }</style><table id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>        <th class=\"col_heading level0 col8\" >TT (Sec)</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >lr</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5831</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6161</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5937</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6030</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5971</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.1652</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.1658</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col8\" class=\"data row0 col8\" >0.6300</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >svm</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >SVM - Linear Kernel</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.5780</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.5975</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6037</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.5845</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.1544</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col7\" class=\"data row1 col7\" >0.1656</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col8\" class=\"data row1 col8\" >0.2050</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >ridge</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >Ridge Classifier</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.5825</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.5937</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6018</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.5966</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.1639</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col7\" class=\"data row2 col7\" >0.1645</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col8\" class=\"data row2 col8\" >0.2000</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >lda</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >Linear Discriminant Analysis</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.5819</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.6107</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.5962</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6003</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.5972</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.1624</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col7\" class=\"data row3 col7\" >0.1629</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col8\" class=\"data row3 col8\" >0.2100</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7ff4940a2880>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluate_model(top_models[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Feature selection (adding triplet)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(181,\n",
+       " 583,\n",
+       " ['ATT_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_mean_a',\n",
+       "  'DEF_mean_diff',\n",
+       "  'DEF_mean_h',\n",
+       "  'MED_a',\n",
+       "  'MED_diff',\n",
+       "  'MED_h',\n",
+       "  'MED_mean_a',\n",
+       "  'MED_mean_diff',\n",
+       "  'MED_mean_h',\n",
+       "  'MED_std_a',\n",
+       "  'MED_std_diff',\n",
+       "  'MED_std_h',\n",
+       "  'accurate_cross_nocorner_std_a',\n",
+       "  'accurate_cross_nocorner_std_diff',\n",
+       "  'accurate_cross_nocorner_std_h',\n",
+       "  'accurate_cross_std_a',\n",
+       "  'accurate_cross_std_diff',\n",
+       "  'accurate_cross_std_h',\n",
+       "  'accurate_fwd_zone_pass_mean_a',\n",
+       "  'accurate_fwd_zone_pass_mean_diff',\n",
+       "  'accurate_fwd_zone_pass_mean_h',\n",
+       "  'accurate_fwd_zone_pass_std_a',\n",
+       "  'accurate_fwd_zone_pass_std_diff',\n",
+       "  'accurate_fwd_zone_pass_std_h',\n",
+       "  'accurate_goal_kicks_std_a',\n",
+       "  'accurate_goal_kicks_std_diff',\n",
+       "  'accurate_goal_kicks_std_h',\n",
+       "  'accurate_keeper_sweeper_mean_a',\n",
+       "  'accurate_keeper_sweeper_mean_diff',\n",
+       "  'accurate_keeper_sweeper_mean_h',\n",
+       "  'accurate_keeper_sweeper_std_a',\n",
+       "  'accurate_keeper_sweeper_std_diff',\n",
+       "  'accurate_keeper_sweeper_std_h',\n",
+       "  'accurate_launches_mean_a',\n",
+       "  'accurate_launches_mean_diff',\n",
+       "  'accurate_launches_mean_h',\n",
+       "  'accurate_layoffs_mean_a',\n",
+       "  'accurate_layoffs_mean_diff',\n",
+       "  'accurate_layoffs_mean_h',\n",
+       "  'accurate_layoffs_std_a',\n",
+       "  'accurate_layoffs_std_diff',\n",
+       "  'accurate_layoffs_std_h',\n",
+       "  'accurate_pass_std_a',\n",
+       "  'accurate_pass_std_diff',\n",
+       "  'accurate_pass_std_h',\n",
+       "  'accurate_through_ball_mean_a',\n",
+       "  'accurate_through_ball_mean_diff',\n",
+       "  'accurate_through_ball_mean_h',\n",
+       "  'accurate_throws_std_a',\n",
+       "  'accurate_throws_std_diff',\n",
+       "  'accurate_throws_std_h',\n",
+       "  'att_assist_openplay_mean_a',\n",
+       "  'att_assist_openplay_mean_diff',\n",
+       "  'att_assist_openplay_mean_h',\n",
+       "  'att_corner_ratio_mean_a',\n",
+       "  'att_corner_ratio_mean_diff',\n",
+       "  'att_corner_ratio_mean_h',\n",
+       "  'att_freekick_goal_std_a',\n",
+       "  'att_freekick_goal_std_diff',\n",
+       "  'att_freekick_goal_std_h',\n",
+       "  'att_freekick_miss_mean_a',\n",
+       "  'att_freekick_miss_mean_diff',\n",
+       "  'att_freekick_miss_mean_h',\n",
+       "  'att_freekick_miss_std_a',\n",
+       "  'att_freekick_miss_std_diff',\n",
+       "  'att_freekick_miss_std_h',\n",
+       "  'att_freekick_total_mean_a',\n",
+       "  'att_freekick_total_mean_diff',\n",
+       "  'att_freekick_total_mean_h',\n",
+       "  'att_goal_high_centre_mean_a',\n",
+       "  'att_goal_high_centre_mean_diff',\n",
+       "  'att_goal_high_centre_mean_h',\n",
+       "  'att_goal_high_centre_std_a',\n",
+       "  'att_goal_high_centre_std_diff',\n",
+       "  'att_goal_high_centre_std_h',\n",
+       "  'att_goal_high_left_mean_a',\n",
+       "  'att_goal_high_left_mean_diff',\n",
+       "  'att_goal_high_left_mean_h',\n",
+       "  'att_goal_high_left_std_a',\n",
+       "  'att_goal_high_left_std_diff',\n",
+       "  'att_goal_high_left_std_h',\n",
+       "  'att_goal_high_right_std_a',\n",
+       "  'att_goal_high_right_std_diff',\n",
+       "  'att_goal_high_right_std_h',\n",
+       "  'att_goal_low_centre_mean_a',\n",
+       "  'att_goal_low_centre_mean_diff',\n",
+       "  'att_goal_low_centre_mean_h',\n",
+       "  'att_goal_low_centre_std_a',\n",
+       "  'att_goal_low_centre_std_diff',\n",
+       "  'att_goal_low_centre_std_h',\n",
+       "  'att_goal_low_right_mean_a',\n",
+       "  'att_goal_low_right_mean_diff',\n",
+       "  'att_goal_low_right_mean_h',\n",
+       "  'att_hd_goal_mean_a',\n",
+       "  'att_hd_goal_mean_diff',\n",
+       "  'att_hd_goal_mean_h',\n",
+       "  'att_hd_target_mean_a',\n",
+       "  'att_hd_target_mean_diff',\n",
+       "  'att_hd_target_mean_h',\n",
+       "  'att_ibox_own_goal_std_a',\n",
+       "  'att_ibox_own_goal_std_diff',\n",
+       "  'att_ibox_own_goal_std_h',\n",
+       "  'att_lf_goal_mean_a',\n",
+       "  'att_lf_goal_mean_diff',\n",
+       "  'att_lf_goal_mean_h',\n",
+       "  'att_lf_goal_std_a',\n",
+       "  'att_lf_goal_std_diff',\n",
+       "  'att_lf_goal_std_h',\n",
+       "  'att_lg_centre_mean_a',\n",
+       "  'att_lg_centre_mean_diff',\n",
+       "  'att_lg_centre_mean_h',\n",
+       "  'att_miss_high_left_std_a',\n",
+       "  'att_miss_high_left_std_diff',\n",
+       "  'att_miss_high_left_std_h',\n",
+       "  'att_miss_right_std_a',\n",
+       "  'att_miss_right_std_diff',\n",
+       "  'att_miss_right_std_h',\n",
+       "  'att_obox_blocked_mean_a',\n",
+       "  'att_obox_blocked_mean_diff',\n",
+       "  'att_obox_blocked_mean_h',\n",
+       "  'att_obox_goal_mean_a',\n",
+       "  'att_obox_goal_mean_diff',\n",
+       "  'att_obox_goal_mean_h',\n",
+       "  'att_obox_goal_std_a',\n",
+       "  'att_obox_goal_std_diff',\n",
+       "  'att_obox_goal_std_h',\n",
+       "  'att_obox_miss_mean_a',\n",
+       "  'att_obox_miss_mean_diff',\n",
+       "  'att_obox_miss_mean_h',\n",
+       "  'att_obox_post_mean_a',\n",
+       "  'att_obox_post_mean_diff',\n",
+       "  'att_obox_post_mean_h',\n",
+       "  'att_obx_right_mean_a',\n",
+       "  'att_obx_right_mean_diff',\n",
+       "  'att_obx_right_mean_h',\n",
+       "  'att_obxd_right_mean_a',\n",
+       "  'att_obxd_right_mean_diff',\n",
+       "  'att_obxd_right_mean_h',\n",
+       "  'att_obxd_right_std_a',\n",
+       "  'att_obxd_right_std_diff',\n",
+       "  'att_obxd_right_std_h',\n",
+       "  'att_openplay_mean_a',\n",
+       "  'att_openplay_mean_diff',\n",
+       "  'att_openplay_mean_h',\n",
+       "  'att_pen_goal_mean_a',\n",
+       "  'att_pen_goal_mean_diff',\n",
+       "  'att_pen_goal_mean_h',\n",
+       "  'att_post_left_mean_a',\n",
+       "  'att_post_left_mean_diff',\n",
+       "  'att_post_left_mean_h',\n",
+       "  'att_post_left_std_a',\n",
+       "  'att_post_left_std_diff',\n",
+       "  'att_post_left_std_h',\n",
+       "  'attempts_conceded_ibox_std_a',\n",
+       "  'attempts_conceded_ibox_std_diff',\n",
+       "  'attempts_conceded_ibox_std_h',\n",
+       "  'attempts_conceded_obox_std_a',\n",
+       "  'attempts_conceded_obox_std_diff',\n",
+       "  'attempts_conceded_obox_std_h',\n",
+       "  'attempts_obox_mean_a',\n",
+       "  'attempts_obox_mean_diff',\n",
+       "  'attempts_obox_mean_h',\n",
+       "  'bc_miss_div_scored_std_a',\n",
+       "  'bc_miss_div_scored_std_diff',\n",
+       "  'bc_miss_div_scored_std_h',\n",
+       "  'big_chance_created_mean_a',\n",
+       "  'big_chance_created_mean_diff',\n",
+       "  'big_chance_created_mean_h',\n",
+       "  'big_chance_created_std_a',\n",
+       "  'big_chance_created_std_diff',\n",
+       "  'big_chance_created_std_h',\n",
+       "  'big_chance_missed_std_a',\n",
+       "  'big_chance_missed_std_diff',\n",
+       "  'big_chance_missed_std_h',\n",
+       "  'big_chance_scored_mean_a',\n",
+       "  'big_chance_scored_mean_diff',\n",
+       "  'big_chance_scored_mean_diff_mul_total_fwd_zone_pass_mean_diff',\n",
+       "  'big_chance_scored_mean_h',\n",
+       "  'blocked_cross_mean_a',\n",
+       "  'blocked_cross_mean_diff',\n",
+       "  'blocked_cross_mean_h',\n",
+       "  'contentious_decision_mean_a',\n",
+       "  'contentious_decision_mean_diff',\n",
+       "  'contentious_decision_mean_h',\n",
+       "  'contentious_decision_std_a',\n",
+       "  'contentious_decision_std_diff',\n",
+       "  'contentious_decision_std_h',\n",
+       "  'corner_taken_std_mul_att_corner_ratio_mean_diffs',\n",
+       "  'corners_ratio_mean_a',\n",
+       "  'corners_ratio_mean_diff',\n",
+       "  'corners_ratio_mean_h',\n",
+       "  'defender_goals_mean_a',\n",
+       "  'defender_goals_mean_diff',\n",
+       "  'defender_goals_mean_h',\n",
+       "  'diving_save_std_a',\n",
+       "  'diving_save_std_diff',\n",
+       "  'diving_save_std_h',\n",
+       "  'effective_clearance_mean_a',\n",
+       "  'effective_clearance_mean_diff',\n",
+       "  'effective_clearance_mean_h',\n",
+       "  'effective_head_clearance_mean_a',\n",
+       "  'effective_head_clearance_mean_diff',\n",
+       "  'effective_head_clearance_mean_h',\n",
+       "  'first_yellow_card_std_a',\n",
+       "  'first_yellow_card_std_diff',\n",
+       "  'first_yellow_card_std_h',\n",
+       "  'forward_goals_mean_a',\n",
+       "  'forward_goals_mean_diff',\n",
+       "  'forward_goals_mean_h',\n",
+       "  'fwd_pass_div_long_pass_own_to_opp_success_std_a',\n",
+       "  'fwd_pass_div_long_pass_own_to_opp_success_std_diff',\n",
+       "  'fwd_pass_div_long_pass_own_to_opp_success_std_h',\n",
+       "  'goal_assist_deadball_mean_a',\n",
+       "  'goal_assist_deadball_mean_diff',\n",
+       "  'goal_assist_deadball_mean_h',\n",
+       "  'goal_assist_deadball_std_a',\n",
+       "  'goal_assist_deadball_std_diff',\n",
+       "  'goal_assist_deadball_std_h',\n",
+       "  'goal_assist_intent_norm_mean_a',\n",
+       "  'goal_assist_intent_norm_mean_diff',\n",
+       "  'goal_assist_intent_norm_mean_h',\n",
+       "  'goal_assist_intentional_mean_a',\n",
+       "  'goal_assist_intentional_mean_diff',\n",
+       "  'goal_assist_intentional_mean_h',\n",
+       "  'goal_assist_intentional_std_a',\n",
+       "  'goal_assist_intentional_std_diff',\n",
+       "  'goal_assist_intentional_std_h',\n",
+       "  'goal_assist_mean_a',\n",
+       "  'goal_assist_mean_diff',\n",
+       "  'goal_assist_mean_h',\n",
+       "  'goal_assist_openplay_std_a',\n",
+       "  'goal_assist_openplay_std_diff',\n",
+       "  'goal_assist_openplay_std_h',\n",
+       "  'goal_assist_setplay_mean_a',\n",
+       "  'goal_assist_setplay_mean_diff',\n",
+       "  'goal_assist_setplay_mean_h',\n",
+       "  'goal_assist_std_a',\n",
+       "  'goal_assist_std_diff',\n",
+       "  'goal_assist_std_h',\n",
+       "  'goal_fastbreak_mean_a',\n",
+       "  'goal_fastbreak_mean_diff',\n",
+       "  'goal_fastbreak_mean_h',\n",
+       "  'goal_kicks_per_shot_mean_a',\n",
+       "  'goal_kicks_per_shot_mean_diff',\n",
+       "  'goal_kicks_per_shot_mean_h',\n",
+       "  'goals_2t_mean_a',\n",
+       "  'goals_2t_mean_diff',\n",
+       "  'goals_2t_mean_h',\n",
+       "  'goals_mean_a',\n",
+       "  'goals_mean_diff',\n",
+       "  'goals_mean_h',\n",
+       "  'goals_openplay_mean_a',\n",
+       "  'goals_openplay_mean_diff',\n",
+       "  'goals_openplay_mean_h',\n",
+       "  'high_to_low_goals_mean_a',\n",
+       "  'high_to_low_goals_mean_diff',\n",
+       "  'high_to_low_goals_mean_h',\n",
+       "  'imp_prob_over_goals_0.5_a',\n",
+       "  'imp_prob_over_goals_0.5_diff',\n",
+       "  'imp_prob_over_goals_0.5_h',\n",
+       "  'imp_prob_over_goals_1.5_a',\n",
+       "  'imp_prob_over_goals_1.5_diff',\n",
+       "  'imp_prob_over_goals_1.5_h',\n",
+       "  'imp_prob_over_goals_2.5_a',\n",
+       "  'imp_prob_over_goals_2.5_diff',\n",
+       "  'imp_prob_over_goals_2.5_h',\n",
+       "  'imp_prob_over_goals_3.5_a',\n",
+       "  'imp_prob_over_goals_3.5_diff',\n",
+       "  'imp_prob_over_goals_3.5_h',\n",
+       "  'imp_prob_over_goals_4.5_a',\n",
+       "  'imp_prob_over_goals_4.5_diff',\n",
+       "  'imp_prob_over_goals_4.5_h',\n",
+       "  'imp_prob_under_goals_0.5_a',\n",
+       "  'imp_prob_under_goals_0.5_diff',\n",
+       "  'imp_prob_under_goals_0.5_h',\n",
+       "  'imp_prob_under_goals_1.5_a',\n",
+       "  'imp_prob_under_goals_1.5_diff',\n",
+       "  'imp_prob_under_goals_1.5_h',\n",
+       "  'imp_prob_under_goals_2.5_a',\n",
+       "  'imp_prob_under_goals_2.5_diff',\n",
+       "  'imp_prob_under_goals_2.5_h',\n",
+       "  'imp_prob_under_goals_3.5_a',\n",
+       "  'imp_prob_under_goals_3.5_diff',\n",
+       "  'imp_prob_under_goals_3.5_h',\n",
+       "  'imp_prob_under_goals_4.5_a',\n",
+       "  'imp_prob_under_goals_4.5_diff',\n",
+       "  'imp_prob_under_goals_4.5_h',\n",
+       "  'interceptions_in_box_std_a',\n",
+       "  'interceptions_in_box_std_diff',\n",
+       "  'interceptions_in_box_std_h',\n",
+       "  'last_man_tackle_mean_a',\n",
+       "  'last_man_tackle_mean_diff',\n",
+       "  'last_man_tackle_mean_h',\n",
+       "  'last_man_tackle_std_a',\n",
+       "  'last_man_tackle_std_diff',\n",
+       "  'last_man_tackle_std_h',\n",
+       "  'left_div_right_foot_goals_mean_a',\n",
+       "  'left_div_right_foot_goals_mean_diff',\n",
+       "  'left_div_right_foot_goals_mean_h',\n",
+       "  'leftside_pass_div_accurate_pass_std_a',\n",
+       "  'leftside_pass_div_accurate_pass_std_diff',\n",
+       "  'leftside_pass_div_accurate_pass_std_h',\n",
+       "  'leftside_pass_mean_a',\n",
+       "  'leftside_pass_mean_diff',\n",
+       "  'leftside_pass_mean_h',\n",
+       "  'long_pass_own_to_opp_success_std_a',\n",
+       "  'long_pass_own_to_opp_success_std_diff',\n",
+       "  'long_pass_own_to_opp_success_std_h',\n",
+       "  'lost_corners_mean_a',\n",
+       "  'lost_corners_mean_diff',\n",
+       "  'lost_corners_mean_h',\n",
+       "  'midfielder_goals_mean_a',\n",
+       "  'midfielder_goals_mean_diff',\n",
+       "  'midfielder_goals_mean_h',\n",
+       "  'no_foot_goals_ratio_mean_a',\n",
+       "  'no_foot_goals_ratio_mean_diff',\n",
+       "  'no_foot_goals_ratio_mean_h',\n",
+       "  'no_foot_goals_ratio_std_a',\n",
+       "  'no_foot_goals_ratio_std_diff',\n",
+       "  'no_foot_goals_ratio_std_h',\n",
+       "  'odd_ratio_over_corners_11.5',\n",
+       "  'odd_ratio_over_corners_9.5',\n",
+       "  'odd_ratio_under_corners_10.5',\n",
+       "  'odd_ratio_under_goals_1.5',\n",
+       "  'odd_ratio_under_goals_2.5',\n",
+       "  'odd_ratio_under_goals_3.5',\n",
+       "  'odd_ratio_under_goals_4.5',\n",
+       "  'odd_ratio_under_goals_4.5',\n",
+       "  'odds_away_over_cards_4.5_a',\n",
+       "  'odds_away_over_cards_4.5_diff',\n",
+       "  'odds_away_over_cards_4.5_h',\n",
+       "  'odds_away_over_cards_5.5_a',\n",
+       "  'odds_away_over_cards_5.5_diff',\n",
+       "  'odds_away_over_cards_5.5_h',\n",
+       "  'odds_away_under_cards_3.5_a',\n",
+       "  'odds_away_under_cards_3.5_diff',\n",
+       "  'odds_away_under_cards_3.5_h',\n",
+       "  'odds_home_under_cards_5.5_a',\n",
+       "  'odds_home_under_cards_5.5_diff',\n",
+       "  'odds_home_under_cards_5.5_h',\n",
+       "  'odds_home_under_goals_0.5_a',\n",
+       "  'odds_home_under_goals_0.5_diff',\n",
+       "  'odds_home_under_goals_0.5_h',\n",
+       "  'ontarget_att_assist_mean_a',\n",
+       "  'ontarget_att_assist_mean_diff',\n",
+       "  'ontarget_att_assist_mean_h',\n",
+       "  'own_goals_mean_a',\n",
+       "  'own_goals_mean_diff',\n",
+       "  'own_goals_mean_h',\n",
+       "  'own_goals_std_a',\n",
+       "  'own_goals_std_diff',\n",
+       "  'own_goals_std_h',\n",
+       "  'passes_left_mean_a',\n",
+       "  'passes_left_mean_diff',\n",
+       "  'passes_left_mean_h',\n",
+       "  'passes_right_mean_a',\n",
+       "  'passes_right_mean_diff',\n",
+       "  'passes_right_mean_h',\n",
+       "  'passes_right_std_a',\n",
+       "  'passes_right_std_diff',\n",
+       "  'passes_right_std_h',\n",
+       "  'pen_goals_conceded_std_a',\n",
+       "  'pen_goals_conceded_std_diff',\n",
+       "  'pen_goals_conceded_std_h',\n",
+       "  'penalty_conceded_mean_a',\n",
+       "  'penalty_conceded_mean_diff',\n",
+       "  'penalty_conceded_mean_h',\n",
+       "  'penalty_faced_std_a',\n",
+       "  'penalty_faced_std_diff',\n",
+       "  'penalty_faced_std_h',\n",
+       "  'penalty_save_std_a',\n",
+       "  'penalty_save_std_diff',\n",
+       "  'penalty_save_std_h',\n",
+       "  'points_a',\n",
+       "  'points_diff',\n",
+       "  'points_h',\n",
+       "  'points_mean_a',\n",
+       "  'points_mean_diff',\n",
+       "  'points_mean_h',\n",
+       "  'points_std_a',\n",
+       "  'points_std_diff',\n",
+       "  'points_std_h',\n",
+       "  'poss_won_att_3rd_std_a',\n",
+       "  'poss_won_att_3rd_std_diff',\n",
+       "  'poss_won_att_3rd_std_h',\n",
+       "  'prob_over_goals_1.5',\n",
+       "  'prob_over_goals_2.5',\n",
+       "  'prob_over_goals_3.5',\n",
+       "  'prob_over_goals_4.5',\n",
+       "  'prob_squared_over_goals_1.5',\n",
+       "  'prob_squared_over_goals_2.5',\n",
+       "  'prob_squared_over_goals_3.5',\n",
+       "  'prob_squared_over_goals_4.5',\n",
+       "  'prob_squared_under_goals_1.5',\n",
+       "  'prob_squared_under_goals_2.5',\n",
+       "  'prob_squared_under_goals_3.5',\n",
+       "  'prob_squared_under_goals_4.5',\n",
+       "  'prob_under_goals_1.5',\n",
+       "  'prob_under_goals_2.5',\n",
+       "  'prob_under_goals_3.5',\n",
+       "  'prob_under_goals_4.5',\n",
+       "  'ratio_over_corners_11.5_a',\n",
+       "  'ratio_over_corners_11.5_diff',\n",
+       "  'ratio_over_corners_11.5_h',\n",
+       "  'ratio_over_corners_9.5_a',\n",
+       "  'ratio_over_corners_9.5_diff',\n",
+       "  'ratio_over_corners_9.5_h',\n",
+       "  'ratio_under_corners_10.5_a',\n",
+       "  'ratio_under_corners_10.5_diff',\n",
+       "  'ratio_under_corners_10.5_h',\n",
+       "  'ratio_under_goals_4.5_a',\n",
+       "  'ratio_under_goals_4.5_diff',\n",
+       "  'ratio_under_goals_4.5_h',\n",
+       "  'raw_prob_over_goals_1.5',\n",
+       "  'raw_prob_over_goals_1.5',\n",
+       "  'raw_prob_over_goals_2.5',\n",
+       "  'raw_prob_over_goals_2.5',\n",
+       "  'raw_prob_over_goals_3.5',\n",
+       "  'raw_prob_over_goals_3.5',\n",
+       "  'raw_prob_over_goals_4.5',\n",
+       "  'raw_prob_over_goals_4.5',\n",
+       "  'raw_prob_under_goals_1.5',\n",
+       "  'raw_prob_under_goals_2.5',\n",
+       "  'raw_prob_under_goals_3.5',\n",
+       "  'raw_prob_under_goals_4.5',\n",
+       "  'red_card_1t_std_a',\n",
+       "  'red_card_1t_std_diff',\n",
+       "  'red_card_1t_std_h',\n",
+       "  'red_card_2t_mean_a',\n",
+       "  'red_card_2t_mean_diff',\n",
+       "  'red_card_2t_mean_h',\n",
+       "  'red_card_mean_a',\n",
+       "  'red_card_mean_diff',\n",
+       "  'red_card_mean_h',\n",
+       "  'red_card_std_a',\n",
+       "  'red_card_std_diff',\n",
+       "  'red_card_std_h',\n",
+       "  'rescinded_red_card_mean_a',\n",
+       "  'rescinded_red_card_mean_diff',\n",
+       "  'rescinded_red_card_mean_h',\n",
+       "  'rescinded_red_card_std_a',\n",
+       "  'rescinded_red_card_std_diff',\n",
+       "  'rescinded_red_card_std_h',\n",
+       "  'right_to_left_goals_mean_a',\n",
+       "  'right_to_left_goals_mean_a',\n",
+       "  'right_to_left_goals_mean_diff',\n",
+       "  'right_to_left_goals_mean_diff',\n",
+       "  'right_to_left_goals_mean_h',\n",
+       "  'right_to_left_goals_mean_h',\n",
+       "  'right_to_left_goals_std_a',\n",
+       "  'right_to_left_goals_std_diff',\n",
+       "  'right_to_left_goals_std_h',\n",
+       "  'rightside_pass_div_accurate_pass_std_a',\n",
+       "  'rightside_pass_div_accurate_pass_std_diff',\n",
+       "  'rightside_pass_div_accurate_pass_std_h',\n",
+       "  'rightside_pass_div_leftside_pass_mean_a',\n",
+       "  'rightside_pass_div_leftside_pass_mean_diff',\n",
+       "  'rightside_pass_div_leftside_pass_mean_h',\n",
+       "  'saves_std_a',\n",
+       "  'saves_std_diff',\n",
+       "  'saves_std_h',\n",
+       "  'second_yellow_mean_a',\n",
+       "  'second_yellow_mean_diff',\n",
+       "  'second_yellow_mean_h',\n",
+       "  'second_yellow_std_a',\n",
+       "  'second_yellow_std_diff',\n",
+       "  'second_yellow_std_h',\n",
+       "  'shots_div_passes_left_mean_a',\n",
+       "  'shots_div_passes_left_mean_diff',\n",
+       "  'shots_div_passes_left_mean_h',\n",
+       "  'shots_div_passes_right_mean_a',\n",
+       "  'shots_div_passes_right_mean_diff',\n",
+       "  'shots_div_passes_right_mean_h',\n",
+       "  'shots_div_passes_right_std_a',\n",
+       "  'shots_div_passes_right_std_diff',\n",
+       "  'shots_div_passes_right_std_h',\n",
+       "  'shots_mul_goals_mean_a',\n",
+       "  'shots_mul_goals_mean_diff',\n",
+       "  'shots_mul_goals_mean_h',\n",
+       "  'shots_std_a',\n",
+       "  'shots_std_diff',\n",
+       "  'shots_std_h',\n",
+       "  'successful_final_third_passes_std_a',\n",
+       "  'successful_final_third_passes_std_diff',\n",
+       "  'successful_final_third_passes_std_h',\n",
+       "  'total_bets_under_cards_4.5_a',\n",
+       "  'total_bets_under_cards_4.5_diff',\n",
+       "  'total_bets_under_cards_4.5_h',\n",
+       "  'total_bets_under_corners_9.5_a',\n",
+       "  'total_bets_under_corners_9.5_diff',\n",
+       "  'total_bets_under_corners_9.5_h',\n",
+       "  'total_clearance_mean_a',\n",
+       "  'total_clearance_mean_diff',\n",
+       "  'total_clearance_mean_h',\n",
+       "  'total_expulsions_ref',\n",
+       "  'total_fastbreak_std_a',\n",
+       "  'total_fastbreak_std_diff',\n",
+       "  'total_fastbreak_std_h',\n",
+       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_a',\n",
+       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_diff',\n",
+       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_h',\n",
+       "  'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_diff',\n",
+       "  'total_fwd_zone_pass_std_a',\n",
+       "  'total_fwd_zone_pass_std_diff',\n",
+       "  'total_fwd_zone_pass_std_h',\n",
+       "  'total_high_claim_mean_a',\n",
+       "  'total_high_claim_mean_diff',\n",
+       "  'total_high_claim_mean_h',\n",
+       "  'total_keeper_sweeper_mean_a',\n",
+       "  'total_keeper_sweeper_mean_diff',\n",
+       "  'total_keeper_sweeper_mean_h',\n",
+       "  'total_keeper_sweeper_std_a',\n",
+       "  'total_keeper_sweeper_std_diff',\n",
+       "  'total_keeper_sweeper_std_h',\n",
+       "  'total_launches_mean_a',\n",
+       "  'total_launches_mean_diff',\n",
+       "  'total_launches_mean_h',\n",
+       "  'total_launches_std_a',\n",
+       "  'total_launches_std_diff',\n",
+       "  'total_launches_std_h',\n",
+       "  'total_layoffs_mean_a',\n",
+       "  'total_layoffs_mean_diff',\n",
+       "  'total_layoffs_mean_h',\n",
+       "  'total_layoffs_std_a',\n",
+       "  'total_layoffs_std_diff',\n",
+       "  'total_layoffs_std_h',\n",
+       "  'total_red_card_match_ref',\n",
+       "  'total_red_card_mean_a',\n",
+       "  'total_red_card_mean_diff',\n",
+       "  'total_red_card_mean_h',\n",
+       "  'total_red_card_std_a',\n",
+       "  'total_red_card_std_diff',\n",
+       "  'total_red_card_std_h',\n",
+       "  'total_second_yel_card_away_ref',\n",
+       "  'total_second_yel_card_home_ref',\n",
+       "  'total_second_yel_card_ref',\n",
+       "  'total_through_ball_mean_a',\n",
+       "  'total_through_ball_mean_diff',\n",
+       "  'total_through_ball_mean_h',\n",
+       "  'total_through_ball_std_a',\n",
+       "  'total_through_ball_std_diff',\n",
+       "  'total_through_ball_std_h',\n",
+       "  'total_throws_std_a',\n",
+       "  'total_throws_std_diff',\n",
+       "  'total_throws_std_h',\n",
+       "  'total_win_pct_over_corners_9.5_a',\n",
+       "  'total_win_pct_over_corners_9.5_diff',\n",
+       "  'total_win_pct_over_corners_9.5_h',\n",
+       "  'total_win_pct_over_goals_2.5_a',\n",
+       "  'total_win_pct_over_goals_2.5_diff',\n",
+       "  'total_win_pct_over_goals_2.5_h',\n",
+       "  'total_win_pct_under_goals_1.5_a',\n",
+       "  'total_win_pct_under_goals_1.5_diff',\n",
+       "  'total_win_pct_under_goals_1.5_h',\n",
+       "  'total_win_pct_under_goals_3.5_a',\n",
+       "  'total_win_pct_under_goals_3.5_diff',\n",
+       "  'total_win_pct_under_goals_3.5_h',\n",
+       "  'total_yel_card_std_a',\n",
+       "  'total_yel_card_std_diff',\n",
+       "  'total_yel_card_std_h',\n",
+       "  'win_pct_away_over_goals_2.5_a',\n",
+       "  'win_pct_away_over_goals_2.5_diff',\n",
+       "  'win_pct_away_over_goals_2.5_h',\n",
+       "  'win_pct_home_over_cards_6.5_a',\n",
+       "  'win_pct_home_over_cards_6.5_diff',\n",
+       "  'win_pct_home_over_cards_6.5_h',\n",
+       "  'win_pct_home_over_goals_2.5_a',\n",
+       "  'win_pct_home_over_goals_2.5_diff',\n",
+       "  'win_pct_home_over_goals_2.5_h',\n",
+       "  'win_pct_home_under_cards_4.5_a',\n",
+       "  'win_pct_home_under_cards_4.5_diff',\n",
+       "  'win_pct_home_under_cards_4.5_h',\n",
+       "  'won_corners_ratio_mean_a',\n",
+       "  'won_corners_ratio_mean_a',\n",
+       "  'won_corners_ratio_mean_diff',\n",
+       "  'won_corners_ratio_mean_diff',\n",
+       "  'won_corners_ratio_mean_h',\n",
+       "  'won_corners_ratio_mean_h',\n",
+       "  'won_corners_ratio_std_a',\n",
+       "  'won_corners_ratio_std_diff',\n",
+       "  'won_corners_ratio_std_h'])"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "common_feat = list(set([\"_\".join(x.split(\"_\")[:-1]) for x in selected_features]))\n",
+    "new_subset = [x for x in train_data.columns for c in common_feat if c in x]\n",
+    "\n",
+    "len(selected_features), len(new_subset), sorted(new_subset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\"\"\"Monkey patch feature_selection module.\"\"\"\n",
+    "from typing import List, Tuple\n",
+    "\n",
+    "# from feature_selection import FeatureSelection\n",
+    "\n",
+    "\n",
+    "def get_root_features(feature_list: List) -> Tuple[List, List]:\n",
+    "    \"\"\"Get the features related to teams, i.e., those ended with \"_a\", \"_h\", \"_diff\".\"\"\"\n",
+    "    # Conditions\n",
+    "    cond1 = lambda x: x.endswith(\"_h\")\n",
+    "    cond2 = lambda x: x.endswith(\"_a\")\n",
+    "    cond3 = lambda x: x.endswith(\"_diff\")\n",
+    "    # Features ended with _h, _a, _diff\n",
+    "    sublist = [c for c in feature_list if (cond1(c) or cond2(c) or cond3(c))]\n",
+    "    return sublist, list(set([\"_\".join(c.split(\"_\")[:-1]) for c in sublist]))\n",
+    "\n",
+    "\n",
+    "def append_list(root_features: List, og_columns: List):\n",
+    "    \"\"\"Append suffixes to the incoming feature labels.\"\"\"\n",
+    "    # Create empty list\n",
+    "    dressed_list = list()\n",
+    "    # Add  suffixes to each root label\n",
+    "    # dressed_list = [c for root in root_features for c in df_columns if c.startswith(root)]\n",
+    "    for c in root_features:\n",
+    "        dressed_list.append(f\"{c}_h\")\n",
+    "        dressed_list.append(f\"{c}_a\")\n",
+    "        dressed_list.append(f\"{c}_diff\")\n",
+    "    dressed_list = [c for c in dressed_list if c in og_columns]\n",
+    "    # return list(set(dressed_list))\n",
+    "    return dressed_list\n",
+    "\n",
+    "\n",
+    "def massage_feat_list(feature_list: List, og_columns: List):\n",
+    "    \"\"\"\n",
+    "    Add team features to the incoming list.\n",
+    "\n",
+    "    Given a feature list, this function adds the full trio of team\n",
+    "    features (*_h, *_a, *_diff) to those already present that are\n",
+    "    related to team statistics.\n",
+    "    \"\"\"\n",
+    "    feature_list = feature_list.copy()\n",
+    "    # Get list of match features and root\n",
+    "    sublist, root = get_root_features(feature_list)\n",
+    "    # Get features not related to teams\n",
+    "    no_team = list(set(feature_list) - set(sublist))\n",
+    "    # Add suffixes to roots\n",
+    "    dressed = append_list(root_features=root, og_columns=og_columns.copy())\n",
+    "    # return list(set(no_team + dressed))\n",
+    "    return no_team + dressed\n",
+    "\n",
+    "\n",
+    "def finalize_list(feature_list: List, df_columns: List):\n",
+    "    \"\"\"\n",
+    "    Complete the feature_list returned by the selection module.\n",
+    "\n",
+    "    Once the number of features has been reduced to a subset of\n",
+    "    relevant features, this function completes the list by adding\n",
+    "    team features to those already present.\n",
+    "\n",
+    "    If a team feature is present, but the set only contains the one\n",
+    "    related to the home or away team, the function adds the corresponding\n",
+    "    feature related to the difference between teams, i.e., team_feat_diff.\n",
+    "    \"\"\"\n",
+    "    feature_list = feature_list.copy()\n",
+    "    # Elements with _h, _a, _diff\n",
+    "    sublist, roots = get_root_features(feature_list)\n",
+    "    # List to be used as building block\n",
+    "    final_list = list(set(feature_list) - set(sublist))\n",
+    "    # Check for team features\n",
+    "    for root in roots:\n",
+    "        triplet = [f\"{root}_h\", f\"{root}_a\", f\"{root}_diff\"]\n",
+    "        subset = [c for tri in triplet for c in sublist if c == tri]\n",
+    "        if len(subset) < 2 and not subset[0].endswith(\"_diff\"):\n",
+    "            subset.append(f\"{root}_diff\")\n",
+    "        final_list = final_list + subset\n",
+    "    final_list = [c for c in final_list if c in df_columns]\n",
+    "    return final_list\n",
+    "\n",
+    "\n",
+    "def patch_repeat_pipeline(self):\n",
+    "    \"\"\"Iterate over the process to create the feature list and repeat it self.repeat times (PATCH).\"\"\"\n",
+    "    i = 0\n",
+    "    while len(self.feature_list) > self.target_features:\n",
+    "        i += 1\n",
+    "        print(f\"{i} iteration\")\n",
+    "        # Call iteration\n",
+    "        self.create_feature_list()\n",
+    "        self.feature_list = massage_feat_list(\n",
+    "            feature_list=self.feature_list, og_columns=self.x_df.columns\n",
+    "        )\n",
+    "        if len(self.feature_list) <= 1:\n",
+    "            break\n",
+    "    return finalize_list(feature_list=self.feature_list, df_columns=self.x_df.columns)\n",
+    "\n",
+    "def patch_train_model(self):\n",
+    "        \"\"\"Preprocess the data and select self.number_models top models.\"\"\"\n",
+    "        # Selected dataset\n",
+    "        selected_cols = self.feature_list + [self.target]\n",
+    "        train_data = self.dataset[selected_cols] if self.x_df.empty else self.x_df[selected_cols]\n",
+    "        # Numeric features\n",
+    "        self.setup_kwargs[\"numeric_features\"] = [\n",
+    "            c for c in self.numeric_features if c in self.feature_list\n",
+    "        ]\n",
+    "        # Ignore features\n",
+    "        self.setup_kwargs[\"ignore_features\"] = [\n",
+    "            c for c in self.ignore_features if c in self.feature_list\n",
+    "        ]\n",
+    "        self.setup_kwargs[\"feature_interaction\"]: False\n",
+    "        # Initialize pycaret setup\n",
+    "        setup(data=train_data, target=self.target, **self.setup_kwargs)\n",
+    "        # Get train dataset and preprocessed dataframe\n",
+    "        self.x_train = get_config(\"X_train\")\n",
+    "        if self.x_df.empty:  # TODO change x_df by dataset and add flag?\n",
+    "            self.x_df = pd.concat([get_config(\"X\"), get_config(\"y\")], axis=1)\n",
+    "            self.setup_kwargs[\"preprocess\"] = False  # Turn off preprocessing\n",
+    "        # Compare models\n",
+    "        self.top_models = self._training_function(**self._args)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING:param: Setting non-Parameter class attribute FeatureSelection.train_model = <function patch_train_model at 0x000001D70BABA670> \n"
+     ]
+    }
+   ],
+   "source": [
+    "FeatureSelection.train_model = patch_train_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING:param: Setting non-Parameter class attribute FeatureSelection.repeat_pipeline = <function patch_repeat_pipeline at 0x000001D716E4D430> \n"
+     ]
+    }
+   ],
+   "source": [
+    "FeatureSelection.repeat_pipeline = patch_repeat_pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1 iteration\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5186</td>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5299</td>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5000</td>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5286</td>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5139</td>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0378</td>\n",
+       "                        <td id=\"T_cf5f85a7_84fb_11ec_b12e_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0378</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706fc9880>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_cae931d1_84fe_11ec_be9a_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706fc9400>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_cb14a8b0_84fe_11ec_88d4_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706d9b670>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_cb40266d_84fe_11ec_b718_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706d9b610>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_cb679eab_84fe_11ec_a77e_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706d04ee0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2 iteration\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5213</td>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5413</td>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4973</td>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5318</td>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5140</td>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0435</td>\n",
+       "                        <td id=\"T_d0aa48a7_84fe_11ec_902b_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0436</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706ec6850>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5337</td>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5494</td>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5243</td>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5434</td>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5337</td>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0677</td>\n",
+       "                        <td id=\"T_eba58a6d_84ff_11ec_a3fe_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0677</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d7071fd850>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5227</td>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5408</td>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4973</td>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5333</td>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5147</td>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0463</td>\n",
+       "                        <td id=\"T_ebc7b90c_84ff_11ec_a5c9_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0464</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d7071fd910>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5227</td>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5408</td>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4973</td>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5333</td>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5147</td>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0463</td>\n",
+       "                        <td id=\"T_ebeaab0b_84ff_11ec_8b36_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0464</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706ec6340>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5213</td>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5413</td>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4973</td>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5318</td>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5140</td>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0435</td>\n",
+       "                        <td id=\"T_ec0ab6e9_84ff_11ec_b449_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0436</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d7071fd3d0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "3 iteration\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5378</td>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5566</td>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5351</td>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5470</td>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5410</td>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0757</td>\n",
+       "                        <td id=\"T_efe901e6_84ff_11ec_99b8_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0757</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d7072de9d0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5378</td>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5566</td>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5351</td>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5470</td>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5410</td>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0757</td>\n",
+       "                        <td id=\"T_e6a59ee0_8500_11ec_988c_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0757</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d7071e9490>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5420</td>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5476</td>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5405</td>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5510</td>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5457</td>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0839</td>\n",
+       "                        <td id=\"T_e6bee4e3_8500_11ec_812e_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0839</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d7071e9580>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5420</td>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5476</td>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5405</td>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5510</td>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5457</td>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0839</td>\n",
+       "                        <td id=\"T_e6d7a919_8500_11ec_aa5d_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0839</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706ec6310>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5447</td>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5481</td>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5459</td>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5534</td>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5497</td>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0893</td>\n",
+       "                        <td id=\"T_e6f2d27c_8500_11ec_9dc4_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0893</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d70697cd90>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "4 iteration\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5543</td>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5732</td>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5703</td>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5612</td>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5657</td>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.1081</td>\n",
+       "                        <td id=\"T_e96811f9_8500_11ec_b9a2_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.1081</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706d38c70>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5488</td>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5709</td>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5595</td>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5565</td>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5580</td>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0973</td>\n",
+       "                        <td id=\"T_b2c6e60a_8501_11ec_8f22_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0973</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d76dc64eb0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5488</td>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5709</td>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5595</td>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5565</td>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5580</td>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0973</td>\n",
+       "                        <td id=\"T_b2e2abf0_8501_11ec_9188_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0973</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d70731c340>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5488</td>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5709</td>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5595</td>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5565</td>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5580</td>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0973</td>\n",
+       "                        <td id=\"T_b2fc9d18_8501_11ec_9a13_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0973</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d70731c6d0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5502</td>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5699</td>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5676</td>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5570</td>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5622</td>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0998</td>\n",
+       "                        <td id=\"T_b3199b7f_8501_11ec_b080_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0998</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1d706ec6250>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "metric_param = {\n",
+    "        \"Accuracy\": 0.1,\n",
+    "        \"AUC\": 0.1,\n",
+    "        \"Recall\": 0.1,\n",
+    "        \"Precision\": 0.1,\n",
+    "        \"F1\": 0.1,\n",
+    "        \"Kappa\": -1.0,\n",
+    "        \"MCC\": -1.0,\n",
+    "    }\n",
+    "feat_sel = FeatureSelection(target=target,\n",
+    "                            dataset=train_data.dropna(),#[list(set(new_subset+new_feat+[target]))],\n",
+    "                            target_features=500,\n",
+    "                            filter_metrics=metric_param,\n",
+    "                            include=[\"lr\"],\n",
+    "                            setup_kwargs=setup_kwargs,\n",
+    "                            optimize=True,\n",
+    "                            opt_list=[\"AUC\" , \"Accuracy\" , \"Precision\" , \"Recall\"],\n",
+    "                            number_features = 0.56\n",
+    "                           )\n",
+    "triplet_selected_features = feat_sel.repeat_pipeline()\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "456"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(triplet_selected_features\n",
+    "   )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(145,\n",
+       " 276,\n",
+       " ['ATT_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'ATT_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'accurate_cross_nocorner_std_a',\n",
+       "  'accurate_cross_nocorner_std_diff',\n",
+       "  'accurate_cross_nocorner_std_h',\n",
+       "  'accurate_cross_std_a',\n",
+       "  'accurate_cross_std_diff',\n",
+       "  'accurate_cross_std_h',\n",
+       "  'accurate_goal_kicks_std_a',\n",
+       "  'accurate_goal_kicks_std_diff',\n",
+       "  'accurate_goal_kicks_std_h',\n",
+       "  'accurate_keeper_sweeper_mean_a',\n",
+       "  'accurate_keeper_sweeper_mean_diff',\n",
+       "  'accurate_keeper_sweeper_mean_h',\n",
+       "  'accurate_keeper_sweeper_std_a',\n",
+       "  'accurate_keeper_sweeper_std_diff',\n",
+       "  'accurate_keeper_sweeper_std_h',\n",
+       "  'accurate_launches_mean_a',\n",
+       "  'accurate_launches_mean_diff',\n",
+       "  'accurate_launches_mean_h',\n",
+       "  'accurate_layoffs_mean_a',\n",
+       "  'accurate_layoffs_mean_diff',\n",
+       "  'accurate_layoffs_mean_h',\n",
+       "  'accurate_layoffs_std_a',\n",
+       "  'accurate_layoffs_std_diff',\n",
+       "  'accurate_layoffs_std_h',\n",
+       "  'accurate_through_ball_mean_a',\n",
+       "  'accurate_through_ball_mean_diff',\n",
+       "  'accurate_through_ball_mean_h',\n",
+       "  'accurate_throws_std_a',\n",
+       "  'accurate_throws_std_diff',\n",
+       "  'accurate_throws_std_h',\n",
+       "  'att_freekick_miss_mean_a',\n",
+       "  'att_freekick_miss_mean_diff',\n",
+       "  'att_freekick_miss_mean_h',\n",
+       "  'att_freekick_miss_std_a',\n",
+       "  'att_freekick_miss_std_diff',\n",
+       "  'att_freekick_miss_std_h',\n",
+       "  'att_goal_low_centre_mean_a',\n",
+       "  'att_goal_low_centre_mean_diff',\n",
+       "  'att_goal_low_centre_mean_h',\n",
+       "  'att_goal_low_centre_std_a',\n",
+       "  'att_goal_low_centre_std_diff',\n",
+       "  'att_goal_low_centre_std_h',\n",
+       "  'att_goal_low_right_mean_a',\n",
+       "  'att_goal_low_right_mean_diff',\n",
+       "  'att_goal_low_right_mean_h',\n",
+       "  'att_hd_goal_mean_a',\n",
+       "  'att_hd_goal_mean_diff',\n",
+       "  'att_hd_goal_mean_h',\n",
+       "  'att_hd_target_mean_a',\n",
+       "  'att_hd_target_mean_diff',\n",
+       "  'att_hd_target_mean_h',\n",
+       "  'att_lf_goal_mean_a',\n",
+       "  'att_lf_goal_mean_diff',\n",
+       "  'att_lf_goal_mean_h',\n",
+       "  'att_lf_goal_std_a',\n",
+       "  'att_lf_goal_std_diff',\n",
+       "  'att_lf_goal_std_h',\n",
+       "  'att_miss_high_left_std_a',\n",
+       "  'att_miss_high_left_std_diff',\n",
+       "  'att_miss_high_left_std_h',\n",
+       "  'att_obox_blocked_mean_a',\n",
+       "  'att_obox_blocked_mean_diff',\n",
+       "  'att_obox_blocked_mean_h',\n",
+       "  'att_pen_goal_mean_a',\n",
+       "  'att_pen_goal_mean_diff',\n",
+       "  'att_pen_goal_mean_h',\n",
+       "  'attempts_conceded_ibox_std_a',\n",
+       "  'attempts_conceded_ibox_std_diff',\n",
+       "  'attempts_conceded_ibox_std_h',\n",
+       "  'attempts_conceded_obox_std_a',\n",
+       "  'attempts_conceded_obox_std_diff',\n",
+       "  'attempts_conceded_obox_std_h',\n",
+       "  'attempts_obox_mean_a',\n",
+       "  'attempts_obox_mean_diff',\n",
+       "  'attempts_obox_mean_h',\n",
+       "  'big_chance_created_mean_a',\n",
+       "  'big_chance_created_mean_diff',\n",
+       "  'big_chance_created_mean_h',\n",
+       "  'big_chance_created_std_a',\n",
+       "  'big_chance_created_std_diff',\n",
+       "  'big_chance_created_std_h',\n",
+       "  'big_chance_missed_std_a',\n",
+       "  'big_chance_missed_std_diff',\n",
+       "  'big_chance_missed_std_h',\n",
+       "  'blocked_cross_mean_a',\n",
+       "  'blocked_cross_mean_diff',\n",
+       "  'blocked_cross_mean_h',\n",
+       "  'contentious_decision_mean_a',\n",
+       "  'contentious_decision_mean_diff',\n",
+       "  'contentious_decision_mean_h',\n",
+       "  'contentious_decision_std_a',\n",
+       "  'contentious_decision_std_diff',\n",
+       "  'contentious_decision_std_h',\n",
+       "  'defender_goals_mean_a',\n",
+       "  'defender_goals_mean_a',\n",
+       "  'defender_goals_mean_diff',\n",
+       "  'defender_goals_mean_diff',\n",
+       "  'defender_goals_mean_h',\n",
+       "  'defender_goals_mean_h',\n",
+       "  'diving_save_std_a',\n",
+       "  'diving_save_std_diff',\n",
+       "  'diving_save_std_h',\n",
+       "  'effective_head_clearance_mean_a',\n",
+       "  'effective_head_clearance_mean_diff',\n",
+       "  'effective_head_clearance_mean_h',\n",
+       "  'forward_goals_mean_a',\n",
+       "  'forward_goals_mean_diff',\n",
+       "  'forward_goals_mean_h',\n",
+       "  'goal_assist_deadball_std_a',\n",
+       "  'goal_assist_deadball_std_diff',\n",
+       "  'goal_assist_deadball_std_h',\n",
+       "  'goal_assist_intent_norm_mean_a',\n",
+       "  'goal_assist_intent_norm_mean_diff',\n",
+       "  'goal_assist_intent_norm_mean_h',\n",
+       "  'goal_assist_intentional_mean_a',\n",
+       "  'goal_assist_intentional_mean_diff',\n",
+       "  'goal_assist_intentional_mean_h',\n",
+       "  'goal_assist_intentional_std_a',\n",
+       "  'goal_assist_intentional_std_diff',\n",
+       "  'goal_assist_intentional_std_h',\n",
+       "  'goal_assist_mean_a',\n",
+       "  'goal_assist_mean_diff',\n",
+       "  'goal_assist_mean_h',\n",
+       "  'goal_assist_openplay_std_a',\n",
+       "  'goal_assist_openplay_std_diff',\n",
+       "  'goal_assist_openplay_std_h',\n",
+       "  'goal_assist_setplay_mean_a',\n",
+       "  'goal_assist_setplay_mean_diff',\n",
+       "  'goal_assist_setplay_mean_h',\n",
+       "  'goals_2t_mean_a',\n",
+       "  'goals_2t_mean_diff',\n",
+       "  'goals_2t_mean_h',\n",
+       "  'goals_mean_a',\n",
+       "  'goals_mean_diff',\n",
+       "  'goals_mean_h',\n",
+       "  'goals_openplay_mean_a',\n",
+       "  'goals_openplay_mean_diff',\n",
+       "  'goals_openplay_mean_h',\n",
+       "  'high_to_low_goals_mean_a',\n",
+       "  'high_to_low_goals_mean_diff',\n",
+       "  'high_to_low_goals_mean_h',\n",
+       "  'interceptions_in_box_std_a',\n",
+       "  'interceptions_in_box_std_diff',\n",
+       "  'interceptions_in_box_std_h',\n",
+       "  'left_div_right_foot_goals_mean_a',\n",
+       "  'left_div_right_foot_goals_mean_a',\n",
+       "  'left_div_right_foot_goals_mean_diff',\n",
+       "  'left_div_right_foot_goals_mean_diff',\n",
+       "  'left_div_right_foot_goals_mean_h',\n",
+       "  'left_div_right_foot_goals_mean_h',\n",
+       "  'lost_corners_mean_a',\n",
+       "  'lost_corners_mean_diff',\n",
+       "  'lost_corners_mean_h',\n",
+       "  'midfielder_goals_mean_a',\n",
+       "  'midfielder_goals_mean_diff',\n",
+       "  'midfielder_goals_mean_h',\n",
+       "  'no_foot_goals_ratio_mean_a',\n",
+       "  'no_foot_goals_ratio_mean_diff',\n",
+       "  'no_foot_goals_ratio_mean_h',\n",
+       "  'no_foot_goals_ratio_std_a',\n",
+       "  'no_foot_goals_ratio_std_diff',\n",
+       "  'no_foot_goals_ratio_std_h',\n",
+       "  'odd_ratio_over_corners_11.5',\n",
+       "  'odd_ratio_under_corners_10.5',\n",
+       "  'odds_away_over_cards_5.5_a',\n",
+       "  'odds_away_over_cards_5.5_diff',\n",
+       "  'odds_away_over_cards_5.5_h',\n",
+       "  'odds_home_under_cards_5.5_a',\n",
+       "  'odds_home_under_cards_5.5_diff',\n",
+       "  'odds_home_under_cards_5.5_h',\n",
+       "  'odds_home_under_goals_0.5_a',\n",
+       "  'odds_home_under_goals_0.5_diff',\n",
+       "  'odds_home_under_goals_0.5_h',\n",
+       "  'ontarget_att_assist_mean_a',\n",
+       "  'ontarget_att_assist_mean_diff',\n",
+       "  'ontarget_att_assist_mean_h',\n",
+       "  'own_goals_mean_a',\n",
+       "  'own_goals_mean_diff',\n",
+       "  'own_goals_mean_h',\n",
+       "  'penalty_conceded_mean_a',\n",
+       "  'penalty_conceded_mean_diff',\n",
+       "  'penalty_conceded_mean_h',\n",
+       "  'penalty_faced_std_a',\n",
+       "  'penalty_faced_std_diff',\n",
+       "  'penalty_faced_std_h',\n",
+       "  'penalty_save_std_a',\n",
+       "  'penalty_save_std_diff',\n",
+       "  'penalty_save_std_h',\n",
+       "  'poss_won_att_3rd_std_a',\n",
+       "  'poss_won_att_3rd_std_diff',\n",
+       "  'poss_won_att_3rd_std_h',\n",
+       "  'ratio_over_corners_11.5_a',\n",
+       "  'ratio_over_corners_11.5_diff',\n",
+       "  'ratio_over_corners_11.5_h',\n",
+       "  'ratio_under_corners_10.5_a',\n",
+       "  'ratio_under_corners_10.5_diff',\n",
+       "  'ratio_under_corners_10.5_h',\n",
+       "  'red_card_std_a',\n",
+       "  'red_card_std_diff',\n",
+       "  'red_card_std_h',\n",
+       "  'rescinded_red_card_std_a',\n",
+       "  'rescinded_red_card_std_diff',\n",
+       "  'rescinded_red_card_std_h',\n",
+       "  'right_to_left_goals_mean_a',\n",
+       "  'right_to_left_goals_mean_diff',\n",
+       "  'right_to_left_goals_mean_h',\n",
+       "  'saves_std_a',\n",
+       "  'saves_std_diff',\n",
+       "  'saves_std_h',\n",
+       "  'shots_mul_goals_mean_a',\n",
+       "  'shots_mul_goals_mean_a',\n",
+       "  'shots_mul_goals_mean_diff',\n",
+       "  'shots_mul_goals_mean_diff',\n",
+       "  'shots_mul_goals_mean_h',\n",
+       "  'shots_mul_goals_mean_h',\n",
+       "  'shots_std_a',\n",
+       "  'shots_std_diff',\n",
+       "  'shots_std_h',\n",
+       "  'total_fastbreak_std_a',\n",
+       "  'total_fastbreak_std_diff',\n",
+       "  'total_fastbreak_std_h',\n",
+       "  'total_keeper_sweeper_mean_a',\n",
+       "  'total_keeper_sweeper_mean_diff',\n",
+       "  'total_keeper_sweeper_mean_h',\n",
+       "  'total_keeper_sweeper_std_a',\n",
+       "  'total_keeper_sweeper_std_diff',\n",
+       "  'total_keeper_sweeper_std_h',\n",
+       "  'total_launches_mean_a',\n",
+       "  'total_launches_mean_diff',\n",
+       "  'total_launches_mean_h',\n",
+       "  'total_launches_std_a',\n",
+       "  'total_launches_std_diff',\n",
+       "  'total_launches_std_h',\n",
+       "  'total_layoffs_std_a',\n",
+       "  'total_layoffs_std_diff',\n",
+       "  'total_layoffs_std_h',\n",
+       "  'total_red_card_std_a',\n",
+       "  'total_red_card_std_diff',\n",
+       "  'total_red_card_std_h',\n",
+       "  'total_through_ball_mean_a',\n",
+       "  'total_through_ball_mean_diff',\n",
+       "  'total_through_ball_mean_h',\n",
+       "  'total_through_ball_std_a',\n",
+       "  'total_through_ball_std_diff',\n",
+       "  'total_through_ball_std_h',\n",
+       "  'total_throws_std_a',\n",
+       "  'total_throws_std_diff',\n",
+       "  'total_throws_std_h',\n",
+       "  'total_win_pct_over_goals_2.5_a',\n",
+       "  'total_win_pct_over_goals_2.5_diff',\n",
+       "  'total_win_pct_over_goals_2.5_h',\n",
+       "  'total_win_pct_under_goals_1.5_a',\n",
+       "  'total_win_pct_under_goals_1.5_diff',\n",
+       "  'total_win_pct_under_goals_1.5_h',\n",
+       "  'total_win_pct_under_goals_3.5_a',\n",
+       "  'total_win_pct_under_goals_3.5_diff',\n",
+       "  'total_win_pct_under_goals_3.5_h',\n",
+       "  'win_pct_away_over_goals_2.5_a',\n",
+       "  'win_pct_away_over_goals_2.5_diff',\n",
+       "  'win_pct_away_over_goals_2.5_h',\n",
+       "  'win_pct_home_over_cards_6.5_a',\n",
+       "  'win_pct_home_over_cards_6.5_diff',\n",
+       "  'win_pct_home_over_cards_6.5_h',\n",
+       "  'win_pct_home_over_goals_2.5_a',\n",
+       "  'win_pct_home_over_goals_2.5_diff',\n",
+       "  'win_pct_home_over_goals_2.5_h',\n",
+       "  'win_pct_home_under_cards_4.5_a',\n",
+       "  'win_pct_home_under_cards_4.5_diff',\n",
+       "  'win_pct_home_under_cards_4.5_h',\n",
+       "  'won_corners_ratio_std_a',\n",
+       "  'won_corners_ratio_std_diff',\n",
+       "  'won_corners_ratio_std_h'])"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "common_feat = list(set([\"_\".join(x.split(\"_\")[:-1]) for x in selected_features]))\n",
+    "new_subset = [x for x in train_data.columns for c in common_feat if c in x]\n",
+    "\n",
+    "len(selected_features), len(new_subset), sorted(new_subset)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### new imports "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
+    "from ml_bets.constants import FEATURES_PATH\n",
+    "from ml_bets.features.features import Features\n",
+    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
+    "#from featsel.feature_selection import FeatureSelection\n",
+    "\n",
+    "from pycaret.utils import check_metric\n",
+    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
+    "    create_model,\n",
+    "    finalize_model,\n",
+    "    optimize_threshold,              \n",
+    "    save_model,\n",
+    "    compare_models, \n",
+    "    evaluate_model,\n",
+    "    get_config,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    "    predict_model,\n",
+    ")\n",
+    "\n",
+    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
+    "from ml_bets.research.datasets import create_dataset\n",
+    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ml_bets.research.tips import calibrate_tips, tips_from_model, combine_tips, predict_dataset, compose_tips, get_tip_probs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ix = np.logical_and(feats.matches[\"date\"].dt.month > 10,\n",
+    "                    feats.matches[\"competition\"].isin({\"mexican_primera\", 'us_major_league_soccer'}))\n",
+    "index = feats.matches[~ix].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_feats = ['acc_cross_nocorner_pct_std_a',\n",
+    " 'acc_cross_nocorner_pct_std_diff',\n",
+    " 'acc_cross_nocorner_pct_std_h',\n",
+    " 'accurate_flick_on_mean_h',\n",
+    " 'accurate_flick_on_mean_a',\n",
+    " 'accurate_freekick_cross_mean_a',\n",
+    " 'accurate_freekick_cross_mean_diff',\n",
+    " 'accurate_freekick_cross_mean_h',\n",
+    " 'accurate_goal_kicks_mean_diff',\n",
+    " 'accurate_goal_kicks_std_a',\n",
+    " 'accurate_goal_kicks_std_diff',\n",
+    " 'accurate_goal_kicks_std_h',\n",
+    " 'accurate_keeper_sweeper_mean_a',\n",
+    " 'accurate_keeper_sweeper_std_diff',\n",
+    " 'accurate_keeper_sweeper_std_h',\n",
+    " 'accurate_pass_std_a',\n",
+    " 'accurate_pass_std_h',\n",
+    " 'accurate_through_ball_mean_diff',\n",
+    " 'att_assist_openplay_std_a',\n",
+    " 'att_assist_openplay_std_h',\n",
+    " 'att_cmiss_left_mean_a',\n",
+    " 'att_cmiss_left_mean_h',\n",
+    " 'att_cmiss_left_std_a',\n",
+    " 'att_cmiss_left_std_diff',\n",
+    " 'att_cmiss_left_std_h',\n",
+    " 'att_goal_high_centre_mean_diff',\n",
+    " 'att_goal_high_centre_std_diff',\n",
+    " 'att_goal_high_right_std_a',\n",
+    " 'att_goal_low_centre_mean_diff',\n",
+    " 'att_goal_low_centre_mean_h',\n",
+    " 'att_goal_low_centre_std_a',\n",
+    " 'att_goal_low_centre_std_h',\n",
+    " 'att_hd_goal_mean_a',\n",
+    " 'att_hd_goal_mean_h',\n",
+    " 'att_hd_target_mean_a',\n",
+    " 'att_hd_target_mean_h',\n",
+    " 'att_ibox_own_goal_mean_diff',\n",
+    " 'att_ibox_own_goal_std_h',\n",
+    " 'att_ibox_own_goal_std_a',\n",
+    " 'att_miss_high_mean_a',\n",
+    " 'att_miss_high_mean_h',\n",
+    " 'att_miss_high_right_mean_diff',\n",
+    " 'att_obxd_right_mean_diff',\n",
+    " 'att_obxd_right_std_diff',\n",
+    " 'att_one_on_one_mean_h',\n",
+    " 'att_one_on_one_mean_a',\n",
+    " 'att_one_on_one_std_diff',\n",
+    " 'att_one_on_one_std_h',\n",
+    " 'att_post_high_std_a',\n",
+    " 'att_post_high_std_h',\n",
+    " 'att_post_right_mean_a',\n",
+    " 'att_post_right_mean_h',\n",
+    " 'attempts_ibox_std_a',\n",
+    " 'attempts_ibox_std_h',\n",
+    " 'backward_pass_mean_diff',\n",
+    " 'backward_pass_mean_h',\n",
+    " 'big_chance_created_mean_h',\n",
+    " 'big_chance_created_std_diff',\n",
+    " 'clean_sheet_std_diff',\n",
+    " 'contentious_decision_mean_a',\n",
+    " 'contentious_decision_mean_diff',\n",
+    " 'contentious_decision_std_a',\n",
+    " 'contentious_decision_std_h',\n",
+    " 'duel_won_pct_mean_diff',\n",
+    " 'effective_clearance_mean_diff',\n",
+    " 'error_lead_to_goal_mean_diff',\n",
+    " 'error_lead_to_goal_std_diff',\n",
+    " 'first_yellow_card_1t_mean_h',\n",
+    " 'first_yellow_card_1t_mean_a',\n",
+    " 'foul_throw_in_std_diff',\n",
+    " 'fouled_final_third_mean_diff',\n",
+    " 'goal_assist_deadball_mean_diff',\n",
+    " 'goal_assist_openplay_std_diff',\n",
+    " 'goal_assist_openplay_std_h',\n",
+    " 'goal_assist_setplay_std_a',\n",
+    " 'goal_assist_setplay_std_diff',\n",
+    " 'goal_assist_setplay_std_h',\n",
+    " 'goal_assist_std_diff',\n",
+    " 'goals_mean_diff',\n",
+    " 'goals_openplay_std_h',\n",
+    " 'goals_openplay_std_a',\n",
+    " 'good_high_claim_mean_diff',\n",
+    " 'high_to_low_goals_mean_a',\n",
+    " 'high_to_low_goals_mean_diff',\n",
+    " 'high_to_low_goals_std_a',\n",
+    " 'high_to_low_goals_std_h',\n",
+    " 'imp_prob_under_goals_0.5_h',\n",
+    " 'imp_prob_under_goals_2.5_diff',\n",
+    " 'interception_mean_a',\n",
+    " 'interception_mean_diff',\n",
+    " 'interception_mean_h',\n",
+    " 'interceptions_in_box_std_a',\n",
+    " 'interceptions_in_box_std_diff',\n",
+    " 'interceptions_in_box_std_h',\n",
+    " 'last_man_tackle_mean_a',\n",
+    " 'last_man_tackle_mean_diff',\n",
+    " 'left_div_right_foot_goals_std_diff',\n",
+    " 'leftside_pass_mean_a',\n",
+    " 'leftside_pass_mean_h',\n",
+    " 'leftside_pass_std_a',\n",
+    " 'long_pass_own_to_opp_mean_diff',\n",
+    " 'long_pass_own_to_opp_mean_h',\n",
+    " 'no_foot_goals_ratio_std_a',\n",
+    " 'no_foot_goals_ratio_std_diff',\n",
+    " 'no_foot_goals_ratio_std_h',\n",
+    " 'odd_ratio_under_corners_10.5',\n",
+    " 'odd_ratio_under_corners_8.5',\n",
+    " 'odds_home_under_both_score_h',\n",
+    " 'odds_away_under_both_score_a',\n",
+    " 'odds_away_under_goals_4.5_a',\n",
+    " 'odds_home_under_goals_4.5_h',\n",
+    " 'odds_away_over_goals_4.5_a',\n",
+    " 'odds_home_over_goals_4.5_h',\n",
+    " 'odds_home_under_goals_0.5_diff',\n",
+    " 'odds_home_under_goals_0.5_h',\n",
+    " 'own_goals_std_h',\n",
+    " 'own_goals_std_a',\n",
+    " 'pen_goals_conceded_mean_diff',\n",
+    " 'penalty_faced_std_a',\n",
+    " 'penalty_won_std_a',\n",
+    " 'penalty_faced_std_h',\n",
+    " 'penalty_won_std_h',\n",
+    " 'poss_won_att_3rd_std_a',\n",
+    " 'poss_won_att_3rd_std_diff',\n",
+    " 'poss_won_att_3rd_std_h',\n",
+    " 'post_scoring_att_std_a',\n",
+    " 'post_scoring_att_std_h',\n",
+    " 'prob_squared_under_goals_2.5',\n",
+    " 'pts_dropped_winning_pos_mean_a',\n",
+    " 'pts_dropped_winning_pos_std_diff',\n",
+    " 'pts_dropped_winning_pos_std_h',\n",
+    " 'ratio_over_goals_2.5_a',\n",
+    " 'ratio_over_goals_2.5_h',\n",
+    " 'ratio_under_goals_2.5_a',\n",
+    " 'ratio_under_goals_2.5_diff',\n",
+    " 'raw_prob_over_goals_1.5',\n",
+    " 'raw_prob_over_goals_2.5',\n",
+    " 'raw_prob_over_goals_3.5',\n",
+    " 'raw_prob_under_goals_1.5',\n",
+    " 'raw_prob_under_goals_2.5',\n",
+    " 'raw_prob_under_goals_3.5',\n",
+    " 'raw_prob_under_goals_4.5',\n",
+    " 'red_card_1t_mean_a',\n",
+    " 'red_card_1t_mean_diff',\n",
+    " 'red_card_1t_mean_h',\n",
+    " 'red_card_2t_mean_a',\n",
+    " 'red_card_2t_mean_diff',\n",
+    " 'red_card_mean_h',\n",
+    " 'red_card_mean_a',\n",
+    " 'red_card_std_diff',\n",
+    " 'right_to_left_goals_mean_diff',\n",
+    " 'right_to_left_goals_std_diff',\n",
+    " 'rightside_pass_div_leftside_pass_mean_a',\n",
+    " 'rightside_pass_div_leftside_pass_mean_h',\n",
+    " 'second_yellow_mean_h',\n",
+    " 'shots_mul_goals_std_h',\n",
+    " 'second_yellow_mean_a',\n",
+    " 'shots_mul_goals_std_a',\n",
+    " 'successful_final_third_passes_mean_a',\n",
+    " 'successful_final_third_passes_mean_diff',\n",
+    " 'successful_final_third_passes_std_a',\n",
+    " 'successful_final_third_passes_std_diff',\n",
+    " 'successful_put_through_std_a',\n",
+    " 'successful_put_through_std_diff',\n",
+    " 'successful_put_through_std_h',\n",
+    " 'total_clearance_mean_a',\n",
+    " 'total_clearance_mean_h',\n",
+    " 'total_fastbreak_std_diff',\n",
+    " 'total_high_claim_mean_a',\n",
+    " 'total_high_claim_mean_diff',\n",
+    " 'total_high_claim_mean_h',\n",
+    " 'total_keeper_sweeper_mean_a',\n",
+    " 'total_keeper_sweeper_mean_h',\n",
+    " 'total_launches_std_h',\n",
+    " 'total_launches_std_a',\n",
+    " 'total_red_card_mean_h',\n",
+    " 'total_red_card_mean_a',\n",
+    " 'total_red_card_std_diff',\n",
+    " 'total_throws_std_a',\n",
+    " 'total_throws_std_diff',\n",
+    " 'total_win_pct_over_goals_1.5_a',\n",
+    " 'total_win_pct_over_goals_1.5_h',\n",
+    " 'total_win_pct_over_goals_2.5_a',\n",
+    " 'total_win_pct_over_goals_2.5_h',\n",
+    " 'total_win_pct_over_goals_3.5_a',\n",
+    " 'total_win_pct_over_goals_3.5_h',\n",
+    " 'total_win_pct_under_goals_1.5_a',\n",
+    " 'total_win_pct_under_goals_1.5_h',\n",
+    " 'total_win_pct_under_goals_2.5_a',\n",
+    " 'total_win_pct_under_goals_2.5_h',\n",
+    " 'total_win_pct_under_goals_3.5_a',\n",
+    " 'total_yel_card_std_a',\n",
+    " 'total_yel_card_std_diff',\n",
+    " 'total_yel_card_std_h',\n",
+    " 'win_pct_away_over_goals_2.5_a',\n",
+    " 'win_pct_away_over_goals_2.5_diff',\n",
+    " 'win_pct_away_under_goals_2.5_a',\n",
+    " 'win_pct_away_under_goals_2.5_diff',\n",
+    " 'win_pct_home_over_goals_2.5_a',\n",
+    " 'win_pct_home_over_goals_2.5_diff',\n",
+    " 'win_pct_home_over_goals_2.5_h',\n",
+    " 'win_pct_home_under_goals_2.5_diff',\n",
+    " 'win_pct_home_under_goals_2.5_h',\n",
+    " 'winner_mean_diff']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def setup_dataset(test_date):\n",
+    "    ds = create_dataset(target=target,\n",
+    "                        test_date=test_date,\n",
+    "                        features=feats,\n",
+    "                        odds_features=True,\n",
+    "                        include_std=True,\n",
+    "                        test_weeks=6,\n",
+    "                        ignore_features=IGNORE_FEATURES,\n",
+    "                        drop_future_matches=False,\n",
+    "                       )\n",
+    "    train_data = ds.train_data.copy()#[ds.train_data.index.map(lambda x: \"us_major_league_soccer\" not in x and \"mexican\" not in x)]\n",
+    "    train_data.drop(columns=[\"hour_rank\", \"hour_before_16\", \"is_weekend\"], inplace=True)\n",
+    "    train_data = train_data[train_data.index.isin(index)][list(set(test_feats)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    test_data = ds.test_set[ds.test_set.index.isin(index)][list(set(test_feats)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    val_data = ds.val_set[ds.val_set.index.isin(index)].copy()\n",
+    "    setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        test_data=test_data[train_data.columns.tolist()],#.dropna(),\n",
+    "        #numeric_features=[x for x in train_data.columns.tolist() if x != target],\n",
+    "        #custom_pipeline=loaded,\n",
+    "        #train_size=0.75,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "        normalize_method=\"robust\",\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.8,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        #ignore_features=ignore_features,\n",
+    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=4,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.5,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=False,\n",
+    "        pca_method=\"linear\",\n",
+    "        pca_components=30,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.05,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=True,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.05,\n",
+    "        fix_imbalance=True,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "    )\n",
+    "    _ = setup(data=train_data, target=target, **setup_kwargs)\n",
+    "    return train_data, test_data, val_data, ds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data, test_data, val_data, ds = setup_dataset(\"4-Dec-2021\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
+    "def train_ensemble():\n",
+    "    top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"et\", \"rf\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )\n",
+    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
+    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
+    "    blend = blend_models(cali)\n",
+    "    opti = tune_model(blend, optimize=\"Precision\", choose_better=True, n_iter=50, search_library=\"optuna\")\n",
+    "    return opti, cali, tuned_models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
+    "def train_linear_models():\n",
+    "    top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )\n",
+    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
+    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
+    "    return tuned_models, cali"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col0,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col1,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col2,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col3,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col4,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col5,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col6{\n",
+       "            background:  yellow;\n",
+       "        }</style><table id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Accuracy</th>        <th class=\"col_heading level0 col1\" >AUC</th>        <th class=\"col_heading level0 col2\" >Recall</th>        <th class=\"col_heading level0 col3\" >Prec.</th>        <th class=\"col_heading level0 col4\" >F1</th>        <th class=\"col_heading level0 col5\" >Kappa</th>        <th class=\"col_heading level0 col6\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >0.6414</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.6893</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6716</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.6507</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6610</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.2806</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.2807</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >0.5997</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.6586</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.6321</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.6124</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6221</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.1969</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.1970</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >0.6216</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.6679</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.6123</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.6442</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6278</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.2435</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.2439</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >0.6126</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.6475</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.6015</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.6345</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6175</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.2257</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.2260</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row4\" class=\"row_heading level0 row4\" >Mean</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col0\" class=\"data row4 col0\" >0.6188</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col1\" class=\"data row4 col1\" >0.6658</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col2\" class=\"data row4 col2\" >0.6294</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col3\" class=\"data row4 col3\" >0.6354</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col4\" class=\"data row4 col4\" >0.6321</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col5\" class=\"data row4 col5\" >0.2367</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col6\" class=\"data row4 col6\" >0.2369</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row5\" class=\"row_heading level0 row5\" >SD</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col0\" class=\"data row5 col0\" >0.0152</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col1\" class=\"data row5 col1\" >0.0154</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col2\" class=\"data row5 col2\" >0.0267</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col3\" class=\"data row5 col3\" >0.0145</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col4\" class=\"data row5 col4\" >0.0171</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col5\" class=\"data row5 col5\" >0.0303</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col6\" class=\"data row5 col6\" >0.0303</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7f44e43353d0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "tuned_linear, cali_linear = train_linear_models()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "    #T_08d08ce2_82b5_11ec_86da_31d766f3ddbd th {\n",
+       "          text-align: left;\n",
+       "    }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col7{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col5{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col8,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col8,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  lightgrey;\n",
+       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "            background-color:  lightgrey;\n",
+       "        }</style><table id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>        <th class=\"col_heading level0 col8\" >TT (Sec)</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >lr</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.6166</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6734</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5836</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6500</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.6081</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.2351</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.2400</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col8\" class=\"data row0 col8\" >0.3000</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >lda</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >Linear Discriminant Analysis</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.6166</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.6705</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.5892</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6475</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.6115</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.2347</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col7\" class=\"data row1 col7\" >0.2387</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col8\" class=\"data row1 col8\" >0.3025</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >ridge</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >Ridge Classifier</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.6166</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.5892</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6476</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.6115</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.2347</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col7\" class=\"data row2 col7\" >0.2387</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col8\" class=\"data row2 col8\" >0.2850</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >svm</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >SVM - Linear Kernel</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.5841</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.5441</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6157</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.5734</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.1708</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col7\" class=\"data row3 col7\" >0.1739</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col8\" class=\"data row3 col8\" >0.3050</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7fb80cbcb2e0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "SGDClassifier(alpha=0.10126692364605047, average=False, class_weight=None,\n",
+       "              early_stopping=False, epsilon=0.1, eta0=0.001022673152073415,\n",
+       "              fit_intercept=False, l1_ratio=0.2203376725525369,\n",
+       "              learning_rate='optimal', loss='hinge', max_iter=1000,\n",
+       "              n_iter_no_change=5, n_jobs=-1, penalty='l1', power_t=0.5,\n",
+       "              random_state=123, shuffle=True, tol=0.001,\n",
+       "              validation_fraction=0.1, verbose=0, warm_start=False)"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tuned_linear[3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6522b430b2a64128ac3789a8d93475f2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluate_model(tuned_linear[3])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a9de03ccce3a4047b615b146297be69e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluate_model(cali_linear[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "market=\"goals\"\n",
+    "cutoff = 2.5\n",
+    "tips = tips_from_model(model=cali_linear[0],\n",
+    "                test_data=test_data,\n",
+    "                features=feats,\n",
+    "                market=market,\n",
+    "                cutoff=cutoff,\n",
+    "                      )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>win</th>\n",
+       "      <th>profit</th>\n",
+       "      <th>exp_payoff</th>\n",
+       "      <th>exp_payoff_prec</th>\n",
+       "      <th>odds</th>\n",
+       "      <th>confidence</th>\n",
+       "      <th>count</th>\n",
+       "      <th>buenas</th>\n",
+       "      <th>model</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>bet_type</th>\n",
+       "      <th>match_week</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"7\" valign=\"top\">goals_over_2.5</th>\n",
+       "      <th>2021_19</th>\n",
+       "      <td>0.818182</td>\n",
+       "      <td>0.434545</td>\n",
+       "      <td>1.193888</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.743636</td>\n",
+       "      <td>0.683591</td>\n",
+       "      <td>11</td>\n",
+       "      <td>9</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_20</th>\n",
+       "      <td>0.700000</td>\n",
+       "      <td>0.144000</td>\n",
+       "      <td>1.087417</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.641000</td>\n",
+       "      <td>0.664270</td>\n",
+       "      <td>10</td>\n",
+       "      <td>7</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_21</th>\n",
+       "      <td>0.800000</td>\n",
+       "      <td>0.359000</td>\n",
+       "      <td>1.130386</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.685000</td>\n",
+       "      <td>0.671600</td>\n",
+       "      <td>10</td>\n",
+       "      <td>8</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_22</th>\n",
+       "      <td>0.571429</td>\n",
+       "      <td>0.040000</td>\n",
+       "      <td>1.130597</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.708571</td>\n",
+       "      <td>0.662800</td>\n",
+       "      <td>7</td>\n",
+       "      <td>4</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_23</th>\n",
+       "      <td>0.500000</td>\n",
+       "      <td>-0.140000</td>\n",
+       "      <td>1.216788</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.815000</td>\n",
+       "      <td>0.667750</td>\n",
+       "      <td>4</td>\n",
+       "      <td>2</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_24</th>\n",
+       "      <td>0.625000</td>\n",
+       "      <td>0.058750</td>\n",
+       "      <td>1.168317</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.730000</td>\n",
+       "      <td>0.674975</td>\n",
+       "      <td>8</td>\n",
+       "      <td>5</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_25</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.000000</td>\n",
+       "      <td>1.200240</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.800000</td>\n",
+       "      <td>0.666800</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">goals_under_2.5</th>\n",
+       "      <th>2021_19</th>\n",
+       "      <td>0.750000</td>\n",
+       "      <td>0.255000</td>\n",
+       "      <td>1.034973</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.630000</td>\n",
+       "      <td>0.634150</td>\n",
+       "      <td>4</td>\n",
+       "      <td>3</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_20</th>\n",
+       "      <td>0.800000</td>\n",
+       "      <td>0.372000</td>\n",
+       "      <td>1.142580</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.706000</td>\n",
+       "      <td>0.670480</td>\n",
+       "      <td>5</td>\n",
+       "      <td>4</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_21</th>\n",
+       "      <td>0.333333</td>\n",
+       "      <td>-0.363333</td>\n",
+       "      <td>1.200528</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.746667</td>\n",
+       "      <td>0.686033</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_22</th>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.670000</td>\n",
+       "      <td>1.025046</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.670000</td>\n",
+       "      <td>0.613800</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_23</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.000000</td>\n",
+       "      <td>1.011686</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.670000</td>\n",
+       "      <td>0.605800</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_24</th>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.755000</td>\n",
+       "      <td>1.155859</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.755000</td>\n",
+       "      <td>0.662375</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                 win    profit  exp_payoff  exp_payoff_prec  \\\n",
+       "bet_type        match_week                                                    \n",
+       "goals_over_2.5  2021_19     0.818182  0.434545    1.193888              NaN   \n",
+       "                2021_20     0.700000  0.144000    1.087417              NaN   \n",
+       "                2021_21     0.800000  0.359000    1.130386              NaN   \n",
+       "                2021_22     0.571429  0.040000    1.130597              NaN   \n",
+       "                2021_23     0.500000 -0.140000    1.216788              NaN   \n",
+       "                2021_24     0.625000  0.058750    1.168317              NaN   \n",
+       "                2021_25     0.000000 -1.000000    1.200240              NaN   \n",
+       "goals_under_2.5 2021_19     0.750000  0.255000    1.034973              NaN   \n",
+       "                2021_20     0.800000  0.372000    1.142580              NaN   \n",
+       "                2021_21     0.333333 -0.363333    1.200528              NaN   \n",
+       "                2021_22     1.000000  0.670000    1.025046              NaN   \n",
+       "                2021_23     0.000000 -1.000000    1.011686              NaN   \n",
+       "                2021_24     1.000000  0.755000    1.155859              NaN   \n",
+       "\n",
+       "                                odds  confidence  count  buenas  model  \n",
+       "bet_type        match_week                                              \n",
+       "goals_over_2.5  2021_19     1.743636    0.683591     11       9  model  \n",
+       "                2021_20     1.641000    0.664270     10       7  model  \n",
+       "                2021_21     1.685000    0.671600     10       8  model  \n",
+       "                2021_22     1.708571    0.662800      7       4  model  \n",
+       "                2021_23     1.815000    0.667750      4       2  model  \n",
+       "                2021_24     1.730000    0.674975      8       5  model  \n",
+       "                2021_25     1.800000    0.666800      1       0  model  \n",
+       "goals_under_2.5 2021_19     1.630000    0.634150      4       3  model  \n",
+       "                2021_20     1.706000    0.670480      5       4  model  \n",
+       "                2021_21     1.746667    0.686033      3       1  model  \n",
+       "                2021_22     1.670000    0.613800      1       1  model  \n",
+       "                2021_23     1.670000    0.605800      1       0  model  \n",
+       "                2021_24     1.755000    0.662375      4       4  model  "
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def show_results(df, groupby=[\"bet_type\", \"match_week\"]):\n",
+    "    ix = np.logical_and(df[\"win\"]>=0, True)# df[\"label\"]>=1)\n",
+    "    #ix = np.logical_and(df[\"consensus\"] >=0.1, ix)\n",
+    "    ix2 = np.logical_and(df[\"odds\"]>=1.5, df[\"odds\"]<2.)\n",
+    "    ix = np.logical_and(ix, ix2)\n",
+    "    #ix = np.logical_and(ix, ~df[\"validation\"])\n",
+    "    #ix = np.logical_and(ix, df[\"exp_payoff_prec\"] >0.88)\n",
+    "    #ix = np.logical_and(ix, df[\"exp_payoff\"] <1.2)\n",
+    "    #ix = np.logical_and(ix, df[\"exp_payoff\"] >0.8)\n",
+    "    ix = np.logical_and(ix, df[\"confidence\"] >=0.60)\n",
+    "    #ix = np.logical_and(ix, df[\"confidence\"] <=0.75)\n",
+    "    x = df[ix].groupby(groupby)[[\"win\", \"profit\", \"exp_payoff\", \"exp_payoff_prec\", \"odds\", \"confidence\"]].mean()\n",
+    "    x[\"count\"] = df[ix].groupby(groupby)[[\"win\"]].count()\n",
+    "    x[\"buenas\"] = df[ix].groupby(groupby)[[\"win\"]].sum().astype(int)\n",
+    "    x[\"model\"] = df[\"model\"].iloc[0]\n",
+    "    return x\n",
+    "show_results(tips)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/notebooks/feat_sel_test_j2.ipynb b/notebooks/feat_sel_test_j2.ipynb
new file mode 100644
index 0000000..e8808f1
--- /dev/null
+++ b/notebooks/feat_sel_test_j2.ipynb
@@ -0,0 +1,4977 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from itertools import product\n",
+    "from typing import Dict, List, Optional, Union\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import param\n",
+    "from pycaret.classification import (\n",
+    "    compare_models,\n",
+    "    create_model,\n",
+    "    get_config,\n",
+    "    predict_model,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    ")\n",
+    "from pycaret.utils import check_metric\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "class FeatureSelection(param.Parameterized):\n",
+    "\n",
+    "    # Class attributes\n",
+    "    model_class_to_name = {\n",
+    "        \"RidgeClassifier\": \"ridge\",\n",
+    "        \"LogisticRegression\": \"lr\",\n",
+    "        \"LinearDiscriminantAnalysis\": \"lda\",\n",
+    "        \"GradientBoostingClassifier\": \"gbc\",\n",
+    "        \"QuadraticDiscriminantAnalysis\": \"qda\",\n",
+    "        \"LGBMClassifier\": \"lightgbm\",\n",
+    "        \"AdaBoostClassifier\": \"ada\",\n",
+    "        \"RandomForestClassifier\": \"rf\",\n",
+    "        \"ExtraTreesClassifier\": \"et\",\n",
+    "        \"GaussianNB\": \"nb\",\n",
+    "        \"DecisionTreeClassifier\": \"dt\",\n",
+    "        \"KNeighborsClassifier\": \"knn\",\n",
+    "        \"SGDClassifier\": \"svm\",\n",
+    "        \"CatBoostClassifier\": \"catboost\",\n",
+    "        \"SVC\": \"rbfsvm\",\n",
+    "        \"GaussianProcessClassifier\": \"gpc\",\n",
+    "        \"MLPClassifier\": \"mlp\",\n",
+    "        \"XGBClassifier\": \"xgboost\",\n",
+    "    }\n",
+    "\n",
+    "    metrics_list = [\"Accuracy\", \"AUC\", \"Recall\", \"Precision\", \"F1\", \"Kappa\", \"MCC\"]\n",
+    "\n",
+    "    # Private class attributes\n",
+    "    _filter_metric = {\n",
+    "        \"Accuracy\": 0.5,\n",
+    "        \"AUC\": 0.5,\n",
+    "        \"Recall\": 0.6,\n",
+    "        \"Precision\": 0.6,\n",
+    "        \"F1\": 0.6,\n",
+    "        \"Kappa\": 0.1,\n",
+    "        \"MCC\": 0.1,\n",
+    "    }\n",
+    "\n",
+    "    _setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        train_size=0.75,\n",
+    "        # test_data=test_data,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.4,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        ignore_features=None,\n",
+    "        fold_strategy=\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=4,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.4,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=False,\n",
+    "        pca_method=\"kernel\",\n",
+    "        pca_components=30,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.01,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=False,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.01,\n",
+    "        fix_imbalance=False,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "        html=False,\n",
+    "    )\n",
+    "\n",
+    "    _numerics = [\"int16\", \"int32\", \"int64\", \"float16\", \"float32\", \"float64\", \"int\", \"float\"]\n",
+    "\n",
+    "    # Init values\n",
+    "    ## Feature selection parameters\n",
+    "    target = param.String(\"goal_2.5\")\n",
+    "    number_features = param.Number(\n",
+    "        0.5,\n",
+    "        bounds=(0, 1),\n",
+    "        inclusive_bounds=(False, False),\n",
+    "        doc=\"Number of features (percentage) selected each iteration. Only the first nth \"\n",
+    "        \"features will be kept for the next iteration.\",\n",
+    "    )\n",
+    "    target_features = param.Number(\n",
+    "        0.3,\n",
+    "        bounds=(0, None),\n",
+    "        inclusive_bounds=(False, True),\n",
+    "        doc=\"Final total number of features. The goal of the package is to reduce \"\n",
+    "        \"the incoming columns of the dataset to this 'target_features' number.\",\n",
+    "    )\n",
+    "    ## Metric parameters\n",
+    "    filter_metrics = param.Dict(_filter_metric)\n",
+    "    ## Model setup and model optimization parameters\n",
+    "    numerics = param.List(_numerics)\n",
+    "    ignore_features = param.List(default=[], allow_None=True)\n",
+    "    setup_kwargs = param.Dict(_setup_kwargs)\n",
+    "    include = param.List(default=None, item_type=str, allow_None=True)\n",
+    "    exclude = param.List([\"qda\", \"knn\", \"nb\"], item_type=str)\n",
+    "    sort = param.String(\"AUC\")\n",
+    "    number_models = param.Integer(10, bounds=(2, 13))\n",
+    "    top_models = param.List(default=None, allow_None=True)\n",
+    "    optimize = param.Boolean(False)\n",
+    "    opt_list = param.List([\"Accuracy\", \"Precision\", \"Recall\", \"F1\", \"AUC\"], item_type=str)\n",
+    "    ## Class selectors\n",
+    "    dataset = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    dict_models = param.ClassSelector(class_=dict)\n",
+    "    tune_dict_models = param.ClassSelector(class_=dict)\n",
+    "    x_train = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    x_df = param.DataFrame(pd.DataFrame())\n",
+    "    model_df = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    model_tuned_df = param.ClassSelector(class_=pd.DataFrame)\n",
+    "    features_df = param.ClassSelector(class_=pd.DataFrame)\n",
+    "\n",
+    "    def __init__(self, dataset: pd.DataFrame, **kwargs):\n",
+    "        # Copy of the incoming dataset\n",
+    "        dataset = dataset.copy()\n",
+    "        # Compute the upper bound of number_features, target_features, number_models\n",
+    "        total_features = dataset.shape[1]\n",
+    "        self.param.target_features.bounds = (0, total_features)\n",
+    "        if \"include\" in kwargs:\n",
+    "            self.param.number_models.default = len(kwargs[\"include\"])\n",
+    "            self.param.number_models.bounds = (0, len(kwargs[\"include\"]))\n",
+    "        # Call super\n",
+    "        super(FeatureSelection, self).__init__(dataset=dataset, **kwargs)\n",
+    "        # Get the features of the dataframe\n",
+    "        self.feature_list = self.dataset.columns.tolist()\n",
+    "        self.feature_list.remove(self.target)  # target column should not be counted\n",
+    "        # Compute target features\n",
+    "        self.target_features = self.calculate_number_features(\n",
+    "            number_features=self.target_features, features=self.feature_list\n",
+    "        )\n",
+    "        # Get the evaluator and the arguments. Depends on the \"include\" parameter\n",
+    "        self._training_function, self._args = self._decide_model_eval()\n",
+    "        # Get all the columns whose type is numeric\n",
+    "        self.numeric_features = self._compute_numeric_features(df=self.dataset[self.feature_list])\n",
+    "\n",
+    "    def _compute_numeric_features(self, df: pd.DataFrame):\n",
+    "        \"\"\"Return those columns from the given dataset whose data type is numeric.\"\"\"\n",
+    "        return df.select_dtypes(include=self.numerics).columns.tolist()\n",
+    "\n",
+    "    def _decide_model_eval(self):\n",
+    "        \"\"\"\n",
+    "        Define the pycaret model evaluator depending on the number of included models.\n",
+    "\n",
+    "        If the 'include' list parameter equals 1, the method will return\n",
+    "        the 'create_models' pycaret object.\n",
+    "        If 'include' parameter list is greatear than 1, the method will\n",
+    "        return the 'compare_model' pycaret object and its arguments.\n",
+    "        If 'include' parameter equals None, the method will return the\n",
+    "        'compare_models' pycaret object, where all possible models are\n",
+    "        considered for evaluation, except those included within the 'exclude'\n",
+    "        list.\n",
+    "        \"\"\"\n",
+    "        args = {\"n_select\": self.number_models, \"sort\": self.sort, \"verbose\": False}\n",
+    "        training_function = compare_models\n",
+    "        if not self.include:\n",
+    "            args[\"exclude\"] = self.exclude\n",
+    "        elif len(self.include) == 1:\n",
+    "            training_function = lambda *rgs, **kwargs: [create_model(*rgs, **kwargs)]\n",
+    "            args = {\"estimator\": self.include[0], \"verbose\": False}\n",
+    "        else:\n",
+    "            args[\"include\"] = self.include\n",
+    "        return training_function, args\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def calculate_number_features(\n",
+    "        number_features: Union[int, float], features: Union[pd.DataFrame, List]\n",
+    "    ) -> int:\n",
+    "        n_features = (\n",
+    "            int(number_features)\n",
+    "            if (number_features >= 1)\n",
+    "            else int(number_features * len(features))\n",
+    "        )\n",
+    "        return n_features\n",
+    "\n",
+    "    def train_model(self):\n",
+    "        \"\"\"Preprocess the data and select self.number_models top models.\"\"\"\n",
+    "        # Selected dataset\n",
+    "        selected_cols = self.feature_list + [self.target]\n",
+    "        train_data = self.dataset[selected_cols] if self.x_df.empty else self.x_df[selected_cols]\n",
+    "        # Numeric features\n",
+    "        self.setup_kwargs[\"numeric_features\"] = [\n",
+    "            c for c in self.numeric_features if c in self.feature_list\n",
+    "        ]\n",
+    "        # Ignore features\n",
+    "        self.setup_kwargs[\"ignore_features\"] = [\n",
+    "            c for c in self.ignore_features if c in self.feature_list\n",
+    "        ]\n",
+    "        # Initialize pycaret setup\n",
+    "        setup(data=train_data, target=self.target, **self.setup_kwargs)\n",
+    "        # Get train dataset and preprocessed dataframe\n",
+    "        self.x_train = get_config(\"X_train\")\n",
+    "        if self.x_df.empty:  # TODO change x_df by dataset and add flag?\n",
+    "            self.x_df = pd.concat([get_config(\"X\"), get_config(\"y\")], axis=1)\n",
+    "            self.setup_kwargs[\"preprocess\"] = False  # Turn off preprocessing\n",
+    "        # Compare models\n",
+    "        self.top_models = self._training_function(**self._args)\n",
+    "\n",
+    "    def create_dict_models(self):\n",
+    "        \"\"\"Create a dictionary whose values are pycaret standard models.\"\"\"\n",
+    "        self.dict_models = {\n",
+    "            str(top_model).split(\"(\")[0]: top_model for top_model in self.top_models\n",
+    "        }\n",
+    "        # Remove bad catboost key\n",
+    "        oldkey = [key for key in self.dict_models.keys() if key.startswith(\"<catboost\")]\n",
+    "        if oldkey:\n",
+    "            self.dict_models[\"CatBoostClassifier\"] = self.dict_models.pop(oldkey[0])\n",
+    "        # Remap\n",
+    "        self.dict_models = {\n",
+    "            self.model_class_to_name[key]: self.dict_models[key] for key in self.dict_models.keys()\n",
+    "        }\n",
+    "\n",
+    "    def create_dict_tuned_models(self):\n",
+    "        \"\"\"Create a dictionary whose keys and values are pycaret tuned models.\"\"\"\n",
+    "        self.tune_dict_models = {}\n",
+    "        for (model_str, py_model), optimize in product(self.dict_models.items(), self.opt_list):\n",
+    "            self.tune_dict_models[f\"{model_str}_tune_{optimize}\"] = tune_model(\n",
+    "                py_model,\n",
+    "                optimize=optimize,\n",
+    "                verbose=False,\n",
+    "                n_iter=30,\n",
+    "                choose_better=True,\n",
+    "            )\n",
+    "\n",
+    "    def get_metrics_df(self, test_predicted, model, dataframe):\n",
+    "        \"\"\"Compute different metric values for the given model.\"\"\"\n",
+    "        value_dct = dict()\n",
+    "        for metric in self.metrics_list:\n",
+    "            try:\n",
+    "                value_dct[metric] = check_metric(\n",
+    "                    actual=test_predicted[self.target],\n",
+    "                    prediction=test_predicted[\"Label\"],\n",
+    "                    metric=metric,\n",
+    "                )\n",
+    "            except AttributeError:\n",
+    "                value_dct[metric] = np.nan\n",
+    "        for key, val in value_dct.items():\n",
+    "            dataframe.loc[model, key] = val\n",
+    "        return dataframe\n",
+    "\n",
+    "    def remove_bad_models(self, dataframe: pd.DataFrame):\n",
+    "        \"\"\"Filter and remove the models whose metrics do not satisfy the given conditions.\"\"\"\n",
+    "        remove_dict = dict()\n",
+    "        models = dataframe.index.tolist()\n",
+    "        for model, (metric, cond) in product(models, self.filter_metrics.items()):\n",
+    "            if dataframe.loc[model, metric] < cond:\n",
+    "                remove_dict[model] = metric\n",
+    "        remove_models = list(set(remove_dict.keys()))\n",
+    "        dataframe.drop(labels=remove_models, axis=\"index\", inplace=True)\n",
+    "        return dataframe\n",
+    "\n",
+    "    def filter_best_features(self, key_model: str, models_dict: Dict):\n",
+    "        \"\"\"Compute the most relevant features used by the given model.\"\"\"\n",
+    "        py_model = models_dict[key_model]\n",
+    "        cond = any([key_model.startswith(name) for name in [\"lr\", \"lda\", \"ridge\", \"svm\"]])\n",
+    "        score_metric = abs(py_model.coef_[0]) if cond else py_model.feature_importances_\n",
+    "        metrics_dict = {\n",
+    "            \"model_id\": key_model,\n",
+    "            \"model\": key_model.split(\"_\")[0],\n",
+    "            \"feature\": self.x_train.columns,\n",
+    "            \"score\": score_metric,\n",
+    "        }\n",
+    "        df = pd.DataFrame(metrics_dict).sort_values(by=\"score\", ascending=False)\n",
+    "        top_n_features = self.calculate_number_features(\n",
+    "            number_features=self.number_features,\n",
+    "            features=df,\n",
+    "        )\n",
+    "        return df.iloc[:top_n_features]\n",
+    "\n",
+    "    def extract_features(self, dataframe: pd.DataFrame, dict_models: Dict):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by the given model.\"\"\"\n",
+    "        models = dataframe.index.tolist()\n",
+    "        for model in models:  # model extracted from dataframe\n",
+    "            # Check\n",
+    "            if (\n",
+    "                model not in dict_models.keys()\n",
+    "            ):  # check no errors have been produced during operations\n",
+    "                raise KeyError(f\"The selected model: {model} is not listed in dict_models.keys()\")\n",
+    "            df_conc = self.filter_best_features(key_model=model, models_dict=dict_models)\n",
+    "            self.features_df = pd.concat([self.features_df, df_conc])\n",
+    "\n",
+    "    def compute_metrics_df(self):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by the standard models.\"\"\"\n",
+    "        self.model_df = pd.DataFrame(\n",
+    "            data=[], index=self.dict_models.keys(), columns=self.metrics_list\n",
+    "        )\n",
+    "        for model, py_model in self.dict_models.items():\n",
+    "            predict = predict_model(py_model)\n",
+    "            self.model_df = self.get_metrics_df(\n",
+    "                test_predicted=predict,\n",
+    "                model=model,\n",
+    "                dataframe=self.model_df,\n",
+    "            )\n",
+    "        self.model_df = self.remove_bad_models(dataframe=self.model_df)\n",
+    "        self.extract_features(dataframe=self.model_df, dict_models=self.dict_models)\n",
+    "\n",
+    "    def filter_tuned_duplicate(self):\n",
+    "        \"\"\"Remove tuned models with identical metrics.\"\"\"\n",
+    "\n",
+    "        def drop(x):\n",
+    "            x.drop_duplicates(subset=self.metrics_list, keep=\"first\", inplace=True)\n",
+    "            return x\n",
+    "\n",
+    "        df = self.model_tuned_df.groupby(\"model\").apply(drop)\n",
+    "        df = (\n",
+    "            df.drop(columns=\"model\").reset_index(level=\"model\")\n",
+    "            if isinstance(df.index, pd.MultiIndex)\n",
+    "            else df\n",
+    "        )\n",
+    "        return df\n",
+    "\n",
+    "    def tune_df(self):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by tuned models.\"\"\"\n",
+    "        # Tune dataframe\n",
+    "        self.model_tuned_df = pd.DataFrame(\n",
+    "            data=[],\n",
+    "            index=self.tune_dict_models.keys(),\n",
+    "            columns=[\"model\"] + self.metrics_list,\n",
+    "        )\n",
+    "        # Model entry\n",
+    "        for prim_model in self.dict_models.keys():\n",
+    "            ix = [ind.startswith(prim_model) for ind in self.model_tuned_df.index]\n",
+    "            self.model_tuned_df.loc[ix, \"model\"] = prim_model\n",
+    "        # Fill dataframe\n",
+    "        for model, py_model in self.tune_dict_models.items():\n",
+    "            predict = predict_model(py_model)\n",
+    "            self.model_tuned_df = self.get_metrics_df(\n",
+    "                test_predicted=predict, model=model, dataframe=self.model_tuned_df\n",
+    "            )\n",
+    "        # Remove duplicate and filter\n",
+    "        self.model_tuned_df = self.filter_tuned_duplicate()\n",
+    "        self.model_tuned_df = self.remove_bad_models(dataframe=self.model_tuned_df)\n",
+    "        # Get features\n",
+    "        self.extract_features(dataframe=self.model_tuned_df, dict_models=self.tune_dict_models)\n",
+    "\n",
+    "    def run_feature_extraction(self):\n",
+    "        \"\"\"Update self.features_df with the most relevant features used by each model.\"\"\"\n",
+    "        # Initialize feature dataframe and train model\n",
+    "        self.features_df = pd.DataFrame(data=[], columns=[\"model_id\", \"model\", \"feature\", \"score\"])\n",
+    "        self.train_model()\n",
+    "        # Run standard models\n",
+    "        self.create_dict_models()\n",
+    "        self.compute_metrics_df()\n",
+    "        # Run tuned models\n",
+    "        if self.optimize:\n",
+    "            self.create_dict_tuned_models()\n",
+    "            if not bool(self.tune_dict_models):\n",
+    "                raise ValueError(\"The tune dictionary is empty!\")\n",
+    "            self.tune_df()\n",
+    "        # Return the list containing the features and their score\n",
+    "        self.features_df.index.name = \"index_rem\"\n",
+    "        return self.features_df.reset_index().drop(columns=\"index_rem\")\n",
+    "\n",
+    "    def remove_zeros(self):\n",
+    "        \"\"\"Remove non-relevant features (those with a zero score).\"\"\"\n",
+    "        ix = self.features_df[\"score\"] <= 0\n",
+    "        self.features_df.drop(index=self.features_df.loc[ix].index, inplace=True)\n",
+    "        self.features_df.reset_index(drop=True, inplace=True)\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def normalize(dataframe):\n",
+    "        \"\"\"Normalize the pycaret score of each feature.\"\"\"\n",
+    "\n",
+    "        def norm(x):\n",
+    "            x[\"normal\"] = x[\"score\"] / x[\"score\"].max()\n",
+    "            return x\n",
+    "\n",
+    "        return (\n",
+    "            dataframe.groupby(\"model_id\")\n",
+    "            .apply(norm)\n",
+    "            .sort_values([\"model_id\", \"normal\"], ascending=[True, False])\n",
+    "        )\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def feature_score(dataframe):\n",
+    "        \"\"\"Assign the score to the selected features.\"\"\"\n",
+    "        group = dataframe.groupby(\"feature\")\n",
+    "        sorted_data = group.agg(\n",
+    "            counts=pd.NamedAgg(column=\"normal\", aggfunc=\"count\"),\n",
+    "            normal_sum=pd.NamedAgg(column=\"normal\", aggfunc=\"sum\"),\n",
+    "        ).sort_values(by=[\"counts\", \"normal_sum\"], ascending=False)\n",
+    "        sorted_data[\"final_score\"] = sorted_data[\"normal_sum\"] / sorted_data[\"counts\"]\n",
+    "        return sorted_data.sort_values(\"final_score\", ascending=False)\n",
+    "\n",
+    "    def create_feature_list(self):\n",
+    "        \"\"\"Run all necessary methods to extract the list of relevant features.\"\"\"\n",
+    "        # Call creation features dataframe\n",
+    "        self.features_df = self.run_feature_extraction()\n",
+    "        # Remove zeros and normalize\n",
+    "        self.remove_zeros()\n",
+    "        self.features_df = self.normalize(dataframe=self.features_df)\n",
+    "        # Get score\n",
+    "        scoreboard = self.feature_score(dataframe=self.features_df)\n",
+    "        top_n_features = self.calculate_number_features(\n",
+    "            number_features=self.number_features,\n",
+    "            features=scoreboard,\n",
+    "        )\n",
+    "        filtered = scoreboard.iloc[:top_n_features]\n",
+    "        self.feature_list = filtered.index.tolist()\n",
+    "\n",
+    "    def repeat_pipeline(self):\n",
+    "        \"\"\"Iterate over the process to create the feature list and repeat it self.repeat times.\"\"\"\n",
+    "        while len(self.feature_list) > self.target_features:\n",
+    "            # Call iteration\n",
+    "            self.create_feature_list()\n",
+    "            if len(self.feature_list) <= 1:\n",
+    "                break\n",
+    "        return self.feature_list\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
+    "from ml_bets.constants import FEATURES_PATH\n",
+    "from ml_bets.features.features import Features\n",
+    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
+    "\n",
+    "from pycaret.utils import check_metric\n",
+    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
+    "    create_model,\n",
+    "    finalize_model,\n",
+    "    optimize_threshold,              \n",
+    "    save_model,\n",
+    "    compare_models, \n",
+    "    evaluate_model,\n",
+    "    get_config,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    "    predict_model,\n",
+    ")\n",
+    "\n",
+    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
+    "from ml_bets.research.datasets import create_dataset\n",
+    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target = \"goals_2.5\"\n",
+    "test_date = \"1-Dec-2021\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ESP1C.xls\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ING1C.xls\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ITA1C.xls\n",
+      "Excel file c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\MEX1C.xls is empty. Skipping\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\FRA1C.xls\n",
+      "Excel file c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\MLS1C.xls is empty. Skipping\n",
+      "c:\\users\\usuario\\desktop\\data science\\ml_bets-master\\data\\future_matches\\ALE1C.xls\n"
+     ]
+    }
+   ],
+   "source": [
+    "feats = Features()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def patched_create_dataset(\n",
+    "    test_date: str,\n",
+    "    target: str,\n",
+    "    columns=None,\n",
+    "    odds_features: bool = True,\n",
+    "    summary: bool = True,\n",
+    "    odds_rankings: bool = True,\n",
+    "    include_std: bool = True,\n",
+    "    features: Features = None,\n",
+    "    ignore_features=None,\n",
+    "    drop_future_matches: bool = True,\n",
+    "    test_weeks: int = 4,\n",
+    "):\n",
+    "    features = features or Features()\n",
+    "    examples = features.create(\n",
+    "        columns=columns,\n",
+    "        odds_features=odds_features,\n",
+    "        odds_rankings=odds_rankings,\n",
+    "        referee_features=True,\n",
+    "        include_std=include_std,\n",
+    "        summary=summary,\n",
+    "    )\n",
+    "    examples = examples[[x for x in examples.columns if \"possession\" not in x]]\n",
+    "    if ignore_features is not None:\n",
+    "        examples.drop(columns=ignore_features, inplace=True)\n",
+    "    pds = PipelineDatasets(\n",
+    "        examples=examples,\n",
+    "        features=features,\n",
+    "        target=target,\n",
+    "        drop_future_matches=drop_future_matches,\n",
+    "        test_size=test_date,\n",
+    "        test_weeks=test_weeks,\n",
+    "    )\n",
+    "    return pds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = patched_create_dataset(target=target,\n",
+    "                    test_date=test_date,\n",
+    "                    features=feats,\n",
+    "                    odds_features=True,\n",
+    "                    include_std=True,\n",
+    "                    ignore_features=IGNORE_FEATURES+[\"referee\", \"hour_rank\", \"hour_before_16\", \"is_weekend\"],\n",
+    "                    drop_future_matches=False,\n",
+    "                    \n",
+    "                   )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data = ds.train_data.copy()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        #custom_pipeline=loaded,\n",
+    "        train_size=0.75,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "       # normalize_method=\"robust\",\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.8,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        #ignore_features=ignore_features,\n",
+    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=3,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.5,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=False,\n",
+    "        pca_method=\"kernel\",\n",
+    "        pca_components=50,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.05,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=True,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.05,\n",
+    "        fix_imbalance=True,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Feature selection (without triplet)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5186</td>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5299</td>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5000</td>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5286</td>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5139</td>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0378</td>\n",
+       "                        <td id=\"T_1dc9dd5b_836d_11ec_9865_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0378</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d124f4f0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_53e72cb2_8370_11ec_8906_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d126d460>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_54171313_8370_11ec_97ee_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d1281dc0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_54461320_8370_11ec_9528_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_54461320_8370_11ec_9528_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_54461320_8370_11ec_9528_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d12812b0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_5471b7d0_8370_11ec_a011_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184b77a1850>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5131</td>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5400</td>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5027</td>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5225</td>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5124</td>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0265</td>\n",
+       "                        <td id=\"T_58300db8_8370_11ec_bad1_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0265</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d0718a60>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5296</td>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5489</td>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5297</td>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5385</td>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5341</td>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0591</td>\n",
+       "                        <td id=\"T_2959aa4f_8371_11ec_9dbc_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0591</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d0718a60>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5034</td>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5386</td>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4811</td>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5130</td>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.4965</td>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0077</td>\n",
+       "                        <td id=\"T_29734d51_8371_11ec_9230_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0077</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d079e550>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5213</td>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5423</td>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5081</td>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5311</td>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5193</td>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0431</td>\n",
+       "                        <td id=\"T_298d176d_8371_11ec_952a_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0431</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d079e7c0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5296</td>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5489</td>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5297</td>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5385</td>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5341</td>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0591</td>\n",
+       "                        <td id=\"T_29a6934d_8371_11ec_bc1a_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0591</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x184d07b12e0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "metric_param = {\n",
+    "        \"Accuracy\": 0.1,\n",
+    "        \"AUC\": 0.1,\n",
+    "        \"Recall\": 0.1,\n",
+    "        \"Precision\": 0.1,\n",
+    "        \"F1\": 0.1,\n",
+    "        \"Kappa\": -1.0,\n",
+    "        \"MCC\": -1.0,\n",
+    "    }\n",
+    "feat_sel = FeatureSelection(target=target,\n",
+    "                            dataset=train_data.dropna(),#[list(set(new_subset+new_feat+[target]))],\n",
+    "                            target_features=500,\n",
+    "                            filter_metrics=metric_param,\n",
+    "                            include=[\"lr\"],\n",
+    "                            setup_kwargs=setup_kwargs,\n",
+    "                            optimize=True,\n",
+    "                            opt_list=[\"AUC\" , \"Accuracy\" , \"Precision\" , \"Recall\"],\n",
+    "                            number_features = 0.56\n",
+    "                           )\n",
+    "selected_features = feat_sel.repeat_pipeline()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "355"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(selected_features)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['DEF_mean_a',\n",
+       " 'MED_mean_diff',\n",
+       " 'acc_corners_intobox_pct_std_a',\n",
+       " 'accurate_back_zone_pass_std_a',\n",
+       " 'accurate_chipped_pass_std_diff',\n",
+       " 'accurate_cross_nocorner_std_diff',\n",
+       " 'accurate_cross_std_a',\n",
+       " 'accurate_cross_std_h',\n",
+       " 'accurate_fwd_zone_pass_mean_diff',\n",
+       " 'accurate_fwd_zone_pass_std_a',\n",
+       " 'accurate_fwd_zone_pass_std_h',\n",
+       " 'accurate_goal_kicks_std_h',\n",
+       " 'accurate_keeper_sweeper_mean_a',\n",
+       " 'accurate_keeper_sweeper_mean_diff',\n",
+       " 'accurate_keeper_sweeper_mean_h',\n",
+       " 'accurate_keeper_sweeper_std_a',\n",
+       " 'accurate_keeper_sweeper_std_diff',\n",
+       " 'accurate_keeper_sweeper_std_h',\n",
+       " 'accurate_keeper_throws_std_diff',\n",
+       " 'accurate_launches_mean_a',\n",
+       " 'accurate_launches_std_a',\n",
+       " 'accurate_layoffs_mean_a',\n",
+       " 'accurate_layoffs_std_a',\n",
+       " 'accurate_layoffs_std_diff',\n",
+       " 'accurate_pass_std_a',\n",
+       " 'accurate_pass_std_diff',\n",
+       " 'accurate_pull_back_mean_diff',\n",
+       " 'accurate_through_ball_mean_diff',\n",
+       " 'accurate_throws_std_a',\n",
+       " 'accurate_throws_std_diff',\n",
+       " 'accurate_throws_std_h',\n",
+       " 'att_assist_openplay_mean_a',\n",
+       " 'att_bx_right_mean_h',\n",
+       " 'att_bx_right_std_diff',\n",
+       " 'att_cmiss_high_mean_h',\n",
+       " 'att_cmiss_left_std_a',\n",
+       " 'att_cmiss_left_std_h',\n",
+       " 'att_corner_mean_a',\n",
+       " 'att_corner_ratio_mean_a',\n",
+       " 'att_fastbreak_std_h',\n",
+       " 'att_freekick_goal_std_diff',\n",
+       " 'att_freekick_miss_mean_h',\n",
+       " 'att_freekick_miss_std_a',\n",
+       " 'att_freekick_miss_std_diff',\n",
+       " 'att_freekick_miss_std_h',\n",
+       " 'att_freekick_target_std_diff',\n",
+       " 'att_freekick_total_mean_a',\n",
+       " 'att_freekick_total_mean_diff',\n",
+       " 'att_freekick_total_mean_h',\n",
+       " 'att_freekick_total_std_diff',\n",
+       " 'att_goal_high_centre_mean_diff',\n",
+       " 'att_goal_high_centre_std_diff',\n",
+       " 'att_goal_high_left_mean_a',\n",
+       " 'att_goal_high_left_mean_diff',\n",
+       " 'att_goal_high_left_std_h',\n",
+       " 'att_goal_high_right_std_a',\n",
+       " 'att_goal_high_right_std_diff',\n",
+       " 'att_goal_low_centre_mean_a',\n",
+       " 'att_goal_low_centre_mean_h',\n",
+       " 'att_goal_low_centre_std_a',\n",
+       " 'att_goal_low_centre_std_h',\n",
+       " 'att_goal_low_right_mean_h',\n",
+       " 'att_hd_goal_mean_a',\n",
+       " 'att_hd_goal_mean_h',\n",
+       " 'att_hd_post_std_h',\n",
+       " 'att_hd_target_mean_a',\n",
+       " 'att_hd_total_mean_diff',\n",
+       " 'att_hd_total_std_h',\n",
+       " 'att_ibox_blocked_mean_a',\n",
+       " 'att_ibox_blocked_std_diff',\n",
+       " 'att_ibox_blocked_std_h',\n",
+       " 'att_ibox_own_goal_mean_h',\n",
+       " 'att_ibox_own_goal_std_diff',\n",
+       " 'att_lf_goal_mean_diff',\n",
+       " 'att_lf_goal_mean_h',\n",
+       " 'att_lf_goal_std_h',\n",
+       " 'att_lg_centre_mean_diff',\n",
+       " 'att_lg_centre_mean_h',\n",
+       " 'att_lg_centre_std_a',\n",
+       " 'att_miss_high_left_std_a',\n",
+       " 'att_miss_high_left_std_h',\n",
+       " 'att_miss_right_std_diff',\n",
+       " 'att_miss_right_std_h',\n",
+       " 'att_obox_blocked_mean_h',\n",
+       " 'att_obox_blocked_std_a',\n",
+       " 'att_obox_goal_mean_h',\n",
+       " 'att_obox_goal_std_a',\n",
+       " 'att_obox_goal_std_diff',\n",
+       " 'att_obox_miss_mean_a',\n",
+       " 'att_obox_miss_mean_diff',\n",
+       " 'att_obox_miss_std_a',\n",
+       " 'att_obox_post_mean_diff',\n",
+       " 'att_obx_centre_std_a',\n",
+       " 'att_obx_right_mean_diff',\n",
+       " 'att_obxd_right_mean_diff',\n",
+       " 'att_obxd_right_std_diff',\n",
+       " 'att_one_on_one_std_diff',\n",
+       " 'att_openplay_mean_h',\n",
+       " 'att_openplay_std_h',\n",
+       " 'att_pen_goal_mean_a',\n",
+       " 'att_pen_goal_mean_h',\n",
+       " 'att_pen_goal_std_a',\n",
+       " 'att_post_left_std_a',\n",
+       " 'att_post_left_std_diff',\n",
+       " 'att_post_left_std_h',\n",
+       " 'att_rf_goal_mean_h',\n",
+       " 'att_rf_target_std_a',\n",
+       " 'att_rf_total_mean_a',\n",
+       " 'att_rf_total_std_diff',\n",
+       " 'att_setpiece_std_a',\n",
+       " 'att_sv_low_centre_mean_h',\n",
+       " 'att_sv_low_centre_std_diff',\n",
+       " 'att_sv_low_right_std_h',\n",
+       " 'attempts_conceded_ibox_mean_diff',\n",
+       " 'attempts_conceded_ibox_std_diff',\n",
+       " 'attempts_ibox_std_a',\n",
+       " 'attempts_ibox_std_h',\n",
+       " 'attempts_obox_mean_h',\n",
+       " 'backward_pass_mean_a',\n",
+       " 'backward_pass_std_h',\n",
+       " 'ball_recovery_mean_diff',\n",
+       " 'bc_miss_div_created_std_h',\n",
+       " 'bc_miss_div_scored_std_h',\n",
+       " 'bc_scored_div_created_mean_a',\n",
+       " 'bc_scored_div_created_mean_diff',\n",
+       " 'bc_scored_div_created_std_h',\n",
+       " 'big_chance_created_mean_h',\n",
+       " 'big_chance_created_std_diff',\n",
+       " 'big_chance_missed_std_diff',\n",
+       " 'big_chance_scored_mean_diff',\n",
+       " 'blocked_cross_mean_a',\n",
+       " 'blocked_cross_std_a',\n",
+       " 'challenge_lost_std_a',\n",
+       " 'challenge_lost_std_diff',\n",
+       " 'clean_sheet_std_diff',\n",
+       " 'contentious_decision_mean_diff',\n",
+       " 'contentious_decision_std_a',\n",
+       " 'corner_taken_std_diff',\n",
+       " 'corners_ratio_mean_h',\n",
+       " 'crosses_18yard_mean_a',\n",
+       " 'dispossessed_std_h',\n",
+       " 'diving_save_mean_h',\n",
+       " 'diving_save_std_diff',\n",
+       " 'draw_streak_h',\n",
+       " 'duel_won_std_a',\n",
+       " 'effective_clearance_mean_a',\n",
+       " 'effective_clearance_mean_diff',\n",
+       " 'effective_head_clearance_mean_h',\n",
+       " 'error_lead_to_goal_mean_diff',\n",
+       " 'first_yellow_card_1t_std_diff',\n",
+       " 'first_yellow_card_std_a',\n",
+       " 'freekick_cross_std_a',\n",
+       " 'fwd_pass_std_a',\n",
+       " 'goal_assist_deadball_mean_diff',\n",
+       " 'goal_assist_deadball_std_h',\n",
+       " 'goal_assist_intent_norm_mean_h',\n",
+       " 'goal_assist_intent_norm_std_diff',\n",
+       " 'goal_assist_intentional_mean_diff',\n",
+       " 'goal_assist_intentional_mean_h',\n",
+       " 'goal_assist_mean_a',\n",
+       " 'goal_assist_mean_h',\n",
+       " 'goal_assist_openplay_mean_h',\n",
+       " 'goal_assist_openplay_std_a',\n",
+       " 'goal_assist_openplay_std_diff',\n",
+       " 'goal_assist_openplay_std_h',\n",
+       " 'goal_assist_setplay_mean_a',\n",
+       " 'goal_assist_std_diff',\n",
+       " 'goal_assist_std_h',\n",
+       " 'goal_fastbreak_mean_diff',\n",
+       " 'goal_fastbreak_mean_h',\n",
+       " 'goal_kicks_div_long_passes_std_diff',\n",
+       " 'goal_kicks_per_shot_mean_a',\n",
+       " 'goals_2t_mean_diff',\n",
+       " 'goals_2t_mean_h',\n",
+       " 'goals_2t_pct_std_h',\n",
+       " 'goals_conceded_ibox_mean_h',\n",
+       " 'goals_conceded_mean_a',\n",
+       " 'goals_mean_a',\n",
+       " 'goals_mean_diff',\n",
+       " 'goals_openplay_mean_a',\n",
+       " 'goals_openplay_std_h',\n",
+       " 'goals_std_h',\n",
+       " 'good_high_claim_mean_a',\n",
+       " 'good_high_claim_mean_diff',\n",
+       " 'good_high_claim_mean_h',\n",
+       " 'hit_woodwork_mean_a',\n",
+       " 'interceptions_in_box_mean_diff',\n",
+       " 'interceptions_in_box_std_a',\n",
+       " 'interceptions_in_box_std_diff',\n",
+       " 'interceptions_in_box_std_h',\n",
+       " 'last_man_tackle_mean_diff',\n",
+       " 'last_man_tackle_std_a',\n",
+       " 'left_div_right_foot_goals_mean_a',\n",
+       " 'leftside_pass_std_diff',\n",
+       " 'lineup_mean_h',\n",
+       " 'long_pass_own_to_opp_success_std_diff',\n",
+       " 'lost_corners_mean_a',\n",
+       " 'lost_corners_std_diff',\n",
+       " 'no_foot_goals_ratio_mean_h',\n",
+       " 'no_foot_goals_ratio_std_a',\n",
+       " 'no_foot_goals_ratio_std_h',\n",
+       " 'odd_ratio_over_cards_5.5',\n",
+       " 'odds_away_over_cards_5.5_a',\n",
+       " 'odds_away_over_corners_9.5_diff',\n",
+       " 'odds_away_under_cards_3.5_h',\n",
+       " 'odds_away_under_corners_6.5_a',\n",
+       " 'odds_away_under_corners_8.5_a',\n",
+       " 'odds_home_over_goals_1.5_a',\n",
+       " 'odds_home_under_cards_4.5_h',\n",
+       " 'odds_home_under_goals_0.5_diff',\n",
+       " 'ontarget_att_assist_std_a',\n",
+       " 'ontarget_att_assist_std_diff',\n",
+       " 'open_play_pass_std_a',\n",
+       " 'outfielder_block_mean_h',\n",
+       " 'own_goals_std_diff',\n",
+       " 'own_goals_std_h',\n",
+       " 'passes_left_div_blocked_pass_std_h',\n",
+       " 'passes_right_mean_diff',\n",
+       " 'passes_right_std_a',\n",
+       " 'passes_right_std_diff',\n",
+       " 'passes_right_std_h',\n",
+       " 'pen_area_entries_mean_a',\n",
+       " 'pen_goals_conceded_std_diff',\n",
+       " 'penalty_conceded_mean_a',\n",
+       " 'penalty_faced_mean_diff',\n",
+       " 'penalty_faced_std_diff',\n",
+       " 'penalty_save_mean_diff',\n",
+       " 'penalty_save_mean_h',\n",
+       " 'penalty_won_mean_a',\n",
+       " 'penalty_won_std_a',\n",
+       " 'points_diff',\n",
+       " 'points_std_a',\n",
+       " 'points_std_h',\n",
+       " 'poss_won_att_3rd_std_a',\n",
+       " 'poss_won_att_3rd_std_diff',\n",
+       " 'poss_won_att_3rd_std_h',\n",
+       " 'poss_won_mid_3rd_std_a',\n",
+       " 'post_scoring_att_std_diff',\n",
+       " 'post_scoring_att_std_h',\n",
+       " 'prob_squared_over_both_score',\n",
+       " 'prob_squared_over_goals_2.5',\n",
+       " 'prob_squared_under_goals_2.5',\n",
+       " 'prob_squared_under_goals_4.5',\n",
+       " 'pts_dropped_winning_pos_mean_a',\n",
+       " 'punches_mean_a',\n",
+       " 'ranking_h',\n",
+       " 'ranking_mean_diff',\n",
+       " 'ratio_over_cards_4.5_a',\n",
+       " 'ratio_over_corners_8.5_diff',\n",
+       " 'ratio_over_corners_9.5_a',\n",
+       " 'ratio_under_corners_10.5_diff',\n",
+       " 'ratio_under_corners_10.5_h',\n",
+       " 'ratio_under_corners_11.5_h',\n",
+       " 'ratio_under_corners_8.5_h',\n",
+       " 'ratio_under_corners_9.5_diff',\n",
+       " 'ratio_under_goals_4.5_h',\n",
+       " 'raw_prob_over_goals_1.5',\n",
+       " 'raw_prob_over_goals_2.5',\n",
+       " 'raw_prob_under_corners_8.5',\n",
+       " 'raw_prob_under_goals_4.5',\n",
+       " 'red_card_1t_mean_a',\n",
+       " 'red_card_2t_mean_a',\n",
+       " 'red_card_away_ref',\n",
+       " 'red_card_home_ref',\n",
+       " 'red_card_std_a',\n",
+       " 'right_to_left_goals_mean_diff',\n",
+       " 'right_to_left_goals_std_diff',\n",
+       " 'rightside_pass_mean_a',\n",
+       " 'rightside_pass_std_diff',\n",
+       " 'saves_std_h',\n",
+       " 'second_yellow_mean_diff',\n",
+       " 'shots_div_passes_right_std_h',\n",
+       " 'shots_mul_goals_mean_a',\n",
+       " 'shots_mul_goals_std_h',\n",
+       " 'shots_std_diff',\n",
+       " 'subs_made_mean_h',\n",
+       " 'successful_fifty_fifty_std_a',\n",
+       " 'successful_final_third_passes_std_h',\n",
+       " 'successful_put_through_std_a',\n",
+       " 'successful_put_through_std_diff',\n",
+       " 'successful_put_through_std_h',\n",
+       " 'total_back_zone_pass_mean_a',\n",
+       " 'total_bets_over_cards_4.5_a',\n",
+       " 'total_bets_over_cards_4.5_diff',\n",
+       " 'total_bets_over_cards_5.5_diff',\n",
+       " 'total_bets_under_cards_4.5_h',\n",
+       " 'total_bets_under_corners_6.5_diff',\n",
+       " 'total_bets_under_corners_9.5_h',\n",
+       " 'total_bets_under_goals_2.5_a',\n",
+       " 'total_chipped_pass_std_a',\n",
+       " 'total_clearance_mean_h',\n",
+       " 'total_cross_nocorner_std_h',\n",
+       " 'total_expulsions_ref',\n",
+       " 'total_final_third_passes_std_diff',\n",
+       " 'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_diff',\n",
+       " 'total_fwd_zone_pass_std_a',\n",
+       " 'total_fwd_zone_pass_std_diff',\n",
+       " 'total_high_claim_mean_diff',\n",
+       " 'total_high_claim_mean_h',\n",
+       " 'total_high_claim_std_a',\n",
+       " 'total_high_claim_std_diff',\n",
+       " 'total_high_claim_std_h',\n",
+       " 'total_keeper_sweeper_mean_a',\n",
+       " 'total_keeper_sweeper_mean_diff',\n",
+       " 'total_keeper_sweeper_mean_h',\n",
+       " 'total_keeper_sweeper_std_diff',\n",
+       " 'total_launches_mean_a',\n",
+       " 'total_launches_std_h',\n",
+       " 'total_layoffs_mean_diff',\n",
+       " 'total_layoffs_std_a',\n",
+       " 'total_layoffs_std_diff',\n",
+       " 'total_layoffs_std_h',\n",
+       " 'total_penalty_match_ref',\n",
+       " 'total_red_card_1t_ref',\n",
+       " 'total_red_card_2t_ref',\n",
+       " 'total_red_card_match_ref',\n",
+       " 'total_second_yel_card_home_ref',\n",
+       " 'total_through_ball_mean_a',\n",
+       " 'total_through_ball_std_a',\n",
+       " 'total_throws_mean_h',\n",
+       " 'total_throws_std_a',\n",
+       " 'total_throws_std_diff',\n",
+       " 'total_throws_std_h',\n",
+       " 'total_win_pct_over_corners_7.5_diff',\n",
+       " 'total_win_pct_over_corners_9.5_a',\n",
+       " 'total_win_pct_over_goals_2.5_h',\n",
+       " 'total_win_pct_under_cards_3.5_diff',\n",
+       " 'total_win_pct_under_goals_1.5_a',\n",
+       " 'total_win_pct_under_goals_3.5_h',\n",
+       " 'total_yel_card_std_a',\n",
+       " 'win_pct_away_over_cards_6.5_a',\n",
+       " 'win_pct_away_over_corners_10.5_diff',\n",
+       " 'win_pct_away_over_corners_8.5_a',\n",
+       " 'win_pct_away_over_goals_0.5_diff',\n",
+       " 'win_pct_away_over_goals_2.5_a',\n",
+       " 'win_pct_away_over_goals_2.5_diff',\n",
+       " 'win_pct_away_under_both_score_h',\n",
+       " 'win_pct_away_under_goals_2.5_a',\n",
+       " 'win_pct_away_under_goals_2.5_diff',\n",
+       " 'win_pct_home_over_cards_6.5_a',\n",
+       " 'win_pct_home_over_goals_2.5_h',\n",
+       " 'win_pct_home_under_cards_4.5_diff',\n",
+       " 'win_pct_home_under_corners_11.5_h',\n",
+       " 'win_pct_home_under_corners_8.5_a',\n",
+       " 'win_pct_home_under_corners_9.5_h',\n",
+       " 'win_pct_home_under_goals_2.5_diff',\n",
+       " 'win_pct_home_under_goals_2.5_h',\n",
+       " 'winner_mean_diff',\n",
+       " 'winner_std_diff',\n",
+       " 'winner_std_h',\n",
+       " 'won_contest_mean_a',\n",
+       " 'won_corners_ratio_mean_a',\n",
+       " 'won_corners_ratio_mean_h',\n",
+       " 'won_corners_ratio_std_h',\n",
+       " 'won_tackle_std_a']"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sorted(selected_features)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "selected_features_save = ['DEF_mean_a',\n",
+    " 'MED_mean_diff',\n",
+    " 'acc_corners_intobox_pct_std_a',\n",
+    " 'accurate_back_zone_pass_std_a',\n",
+    " 'accurate_chipped_pass_std_diff',\n",
+    " 'accurate_cross_nocorner_std_diff',\n",
+    " 'accurate_cross_std_a',\n",
+    " 'accurate_cross_std_h',\n",
+    " 'accurate_fwd_zone_pass_mean_diff',\n",
+    " 'accurate_fwd_zone_pass_std_a',\n",
+    " 'accurate_fwd_zone_pass_std_h',\n",
+    " 'accurate_goal_kicks_std_h',\n",
+    " 'accurate_keeper_sweeper_mean_a',\n",
+    " 'accurate_keeper_sweeper_mean_diff',\n",
+    " 'accurate_keeper_sweeper_mean_h',\n",
+    " 'accurate_keeper_sweeper_std_a',\n",
+    " 'accurate_keeper_sweeper_std_diff',\n",
+    " 'accurate_keeper_sweeper_std_h',\n",
+    " 'accurate_keeper_throws_std_diff',\n",
+    " 'accurate_launches_mean_a',\n",
+    " 'accurate_launches_std_a',\n",
+    " 'accurate_layoffs_mean_a',\n",
+    " 'accurate_layoffs_std_a',\n",
+    " 'accurate_layoffs_std_diff',\n",
+    " 'accurate_pass_std_a',\n",
+    " 'accurate_pass_std_diff',\n",
+    " 'accurate_pull_back_mean_diff',\n",
+    " 'accurate_through_ball_mean_diff',\n",
+    " 'accurate_throws_std_a',\n",
+    " 'accurate_throws_std_diff',\n",
+    " 'accurate_throws_std_h',\n",
+    " 'att_assist_openplay_mean_a',\n",
+    " 'att_bx_right_mean_h',\n",
+    " 'att_bx_right_std_diff',\n",
+    " 'att_cmiss_high_mean_h',\n",
+    " 'att_cmiss_left_std_a',\n",
+    " 'att_cmiss_left_std_h',\n",
+    " 'att_corner_mean_a',\n",
+    " 'att_corner_ratio_mean_a',\n",
+    " 'att_fastbreak_std_h',\n",
+    " 'att_freekick_goal_std_diff',\n",
+    " 'att_freekick_miss_mean_h',\n",
+    " 'att_freekick_miss_std_a',\n",
+    " 'att_freekick_miss_std_diff',\n",
+    " 'att_freekick_miss_std_h',\n",
+    " 'att_freekick_target_std_diff',\n",
+    " 'att_freekick_total_mean_a',\n",
+    " 'att_freekick_total_mean_diff',\n",
+    " 'att_freekick_total_mean_h',\n",
+    " 'att_freekick_total_std_diff',\n",
+    " 'att_goal_high_centre_mean_diff',\n",
+    " 'att_goal_high_centre_std_diff',\n",
+    " 'att_goal_high_left_mean_a',\n",
+    " 'att_goal_high_left_mean_diff',\n",
+    " 'att_goal_high_left_std_h',\n",
+    " 'att_goal_high_right_std_a',\n",
+    " 'att_goal_high_right_std_diff',\n",
+    " 'att_goal_low_centre_mean_a',\n",
+    " 'att_goal_low_centre_mean_h',\n",
+    " 'att_goal_low_centre_std_a',\n",
+    " 'att_goal_low_centre_std_h',\n",
+    " 'att_goal_low_right_mean_h',\n",
+    " 'att_hd_goal_mean_a',\n",
+    " 'att_hd_goal_mean_h',\n",
+    " 'att_hd_post_std_h',\n",
+    " 'att_hd_target_mean_a',\n",
+    " 'att_hd_total_mean_diff',\n",
+    " 'att_hd_total_std_h',\n",
+    " 'att_ibox_blocked_mean_a',\n",
+    " 'att_ibox_blocked_std_diff',\n",
+    " 'att_ibox_blocked_std_h',\n",
+    " 'att_ibox_own_goal_mean_h',\n",
+    " 'att_ibox_own_goal_std_diff',\n",
+    " 'att_lf_goal_mean_diff',\n",
+    " 'att_lf_goal_mean_h',\n",
+    " 'att_lf_goal_std_h',\n",
+    " 'att_lg_centre_mean_diff',\n",
+    " 'att_lg_centre_mean_h',\n",
+    " 'att_lg_centre_std_a',\n",
+    " 'att_miss_high_left_std_a',\n",
+    " 'att_miss_high_left_std_h',\n",
+    " 'att_miss_right_std_diff',\n",
+    " 'att_miss_right_std_h',\n",
+    " 'att_obox_blocked_mean_h',\n",
+    " 'att_obox_blocked_std_a',\n",
+    " 'att_obox_goal_mean_h',\n",
+    " 'att_obox_goal_std_a',\n",
+    " 'att_obox_goal_std_diff',\n",
+    " 'att_obox_miss_mean_a',\n",
+    " 'att_obox_miss_mean_diff',\n",
+    " 'att_obox_miss_std_a',\n",
+    " 'att_obox_post_mean_diff',\n",
+    " 'att_obx_centre_std_a',\n",
+    " 'att_obx_right_mean_diff',\n",
+    " 'att_obxd_right_mean_diff',\n",
+    " 'att_obxd_right_std_diff',\n",
+    " 'att_one_on_one_std_diff',\n",
+    " 'att_openplay_mean_h',\n",
+    " 'att_openplay_std_h',\n",
+    " 'att_pen_goal_mean_a',\n",
+    " 'att_pen_goal_mean_h',\n",
+    " 'att_pen_goal_std_a',\n",
+    " 'att_post_left_std_a',\n",
+    " 'att_post_left_std_diff',\n",
+    " 'att_post_left_std_h',\n",
+    " 'att_rf_goal_mean_h',\n",
+    " 'att_rf_target_std_a',\n",
+    " 'att_rf_total_mean_a',\n",
+    " 'att_rf_total_std_diff',\n",
+    " 'att_setpiece_std_a',\n",
+    " 'att_sv_low_centre_mean_h',\n",
+    " 'att_sv_low_centre_std_diff',\n",
+    " 'att_sv_low_right_std_h',\n",
+    " 'attempts_conceded_ibox_mean_diff',\n",
+    " 'attempts_conceded_ibox_std_diff',\n",
+    " 'attempts_ibox_std_a',\n",
+    " 'attempts_ibox_std_h',\n",
+    " 'attempts_obox_mean_h',\n",
+    " 'backward_pass_mean_a',\n",
+    " 'backward_pass_std_h',\n",
+    " 'ball_recovery_mean_diff',\n",
+    " 'bc_miss_div_created_std_h',\n",
+    " 'bc_miss_div_scored_std_h',\n",
+    " 'bc_scored_div_created_mean_a',\n",
+    " 'bc_scored_div_created_mean_diff',\n",
+    " 'bc_scored_div_created_std_h',\n",
+    " 'big_chance_created_mean_h',\n",
+    " 'big_chance_created_std_diff',\n",
+    " 'big_chance_missed_std_diff',\n",
+    " 'big_chance_scored_mean_diff',\n",
+    " 'blocked_cross_mean_a',\n",
+    " 'blocked_cross_std_a',\n",
+    " 'challenge_lost_std_a',\n",
+    " 'challenge_lost_std_diff',\n",
+    " 'clean_sheet_std_diff',\n",
+    " 'contentious_decision_mean_diff',\n",
+    " 'contentious_decision_std_a',\n",
+    " 'corner_taken_std_diff',\n",
+    " 'corners_ratio_mean_h',\n",
+    " 'crosses_18yard_mean_a',\n",
+    " 'dispossessed_std_h',\n",
+    " 'diving_save_mean_h',\n",
+    " 'diving_save_std_diff',\n",
+    " 'draw_streak_h',\n",
+    " 'duel_won_std_a',\n",
+    " 'effective_clearance_mean_a',\n",
+    " 'effective_clearance_mean_diff',\n",
+    " 'effective_head_clearance_mean_h',\n",
+    " 'error_lead_to_goal_mean_diff',\n",
+    " 'first_yellow_card_1t_std_diff',\n",
+    " 'first_yellow_card_std_a',\n",
+    " 'freekick_cross_std_a',\n",
+    " 'fwd_pass_std_a',\n",
+    " 'goal_assist_deadball_mean_diff',\n",
+    " 'goal_assist_deadball_std_h',\n",
+    " 'goal_assist_intent_norm_mean_h',\n",
+    " 'goal_assist_intent_norm_std_diff',\n",
+    " 'goal_assist_intentional_mean_diff',\n",
+    " 'goal_assist_intentional_mean_h',\n",
+    " 'goal_assist_mean_a',\n",
+    " 'goal_assist_mean_h',\n",
+    " 'goal_assist_openplay_mean_h',\n",
+    " 'goal_assist_openplay_std_a',\n",
+    " 'goal_assist_openplay_std_diff',\n",
+    " 'goal_assist_openplay_std_h',\n",
+    " 'goal_assist_setplay_mean_a',\n",
+    " 'goal_assist_std_diff',\n",
+    " 'goal_assist_std_h',\n",
+    " 'goal_fastbreak_mean_diff',\n",
+    " 'goal_fastbreak_mean_h',\n",
+    " 'goal_kicks_div_long_passes_std_diff',\n",
+    " 'goal_kicks_per_shot_mean_a',\n",
+    " 'goals_2t_mean_diff',\n",
+    " 'goals_2t_mean_h',\n",
+    " 'goals_2t_pct_std_h',\n",
+    " 'goals_conceded_ibox_mean_h',\n",
+    " 'goals_conceded_mean_a',\n",
+    " 'goals_mean_a',\n",
+    " 'goals_mean_diff',\n",
+    " 'goals_openplay_mean_a',\n",
+    " 'goals_openplay_std_h',\n",
+    " 'goals_std_h',\n",
+    " 'good_high_claim_mean_a',\n",
+    " 'good_high_claim_mean_diff',\n",
+    " 'good_high_claim_mean_h',\n",
+    " 'hit_woodwork_mean_a',\n",
+    " 'interceptions_in_box_mean_diff',\n",
+    " 'interceptions_in_box_std_a',\n",
+    " 'interceptions_in_box_std_diff',\n",
+    " 'interceptions_in_box_std_h',\n",
+    " 'last_man_tackle_mean_diff',\n",
+    " 'last_man_tackle_std_a',\n",
+    " 'left_div_right_foot_goals_mean_a',\n",
+    " 'leftside_pass_std_diff',\n",
+    " 'lineup_mean_h',\n",
+    " 'long_pass_own_to_opp_success_std_diff',\n",
+    " 'lost_corners_mean_a',\n",
+    " 'lost_corners_std_diff',\n",
+    " 'no_foot_goals_ratio_mean_h',\n",
+    " 'no_foot_goals_ratio_std_a',\n",
+    " 'no_foot_goals_ratio_std_h',\n",
+    " 'odd_ratio_over_cards_5.5',\n",
+    " 'odds_away_over_cards_5.5_a',\n",
+    " 'odds_away_over_corners_9.5_diff',\n",
+    " 'odds_away_under_cards_3.5_h',\n",
+    " 'odds_away_under_corners_6.5_a',\n",
+    " 'odds_away_under_corners_8.5_a',\n",
+    " 'odds_home_over_goals_1.5_a',\n",
+    " 'odds_home_under_cards_4.5_h',\n",
+    " 'odds_home_under_goals_0.5_diff',\n",
+    " 'ontarget_att_assist_std_a',\n",
+    " 'ontarget_att_assist_std_diff',\n",
+    " 'open_play_pass_std_a',\n",
+    " 'outfielder_block_mean_h',\n",
+    " 'own_goals_std_diff',\n",
+    " 'own_goals_std_h',\n",
+    " 'passes_left_div_blocked_pass_std_h',\n",
+    " 'passes_right_mean_diff',\n",
+    " 'passes_right_std_a',\n",
+    " 'passes_right_std_diff',\n",
+    " 'passes_right_std_h',\n",
+    " 'pen_area_entries_mean_a',\n",
+    " 'pen_goals_conceded_std_diff',\n",
+    " 'penalty_conceded_mean_a',\n",
+    " 'penalty_faced_mean_diff',\n",
+    " 'penalty_faced_std_diff',\n",
+    " 'penalty_save_mean_diff',\n",
+    " 'penalty_save_mean_h',\n",
+    " 'penalty_won_mean_a',\n",
+    " 'penalty_won_std_a',\n",
+    " 'points_diff',\n",
+    " 'points_std_a',\n",
+    " 'points_std_h',\n",
+    " 'poss_won_att_3rd_std_a',\n",
+    " 'poss_won_att_3rd_std_diff',\n",
+    " 'poss_won_att_3rd_std_h',\n",
+    " 'poss_won_mid_3rd_std_a',\n",
+    " 'post_scoring_att_std_diff',\n",
+    " 'post_scoring_att_std_h',\n",
+    " 'prob_squared_over_both_score',\n",
+    " 'prob_squared_over_goals_2.5',\n",
+    " 'prob_squared_under_goals_2.5',\n",
+    " 'prob_squared_under_goals_4.5',\n",
+    " 'pts_dropped_winning_pos_mean_a',\n",
+    " 'punches_mean_a',\n",
+    " 'ranking_h',\n",
+    " 'ranking_mean_diff',\n",
+    " 'ratio_over_cards_4.5_a',\n",
+    " 'ratio_over_corners_8.5_diff',\n",
+    " 'ratio_over_corners_9.5_a',\n",
+    " 'ratio_under_corners_10.5_diff',\n",
+    " 'ratio_under_corners_10.5_h',\n",
+    " 'ratio_under_corners_11.5_h',\n",
+    " 'ratio_under_corners_8.5_h',\n",
+    " 'ratio_under_corners_9.5_diff',\n",
+    " 'ratio_under_goals_4.5_h',\n",
+    " 'raw_prob_over_goals_1.5',\n",
+    " 'raw_prob_over_goals_2.5',\n",
+    " 'raw_prob_under_corners_8.5',\n",
+    " 'raw_prob_under_goals_4.5',\n",
+    " 'red_card_1t_mean_a',\n",
+    " 'red_card_2t_mean_a',\n",
+    " 'red_card_away_ref',\n",
+    " 'red_card_home_ref',\n",
+    " 'red_card_std_a',\n",
+    " 'right_to_left_goals_mean_diff',\n",
+    " 'right_to_left_goals_std_diff',\n",
+    " 'rightside_pass_mean_a',\n",
+    " 'rightside_pass_std_diff',\n",
+    " 'saves_std_h',\n",
+    " 'second_yellow_mean_diff',\n",
+    " 'shots_div_passes_right_std_h',\n",
+    " 'shots_mul_goals_mean_a',\n",
+    " 'shots_mul_goals_std_h',\n",
+    " 'shots_std_diff',\n",
+    " 'subs_made_mean_h',\n",
+    " 'successful_fifty_fifty_std_a',\n",
+    " 'successful_final_third_passes_std_h',\n",
+    " 'successful_put_through_std_a',\n",
+    " 'successful_put_through_std_diff',\n",
+    " 'successful_put_through_std_h',\n",
+    " 'total_back_zone_pass_mean_a',\n",
+    " 'total_bets_over_cards_4.5_a',\n",
+    " 'total_bets_over_cards_4.5_diff',\n",
+    " 'total_bets_over_cards_5.5_diff',\n",
+    " 'total_bets_under_cards_4.5_h',\n",
+    " 'total_bets_under_corners_6.5_diff',\n",
+    " 'total_bets_under_corners_9.5_h',\n",
+    " 'total_bets_under_goals_2.5_a',\n",
+    " 'total_chipped_pass_std_a',\n",
+    " 'total_clearance_mean_h',\n",
+    " 'total_cross_nocorner_std_h',\n",
+    " 'total_expulsions_ref',\n",
+    " 'total_final_third_passes_std_diff',\n",
+    " 'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_diff',\n",
+    " 'total_fwd_zone_pass_std_a',\n",
+    " 'total_fwd_zone_pass_std_diff',\n",
+    " 'total_high_claim_mean_diff',\n",
+    " 'total_high_claim_mean_h',\n",
+    " 'total_high_claim_std_a',\n",
+    " 'total_high_claim_std_diff',\n",
+    " 'total_high_claim_std_h',\n",
+    " 'total_keeper_sweeper_mean_a',\n",
+    " 'total_keeper_sweeper_mean_diff',\n",
+    " 'total_keeper_sweeper_mean_h',\n",
+    " 'total_keeper_sweeper_std_diff',\n",
+    " 'total_launches_mean_a',\n",
+    " 'total_launches_std_h',\n",
+    " 'total_layoffs_mean_diff',\n",
+    " 'total_layoffs_std_a',\n",
+    " 'total_layoffs_std_diff',\n",
+    " 'total_layoffs_std_h',\n",
+    " 'total_penalty_match_ref',\n",
+    " 'total_red_card_1t_ref',\n",
+    " 'total_red_card_2t_ref',\n",
+    " 'total_red_card_match_ref',\n",
+    " 'total_second_yel_card_home_ref',\n",
+    " 'total_through_ball_mean_a',\n",
+    " 'total_through_ball_std_a',\n",
+    " 'total_throws_mean_h',\n",
+    " 'total_throws_std_a',\n",
+    " 'total_throws_std_diff',\n",
+    " 'total_throws_std_h',\n",
+    " 'total_win_pct_over_corners_7.5_diff',\n",
+    " 'total_win_pct_over_corners_9.5_a',\n",
+    " 'total_win_pct_over_goals_2.5_h',\n",
+    " 'total_win_pct_under_cards_3.5_diff',\n",
+    " 'total_win_pct_under_goals_1.5_a',\n",
+    " 'total_win_pct_under_goals_3.5_h',\n",
+    " 'total_yel_card_std_a',\n",
+    " 'win_pct_away_over_cards_6.5_a',\n",
+    " 'win_pct_away_over_corners_10.5_diff',\n",
+    " 'win_pct_away_over_corners_8.5_a',\n",
+    " 'win_pct_away_over_goals_0.5_diff',\n",
+    " 'win_pct_away_over_goals_2.5_a',\n",
+    " 'win_pct_away_over_goals_2.5_diff',\n",
+    " 'win_pct_away_under_both_score_h',\n",
+    " 'win_pct_away_under_goals_2.5_a',\n",
+    " 'win_pct_away_under_goals_2.5_diff',\n",
+    " 'win_pct_home_over_cards_6.5_a',\n",
+    " 'win_pct_home_over_goals_2.5_h',\n",
+    " 'win_pct_home_under_cards_4.5_diff',\n",
+    " 'win_pct_home_under_corners_11.5_h',\n",
+    " 'win_pct_home_under_corners_8.5_a',\n",
+    " 'win_pct_home_under_corners_9.5_h',\n",
+    " 'win_pct_home_under_goals_2.5_diff',\n",
+    " 'win_pct_home_under_goals_2.5_h',\n",
+    " 'winner_mean_diff',\n",
+    " 'winner_std_diff',\n",
+    " 'winner_std_h',\n",
+    " 'won_contest_mean_a',\n",
+    " 'won_corners_ratio_mean_a',\n",
+    " 'won_corners_ratio_mean_h',\n",
+    " 'won_corners_ratio_std_h',\n",
+    " 'won_tackle_std_a']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### new imports "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
+    "from ml_bets.constants import FEATURES_PATH\n",
+    "from ml_bets.features.features import Features\n",
+    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
+    "#from featsel.feature_selection import FeatureSelection\n",
+    "\n",
+    "from pycaret.utils import check_metric\n",
+    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
+    "    create_model,\n",
+    "    finalize_model,\n",
+    "    optimize_threshold,              \n",
+    "    save_model,\n",
+    "    compare_models, \n",
+    "    evaluate_model,\n",
+    "    get_config,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    "    predict_model,\n",
+    ")\n",
+    "\n",
+    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
+    "from ml_bets.research.datasets import create_dataset\n",
+    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ml_bets.research.tips import calibrate_tips, tips_from_model, combine_tips, predict_dataset, compose_tips, get_tip_probs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ix = np.logical_and(feats.matches[\"date\"].dt.month > 10,\n",
+    "                    feats.matches[\"competition\"].isin({\"mexican_primera\", 'us_major_league_soccer'}))\n",
+    "index = feats.matches[~ix].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def setup_dataset(test_date):\n",
+    "    ds = create_dataset(target=target,\n",
+    "                        test_date=test_date,\n",
+    "                        features=feats,\n",
+    "                        odds_features=True,\n",
+    "                        include_std=True,\n",
+    "                        test_weeks=4,\n",
+    "                        ignore_features=IGNORE_FEATURES,\n",
+    "                        drop_future_matches=False,\n",
+    "                       )\n",
+    "    train_data = ds.train_data.copy()#[ds.train_data.index.map(lambda x: \"us_major_league_soccer\" not in x and \"mexican\" not in x)]\n",
+    "    train_data.drop(columns=[\"hour_rank\", \"hour_before_16\", \"is_weekend\"], inplace=True)\n",
+    "    train_data = train_data[train_data.index.isin(index)][list(set(selected_features)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    test_data = ds.test_set[ds.test_set.index.isin(index)][list(set(selected_features)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    val_data = ds.val_set[ds.val_set.index.isin(index)].copy()\n",
+    "    setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        test_data=test_data[train_data.columns.tolist()],#.dropna(),\n",
+    "        #numeric_features=[x for x in train_data.columns.tolist() if x != target],\n",
+    "        #custom_pipeline=loaded,\n",
+    "        #train_size=0.75,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "        normalize_method=\"robust\",\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.8,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        #ignore_features=ignore_features,\n",
+    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=4,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.5,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=False,\n",
+    "        pca_method=\"kernel\",\n",
+    "        pca_components=50,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.05,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=True,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.05,\n",
+    "        fix_imbalance=True,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "    )\n",
+    "    _ = setup(data=train_data, target=target, **setup_kwargs)\n",
+    "    return train_data, test_data, val_data, ds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data, test_data, val_data, ds = setup_dataset(\"4-Dec-2021\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
+    "def train_ensemble():\n",
+    "    top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"et\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )\n",
+    "    tuned_models = [tune_model(model, optimize=\"AUC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
+    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
+    "    blend = blend_models(cali)\n",
+    "    opti = tune_model(blend, optimize=\"Precision\", choose_better=True, n_iter=50, search_library=\"optuna\")\n",
+    "    return opti, cali, tuned_models, blend"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "#T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col0,#T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col1,#T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col2,#T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col3,#T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col4,#T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col5,#T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col6{\n",
+       "            background:  yellow;\n",
+       "        }</style><table id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Accuracy</th>        <th class=\"col_heading level0 col1\" >AUC</th>        <th class=\"col_heading level0 col2\" >Recall</th>        <th class=\"col_heading level0 col3\" >Prec.</th>        <th class=\"col_heading level0 col4\" >F1</th>        <th class=\"col_heading level0 col5\" >Kappa</th>        <th class=\"col_heading level0 col6\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >0.6208</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.0000</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.6404</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.6357</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.6380</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.2399</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.2399</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row1_col0\" class=\"data row1 col0\" >0.6049</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row1_col1\" class=\"data row1 col1\" >0.0000</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row1_col2\" class=\"data row1 col2\" >0.6232</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row1_col3\" class=\"data row1 col3\" >0.6216</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row1_col4\" class=\"data row1 col4\" >0.6224</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row1_col5\" class=\"data row1 col5\" >0.2081</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row1_col6\" class=\"data row1 col6\" >0.2081</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row2_col0\" class=\"data row2 col0\" >0.6422</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row2_col1\" class=\"data row2 col1\" >0.0000</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row2_col2\" class=\"data row2 col2\" >0.6404</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row2_col3\" class=\"data row2 col3\" >0.6633</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row2_col4\" class=\"data row2 col4\" >0.6516</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row2_col5\" class=\"data row2 col5\" >0.2841</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row2_col6\" class=\"data row2 col6\" >0.2843</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row3_col0\" class=\"data row3 col0\" >0.5997</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row3_col1\" class=\"data row3 col1\" >0.0000</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row3_col2\" class=\"data row3 col2\" >0.5246</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row3_col3\" class=\"data row3 col3\" >0.6435</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row3_col4\" class=\"data row3 col4\" >0.5780</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row3_col5\" class=\"data row3 col5\" >0.2048</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row3_col6\" class=\"data row3 col6\" >0.2087</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2level0_row4\" class=\"row_heading level0 row4\" >Mean</th>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col0\" class=\"data row4 col0\" >0.6169</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col1\" class=\"data row4 col1\" >0.0000</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col2\" class=\"data row4 col2\" >0.6071</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col3\" class=\"data row4 col3\" >0.6410</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col4\" class=\"data row4 col4\" >0.6225</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col5\" class=\"data row4 col5\" >0.2342</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row4_col6\" class=\"data row4 col6\" >0.2353</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2level0_row5\" class=\"row_heading level0 row5\" >SD</th>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row5_col0\" class=\"data row5 col0\" >0.0165</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row5_col1\" class=\"data row5 col1\" >0.0000</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row5_col2\" class=\"data row5 col2\" >0.0482</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row5_col3\" class=\"data row5 col3\" >0.0150</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row5_col4\" class=\"data row5 col4\" >0.0277</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row5_col5\" class=\"data row5 col5\" >0.0319</td>\n",
+       "                        <td id=\"T_00dd4764_837e_11ec_a443_d8c0a63024d2row5_col6\" class=\"data row5 col6\" >0.0311</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1851208cac0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "ename": "TypeError",
+     "evalue": "calibrate_model() got an unexpected keyword argument 'calibrate_fold'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[1;32m<ipython-input-23-3e0a882fe730>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mopti_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcali_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtune_model\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mblend_model\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_ensemble\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[1;32m<ipython-input-22-be2a267acf2e>\u001b[0m in \u001b[0;36mtrain_ensemble\u001b[1;34m()\u001b[0m\n\u001b[0;32m      8\u001b[0m         )\n\u001b[0;32m      9\u001b[0m     \u001b[0mtuned_models\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtune_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"AUC\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchoose_better\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_iter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msearch_library\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"optuna\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtop_models\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m     \u001b[0mcali\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mcalibrate_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtuned\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"sigmoid\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcalibrate_fold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtuned\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtuned_models\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m     \u001b[0mblend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mblend_models\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcali\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mopti\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtune_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mblend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"Precision\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchoose_better\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_iter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msearch_library\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"optuna\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32m<ipython-input-22-be2a267acf2e>\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      8\u001b[0m         )\n\u001b[0;32m      9\u001b[0m     \u001b[0mtuned_models\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtune_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"AUC\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchoose_better\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_iter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msearch_library\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"optuna\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtop_models\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m     \u001b[0mcali\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mcalibrate_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtuned\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"sigmoid\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcalibrate_fold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtuned\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtuned_models\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m     \u001b[0mblend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mblend_models\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcali\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mopti\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtune_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mblend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"Precision\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchoose_better\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_iter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msearch_library\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"optuna\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;31mTypeError\u001b[0m: calibrate_model() got an unexpected keyword argument 'calibrate_fold'"
+     ]
+    }
+   ],
+   "source": [
+    "opti_model, cali_model, tune_model, blend_model = train_ensemble()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "    #T_21ce2d37_829b_11ec_86da_31d766f3ddbd th {\n",
+       "          text-align: left;\n",
+       "    }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col5,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col0,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col7{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col1,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col2,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col6,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col7,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col3,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col4,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col5{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col8,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col8,#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  lightgrey;\n",
+       "        }#T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "            background-color:  lightgrey;\n",
+       "        }</style><table id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>        <th class=\"col_heading level0 col8\" >TT (Sec)</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >lr</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.5831</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6161</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5937</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6030</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.5971</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.1652</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.1658</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow0_col8\" class=\"data row0 col8\" >0.6300</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >svm</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >SVM - Linear Kernel</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.5780</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.5975</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6037</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.5845</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.1544</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col7\" class=\"data row1 col7\" >0.1656</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow1_col8\" class=\"data row1 col8\" >0.2050</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >ridge</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >Ridge Classifier</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.5825</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.5937</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6018</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.5966</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.1639</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col7\" class=\"data row2 col7\" >0.1645</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow2_col8\" class=\"data row2 col8\" >0.2000</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >lda</th>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >Linear Discriminant Analysis</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.5819</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.6107</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.5962</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6003</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.5972</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.1624</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col7\" class=\"data row3 col7\" >0.1629</td>\n",
+       "                        <td id=\"T_21ce2d37_829b_11ec_86da_31d766f3ddbdrow3_col8\" class=\"data row3 col8\" >0.2100</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7ff4940a2880>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluate_model(top_models[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Feature selection (adding triplet)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(181,\n",
+       " 583,\n",
+       " ['ATT_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_mean_a',\n",
+       "  'DEF_mean_diff',\n",
+       "  'DEF_mean_h',\n",
+       "  'MED_a',\n",
+       "  'MED_diff',\n",
+       "  'MED_h',\n",
+       "  'MED_mean_a',\n",
+       "  'MED_mean_diff',\n",
+       "  'MED_mean_h',\n",
+       "  'MED_std_a',\n",
+       "  'MED_std_diff',\n",
+       "  'MED_std_h',\n",
+       "  'accurate_cross_nocorner_std_a',\n",
+       "  'accurate_cross_nocorner_std_diff',\n",
+       "  'accurate_cross_nocorner_std_h',\n",
+       "  'accurate_cross_std_a',\n",
+       "  'accurate_cross_std_diff',\n",
+       "  'accurate_cross_std_h',\n",
+       "  'accurate_fwd_zone_pass_mean_a',\n",
+       "  'accurate_fwd_zone_pass_mean_diff',\n",
+       "  'accurate_fwd_zone_pass_mean_h',\n",
+       "  'accurate_fwd_zone_pass_std_a',\n",
+       "  'accurate_fwd_zone_pass_std_diff',\n",
+       "  'accurate_fwd_zone_pass_std_h',\n",
+       "  'accurate_goal_kicks_std_a',\n",
+       "  'accurate_goal_kicks_std_diff',\n",
+       "  'accurate_goal_kicks_std_h',\n",
+       "  'accurate_keeper_sweeper_mean_a',\n",
+       "  'accurate_keeper_sweeper_mean_diff',\n",
+       "  'accurate_keeper_sweeper_mean_h',\n",
+       "  'accurate_keeper_sweeper_std_a',\n",
+       "  'accurate_keeper_sweeper_std_diff',\n",
+       "  'accurate_keeper_sweeper_std_h',\n",
+       "  'accurate_launches_mean_a',\n",
+       "  'accurate_launches_mean_diff',\n",
+       "  'accurate_launches_mean_h',\n",
+       "  'accurate_layoffs_mean_a',\n",
+       "  'accurate_layoffs_mean_diff',\n",
+       "  'accurate_layoffs_mean_h',\n",
+       "  'accurate_layoffs_std_a',\n",
+       "  'accurate_layoffs_std_diff',\n",
+       "  'accurate_layoffs_std_h',\n",
+       "  'accurate_pass_std_a',\n",
+       "  'accurate_pass_std_diff',\n",
+       "  'accurate_pass_std_h',\n",
+       "  'accurate_through_ball_mean_a',\n",
+       "  'accurate_through_ball_mean_diff',\n",
+       "  'accurate_through_ball_mean_h',\n",
+       "  'accurate_throws_std_a',\n",
+       "  'accurate_throws_std_diff',\n",
+       "  'accurate_throws_std_h',\n",
+       "  'att_assist_openplay_mean_a',\n",
+       "  'att_assist_openplay_mean_diff',\n",
+       "  'att_assist_openplay_mean_h',\n",
+       "  'att_corner_ratio_mean_a',\n",
+       "  'att_corner_ratio_mean_diff',\n",
+       "  'att_corner_ratio_mean_h',\n",
+       "  'att_freekick_goal_std_a',\n",
+       "  'att_freekick_goal_std_diff',\n",
+       "  'att_freekick_goal_std_h',\n",
+       "  'att_freekick_miss_mean_a',\n",
+       "  'att_freekick_miss_mean_diff',\n",
+       "  'att_freekick_miss_mean_h',\n",
+       "  'att_freekick_miss_std_a',\n",
+       "  'att_freekick_miss_std_diff',\n",
+       "  'att_freekick_miss_std_h',\n",
+       "  'att_freekick_total_mean_a',\n",
+       "  'att_freekick_total_mean_diff',\n",
+       "  'att_freekick_total_mean_h',\n",
+       "  'att_goal_high_centre_mean_a',\n",
+       "  'att_goal_high_centre_mean_diff',\n",
+       "  'att_goal_high_centre_mean_h',\n",
+       "  'att_goal_high_centre_std_a',\n",
+       "  'att_goal_high_centre_std_diff',\n",
+       "  'att_goal_high_centre_std_h',\n",
+       "  'att_goal_high_left_mean_a',\n",
+       "  'att_goal_high_left_mean_diff',\n",
+       "  'att_goal_high_left_mean_h',\n",
+       "  'att_goal_high_left_std_a',\n",
+       "  'att_goal_high_left_std_diff',\n",
+       "  'att_goal_high_left_std_h',\n",
+       "  'att_goal_high_right_std_a',\n",
+       "  'att_goal_high_right_std_diff',\n",
+       "  'att_goal_high_right_std_h',\n",
+       "  'att_goal_low_centre_mean_a',\n",
+       "  'att_goal_low_centre_mean_diff',\n",
+       "  'att_goal_low_centre_mean_h',\n",
+       "  'att_goal_low_centre_std_a',\n",
+       "  'att_goal_low_centre_std_diff',\n",
+       "  'att_goal_low_centre_std_h',\n",
+       "  'att_goal_low_right_mean_a',\n",
+       "  'att_goal_low_right_mean_diff',\n",
+       "  'att_goal_low_right_mean_h',\n",
+       "  'att_hd_goal_mean_a',\n",
+       "  'att_hd_goal_mean_diff',\n",
+       "  'att_hd_goal_mean_h',\n",
+       "  'att_hd_target_mean_a',\n",
+       "  'att_hd_target_mean_diff',\n",
+       "  'att_hd_target_mean_h',\n",
+       "  'att_ibox_own_goal_std_a',\n",
+       "  'att_ibox_own_goal_std_diff',\n",
+       "  'att_ibox_own_goal_std_h',\n",
+       "  'att_lf_goal_mean_a',\n",
+       "  'att_lf_goal_mean_diff',\n",
+       "  'att_lf_goal_mean_h',\n",
+       "  'att_lf_goal_std_a',\n",
+       "  'att_lf_goal_std_diff',\n",
+       "  'att_lf_goal_std_h',\n",
+       "  'att_lg_centre_mean_a',\n",
+       "  'att_lg_centre_mean_diff',\n",
+       "  'att_lg_centre_mean_h',\n",
+       "  'att_miss_high_left_std_a',\n",
+       "  'att_miss_high_left_std_diff',\n",
+       "  'att_miss_high_left_std_h',\n",
+       "  'att_miss_right_std_a',\n",
+       "  'att_miss_right_std_diff',\n",
+       "  'att_miss_right_std_h',\n",
+       "  'att_obox_blocked_mean_a',\n",
+       "  'att_obox_blocked_mean_diff',\n",
+       "  'att_obox_blocked_mean_h',\n",
+       "  'att_obox_goal_mean_a',\n",
+       "  'att_obox_goal_mean_diff',\n",
+       "  'att_obox_goal_mean_h',\n",
+       "  'att_obox_goal_std_a',\n",
+       "  'att_obox_goal_std_diff',\n",
+       "  'att_obox_goal_std_h',\n",
+       "  'att_obox_miss_mean_a',\n",
+       "  'att_obox_miss_mean_diff',\n",
+       "  'att_obox_miss_mean_h',\n",
+       "  'att_obox_post_mean_a',\n",
+       "  'att_obox_post_mean_diff',\n",
+       "  'att_obox_post_mean_h',\n",
+       "  'att_obx_right_mean_a',\n",
+       "  'att_obx_right_mean_diff',\n",
+       "  'att_obx_right_mean_h',\n",
+       "  'att_obxd_right_mean_a',\n",
+       "  'att_obxd_right_mean_diff',\n",
+       "  'att_obxd_right_mean_h',\n",
+       "  'att_obxd_right_std_a',\n",
+       "  'att_obxd_right_std_diff',\n",
+       "  'att_obxd_right_std_h',\n",
+       "  'att_openplay_mean_a',\n",
+       "  'att_openplay_mean_diff',\n",
+       "  'att_openplay_mean_h',\n",
+       "  'att_pen_goal_mean_a',\n",
+       "  'att_pen_goal_mean_diff',\n",
+       "  'att_pen_goal_mean_h',\n",
+       "  'att_post_left_mean_a',\n",
+       "  'att_post_left_mean_diff',\n",
+       "  'att_post_left_mean_h',\n",
+       "  'att_post_left_std_a',\n",
+       "  'att_post_left_std_diff',\n",
+       "  'att_post_left_std_h',\n",
+       "  'attempts_conceded_ibox_std_a',\n",
+       "  'attempts_conceded_ibox_std_diff',\n",
+       "  'attempts_conceded_ibox_std_h',\n",
+       "  'attempts_conceded_obox_std_a',\n",
+       "  'attempts_conceded_obox_std_diff',\n",
+       "  'attempts_conceded_obox_std_h',\n",
+       "  'attempts_obox_mean_a',\n",
+       "  'attempts_obox_mean_diff',\n",
+       "  'attempts_obox_mean_h',\n",
+       "  'bc_miss_div_scored_std_a',\n",
+       "  'bc_miss_div_scored_std_diff',\n",
+       "  'bc_miss_div_scored_std_h',\n",
+       "  'big_chance_created_mean_a',\n",
+       "  'big_chance_created_mean_diff',\n",
+       "  'big_chance_created_mean_h',\n",
+       "  'big_chance_created_std_a',\n",
+       "  'big_chance_created_std_diff',\n",
+       "  'big_chance_created_std_h',\n",
+       "  'big_chance_missed_std_a',\n",
+       "  'big_chance_missed_std_diff',\n",
+       "  'big_chance_missed_std_h',\n",
+       "  'big_chance_scored_mean_a',\n",
+       "  'big_chance_scored_mean_diff',\n",
+       "  'big_chance_scored_mean_diff_mul_total_fwd_zone_pass_mean_diff',\n",
+       "  'big_chance_scored_mean_h',\n",
+       "  'blocked_cross_mean_a',\n",
+       "  'blocked_cross_mean_diff',\n",
+       "  'blocked_cross_mean_h',\n",
+       "  'contentious_decision_mean_a',\n",
+       "  'contentious_decision_mean_diff',\n",
+       "  'contentious_decision_mean_h',\n",
+       "  'contentious_decision_std_a',\n",
+       "  'contentious_decision_std_diff',\n",
+       "  'contentious_decision_std_h',\n",
+       "  'corner_taken_std_mul_att_corner_ratio_mean_diffs',\n",
+       "  'corners_ratio_mean_a',\n",
+       "  'corners_ratio_mean_diff',\n",
+       "  'corners_ratio_mean_h',\n",
+       "  'defender_goals_mean_a',\n",
+       "  'defender_goals_mean_diff',\n",
+       "  'defender_goals_mean_h',\n",
+       "  'diving_save_std_a',\n",
+       "  'diving_save_std_diff',\n",
+       "  'diving_save_std_h',\n",
+       "  'effective_clearance_mean_a',\n",
+       "  'effective_clearance_mean_diff',\n",
+       "  'effective_clearance_mean_h',\n",
+       "  'effective_head_clearance_mean_a',\n",
+       "  'effective_head_clearance_mean_diff',\n",
+       "  'effective_head_clearance_mean_h',\n",
+       "  'first_yellow_card_std_a',\n",
+       "  'first_yellow_card_std_diff',\n",
+       "  'first_yellow_card_std_h',\n",
+       "  'forward_goals_mean_a',\n",
+       "  'forward_goals_mean_diff',\n",
+       "  'forward_goals_mean_h',\n",
+       "  'fwd_pass_div_long_pass_own_to_opp_success_std_a',\n",
+       "  'fwd_pass_div_long_pass_own_to_opp_success_std_diff',\n",
+       "  'fwd_pass_div_long_pass_own_to_opp_success_std_h',\n",
+       "  'goal_assist_deadball_mean_a',\n",
+       "  'goal_assist_deadball_mean_diff',\n",
+       "  'goal_assist_deadball_mean_h',\n",
+       "  'goal_assist_deadball_std_a',\n",
+       "  'goal_assist_deadball_std_diff',\n",
+       "  'goal_assist_deadball_std_h',\n",
+       "  'goal_assist_intent_norm_mean_a',\n",
+       "  'goal_assist_intent_norm_mean_diff',\n",
+       "  'goal_assist_intent_norm_mean_h',\n",
+       "  'goal_assist_intentional_mean_a',\n",
+       "  'goal_assist_intentional_mean_diff',\n",
+       "  'goal_assist_intentional_mean_h',\n",
+       "  'goal_assist_intentional_std_a',\n",
+       "  'goal_assist_intentional_std_diff',\n",
+       "  'goal_assist_intentional_std_h',\n",
+       "  'goal_assist_mean_a',\n",
+       "  'goal_assist_mean_diff',\n",
+       "  'goal_assist_mean_h',\n",
+       "  'goal_assist_openplay_std_a',\n",
+       "  'goal_assist_openplay_std_diff',\n",
+       "  'goal_assist_openplay_std_h',\n",
+       "  'goal_assist_setplay_mean_a',\n",
+       "  'goal_assist_setplay_mean_diff',\n",
+       "  'goal_assist_setplay_mean_h',\n",
+       "  'goal_assist_std_a',\n",
+       "  'goal_assist_std_diff',\n",
+       "  'goal_assist_std_h',\n",
+       "  'goal_fastbreak_mean_a',\n",
+       "  'goal_fastbreak_mean_diff',\n",
+       "  'goal_fastbreak_mean_h',\n",
+       "  'goal_kicks_per_shot_mean_a',\n",
+       "  'goal_kicks_per_shot_mean_diff',\n",
+       "  'goal_kicks_per_shot_mean_h',\n",
+       "  'goals_2t_mean_a',\n",
+       "  'goals_2t_mean_diff',\n",
+       "  'goals_2t_mean_h',\n",
+       "  'goals_mean_a',\n",
+       "  'goals_mean_diff',\n",
+       "  'goals_mean_h',\n",
+       "  'goals_openplay_mean_a',\n",
+       "  'goals_openplay_mean_diff',\n",
+       "  'goals_openplay_mean_h',\n",
+       "  'high_to_low_goals_mean_a',\n",
+       "  'high_to_low_goals_mean_diff',\n",
+       "  'high_to_low_goals_mean_h',\n",
+       "  'imp_prob_over_goals_0.5_a',\n",
+       "  'imp_prob_over_goals_0.5_diff',\n",
+       "  'imp_prob_over_goals_0.5_h',\n",
+       "  'imp_prob_over_goals_1.5_a',\n",
+       "  'imp_prob_over_goals_1.5_diff',\n",
+       "  'imp_prob_over_goals_1.5_h',\n",
+       "  'imp_prob_over_goals_2.5_a',\n",
+       "  'imp_prob_over_goals_2.5_diff',\n",
+       "  'imp_prob_over_goals_2.5_h',\n",
+       "  'imp_prob_over_goals_3.5_a',\n",
+       "  'imp_prob_over_goals_3.5_diff',\n",
+       "  'imp_prob_over_goals_3.5_h',\n",
+       "  'imp_prob_over_goals_4.5_a',\n",
+       "  'imp_prob_over_goals_4.5_diff',\n",
+       "  'imp_prob_over_goals_4.5_h',\n",
+       "  'imp_prob_under_goals_0.5_a',\n",
+       "  'imp_prob_under_goals_0.5_diff',\n",
+       "  'imp_prob_under_goals_0.5_h',\n",
+       "  'imp_prob_under_goals_1.5_a',\n",
+       "  'imp_prob_under_goals_1.5_diff',\n",
+       "  'imp_prob_under_goals_1.5_h',\n",
+       "  'imp_prob_under_goals_2.5_a',\n",
+       "  'imp_prob_under_goals_2.5_diff',\n",
+       "  'imp_prob_under_goals_2.5_h',\n",
+       "  'imp_prob_under_goals_3.5_a',\n",
+       "  'imp_prob_under_goals_3.5_diff',\n",
+       "  'imp_prob_under_goals_3.5_h',\n",
+       "  'imp_prob_under_goals_4.5_a',\n",
+       "  'imp_prob_under_goals_4.5_diff',\n",
+       "  'imp_prob_under_goals_4.5_h',\n",
+       "  'interceptions_in_box_std_a',\n",
+       "  'interceptions_in_box_std_diff',\n",
+       "  'interceptions_in_box_std_h',\n",
+       "  'last_man_tackle_mean_a',\n",
+       "  'last_man_tackle_mean_diff',\n",
+       "  'last_man_tackle_mean_h',\n",
+       "  'last_man_tackle_std_a',\n",
+       "  'last_man_tackle_std_diff',\n",
+       "  'last_man_tackle_std_h',\n",
+       "  'left_div_right_foot_goals_mean_a',\n",
+       "  'left_div_right_foot_goals_mean_diff',\n",
+       "  'left_div_right_foot_goals_mean_h',\n",
+       "  'leftside_pass_div_accurate_pass_std_a',\n",
+       "  'leftside_pass_div_accurate_pass_std_diff',\n",
+       "  'leftside_pass_div_accurate_pass_std_h',\n",
+       "  'leftside_pass_mean_a',\n",
+       "  'leftside_pass_mean_diff',\n",
+       "  'leftside_pass_mean_h',\n",
+       "  'long_pass_own_to_opp_success_std_a',\n",
+       "  'long_pass_own_to_opp_success_std_diff',\n",
+       "  'long_pass_own_to_opp_success_std_h',\n",
+       "  'lost_corners_mean_a',\n",
+       "  'lost_corners_mean_diff',\n",
+       "  'lost_corners_mean_h',\n",
+       "  'midfielder_goals_mean_a',\n",
+       "  'midfielder_goals_mean_diff',\n",
+       "  'midfielder_goals_mean_h',\n",
+       "  'no_foot_goals_ratio_mean_a',\n",
+       "  'no_foot_goals_ratio_mean_diff',\n",
+       "  'no_foot_goals_ratio_mean_h',\n",
+       "  'no_foot_goals_ratio_std_a',\n",
+       "  'no_foot_goals_ratio_std_diff',\n",
+       "  'no_foot_goals_ratio_std_h',\n",
+       "  'odd_ratio_over_corners_11.5',\n",
+       "  'odd_ratio_over_corners_9.5',\n",
+       "  'odd_ratio_under_corners_10.5',\n",
+       "  'odd_ratio_under_goals_1.5',\n",
+       "  'odd_ratio_under_goals_2.5',\n",
+       "  'odd_ratio_under_goals_3.5',\n",
+       "  'odd_ratio_under_goals_4.5',\n",
+       "  'odd_ratio_under_goals_4.5',\n",
+       "  'odds_away_over_cards_4.5_a',\n",
+       "  'odds_away_over_cards_4.5_diff',\n",
+       "  'odds_away_over_cards_4.5_h',\n",
+       "  'odds_away_over_cards_5.5_a',\n",
+       "  'odds_away_over_cards_5.5_diff',\n",
+       "  'odds_away_over_cards_5.5_h',\n",
+       "  'odds_away_under_cards_3.5_a',\n",
+       "  'odds_away_under_cards_3.5_diff',\n",
+       "  'odds_away_under_cards_3.5_h',\n",
+       "  'odds_home_under_cards_5.5_a',\n",
+       "  'odds_home_under_cards_5.5_diff',\n",
+       "  'odds_home_under_cards_5.5_h',\n",
+       "  'odds_home_under_goals_0.5_a',\n",
+       "  'odds_home_under_goals_0.5_diff',\n",
+       "  'odds_home_under_goals_0.5_h',\n",
+       "  'ontarget_att_assist_mean_a',\n",
+       "  'ontarget_att_assist_mean_diff',\n",
+       "  'ontarget_att_assist_mean_h',\n",
+       "  'own_goals_mean_a',\n",
+       "  'own_goals_mean_diff',\n",
+       "  'own_goals_mean_h',\n",
+       "  'own_goals_std_a',\n",
+       "  'own_goals_std_diff',\n",
+       "  'own_goals_std_h',\n",
+       "  'passes_left_mean_a',\n",
+       "  'passes_left_mean_diff',\n",
+       "  'passes_left_mean_h',\n",
+       "  'passes_right_mean_a',\n",
+       "  'passes_right_mean_diff',\n",
+       "  'passes_right_mean_h',\n",
+       "  'passes_right_std_a',\n",
+       "  'passes_right_std_diff',\n",
+       "  'passes_right_std_h',\n",
+       "  'pen_goals_conceded_std_a',\n",
+       "  'pen_goals_conceded_std_diff',\n",
+       "  'pen_goals_conceded_std_h',\n",
+       "  'penalty_conceded_mean_a',\n",
+       "  'penalty_conceded_mean_diff',\n",
+       "  'penalty_conceded_mean_h',\n",
+       "  'penalty_faced_std_a',\n",
+       "  'penalty_faced_std_diff',\n",
+       "  'penalty_faced_std_h',\n",
+       "  'penalty_save_std_a',\n",
+       "  'penalty_save_std_diff',\n",
+       "  'penalty_save_std_h',\n",
+       "  'points_a',\n",
+       "  'points_diff',\n",
+       "  'points_h',\n",
+       "  'points_mean_a',\n",
+       "  'points_mean_diff',\n",
+       "  'points_mean_h',\n",
+       "  'points_std_a',\n",
+       "  'points_std_diff',\n",
+       "  'points_std_h',\n",
+       "  'poss_won_att_3rd_std_a',\n",
+       "  'poss_won_att_3rd_std_diff',\n",
+       "  'poss_won_att_3rd_std_h',\n",
+       "  'prob_over_goals_1.5',\n",
+       "  'prob_over_goals_2.5',\n",
+       "  'prob_over_goals_3.5',\n",
+       "  'prob_over_goals_4.5',\n",
+       "  'prob_squared_over_goals_1.5',\n",
+       "  'prob_squared_over_goals_2.5',\n",
+       "  'prob_squared_over_goals_3.5',\n",
+       "  'prob_squared_over_goals_4.5',\n",
+       "  'prob_squared_under_goals_1.5',\n",
+       "  'prob_squared_under_goals_2.5',\n",
+       "  'prob_squared_under_goals_3.5',\n",
+       "  'prob_squared_under_goals_4.5',\n",
+       "  'prob_under_goals_1.5',\n",
+       "  'prob_under_goals_2.5',\n",
+       "  'prob_under_goals_3.5',\n",
+       "  'prob_under_goals_4.5',\n",
+       "  'ratio_over_corners_11.5_a',\n",
+       "  'ratio_over_corners_11.5_diff',\n",
+       "  'ratio_over_corners_11.5_h',\n",
+       "  'ratio_over_corners_9.5_a',\n",
+       "  'ratio_over_corners_9.5_diff',\n",
+       "  'ratio_over_corners_9.5_h',\n",
+       "  'ratio_under_corners_10.5_a',\n",
+       "  'ratio_under_corners_10.5_diff',\n",
+       "  'ratio_under_corners_10.5_h',\n",
+       "  'ratio_under_goals_4.5_a',\n",
+       "  'ratio_under_goals_4.5_diff',\n",
+       "  'ratio_under_goals_4.5_h',\n",
+       "  'raw_prob_over_goals_1.5',\n",
+       "  'raw_prob_over_goals_1.5',\n",
+       "  'raw_prob_over_goals_2.5',\n",
+       "  'raw_prob_over_goals_2.5',\n",
+       "  'raw_prob_over_goals_3.5',\n",
+       "  'raw_prob_over_goals_3.5',\n",
+       "  'raw_prob_over_goals_4.5',\n",
+       "  'raw_prob_over_goals_4.5',\n",
+       "  'raw_prob_under_goals_1.5',\n",
+       "  'raw_prob_under_goals_2.5',\n",
+       "  'raw_prob_under_goals_3.5',\n",
+       "  'raw_prob_under_goals_4.5',\n",
+       "  'red_card_1t_std_a',\n",
+       "  'red_card_1t_std_diff',\n",
+       "  'red_card_1t_std_h',\n",
+       "  'red_card_2t_mean_a',\n",
+       "  'red_card_2t_mean_diff',\n",
+       "  'red_card_2t_mean_h',\n",
+       "  'red_card_mean_a',\n",
+       "  'red_card_mean_diff',\n",
+       "  'red_card_mean_h',\n",
+       "  'red_card_std_a',\n",
+       "  'red_card_std_diff',\n",
+       "  'red_card_std_h',\n",
+       "  'rescinded_red_card_mean_a',\n",
+       "  'rescinded_red_card_mean_diff',\n",
+       "  'rescinded_red_card_mean_h',\n",
+       "  'rescinded_red_card_std_a',\n",
+       "  'rescinded_red_card_std_diff',\n",
+       "  'rescinded_red_card_std_h',\n",
+       "  'right_to_left_goals_mean_a',\n",
+       "  'right_to_left_goals_mean_a',\n",
+       "  'right_to_left_goals_mean_diff',\n",
+       "  'right_to_left_goals_mean_diff',\n",
+       "  'right_to_left_goals_mean_h',\n",
+       "  'right_to_left_goals_mean_h',\n",
+       "  'right_to_left_goals_std_a',\n",
+       "  'right_to_left_goals_std_diff',\n",
+       "  'right_to_left_goals_std_h',\n",
+       "  'rightside_pass_div_accurate_pass_std_a',\n",
+       "  'rightside_pass_div_accurate_pass_std_diff',\n",
+       "  'rightside_pass_div_accurate_pass_std_h',\n",
+       "  'rightside_pass_div_leftside_pass_mean_a',\n",
+       "  'rightside_pass_div_leftside_pass_mean_diff',\n",
+       "  'rightside_pass_div_leftside_pass_mean_h',\n",
+       "  'saves_std_a',\n",
+       "  'saves_std_diff',\n",
+       "  'saves_std_h',\n",
+       "  'second_yellow_mean_a',\n",
+       "  'second_yellow_mean_diff',\n",
+       "  'second_yellow_mean_h',\n",
+       "  'second_yellow_std_a',\n",
+       "  'second_yellow_std_diff',\n",
+       "  'second_yellow_std_h',\n",
+       "  'shots_div_passes_left_mean_a',\n",
+       "  'shots_div_passes_left_mean_diff',\n",
+       "  'shots_div_passes_left_mean_h',\n",
+       "  'shots_div_passes_right_mean_a',\n",
+       "  'shots_div_passes_right_mean_diff',\n",
+       "  'shots_div_passes_right_mean_h',\n",
+       "  'shots_div_passes_right_std_a',\n",
+       "  'shots_div_passes_right_std_diff',\n",
+       "  'shots_div_passes_right_std_h',\n",
+       "  'shots_mul_goals_mean_a',\n",
+       "  'shots_mul_goals_mean_diff',\n",
+       "  'shots_mul_goals_mean_h',\n",
+       "  'shots_std_a',\n",
+       "  'shots_std_diff',\n",
+       "  'shots_std_h',\n",
+       "  'successful_final_third_passes_std_a',\n",
+       "  'successful_final_third_passes_std_diff',\n",
+       "  'successful_final_third_passes_std_h',\n",
+       "  'total_bets_under_cards_4.5_a',\n",
+       "  'total_bets_under_cards_4.5_diff',\n",
+       "  'total_bets_under_cards_4.5_h',\n",
+       "  'total_bets_under_corners_9.5_a',\n",
+       "  'total_bets_under_corners_9.5_diff',\n",
+       "  'total_bets_under_corners_9.5_h',\n",
+       "  'total_clearance_mean_a',\n",
+       "  'total_clearance_mean_diff',\n",
+       "  'total_clearance_mean_h',\n",
+       "  'total_expulsions_ref',\n",
+       "  'total_fastbreak_std_a',\n",
+       "  'total_fastbreak_std_diff',\n",
+       "  'total_fastbreak_std_h',\n",
+       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_a',\n",
+       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_diff',\n",
+       "  'total_fwd_zone_pass_div_long_pass_own_to_opp_success_std_h',\n",
+       "  'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_diff',\n",
+       "  'total_fwd_zone_pass_std_a',\n",
+       "  'total_fwd_zone_pass_std_diff',\n",
+       "  'total_fwd_zone_pass_std_h',\n",
+       "  'total_high_claim_mean_a',\n",
+       "  'total_high_claim_mean_diff',\n",
+       "  'total_high_claim_mean_h',\n",
+       "  'total_keeper_sweeper_mean_a',\n",
+       "  'total_keeper_sweeper_mean_diff',\n",
+       "  'total_keeper_sweeper_mean_h',\n",
+       "  'total_keeper_sweeper_std_a',\n",
+       "  'total_keeper_sweeper_std_diff',\n",
+       "  'total_keeper_sweeper_std_h',\n",
+       "  'total_launches_mean_a',\n",
+       "  'total_launches_mean_diff',\n",
+       "  'total_launches_mean_h',\n",
+       "  'total_launches_std_a',\n",
+       "  'total_launches_std_diff',\n",
+       "  'total_launches_std_h',\n",
+       "  'total_layoffs_mean_a',\n",
+       "  'total_layoffs_mean_diff',\n",
+       "  'total_layoffs_mean_h',\n",
+       "  'total_layoffs_std_a',\n",
+       "  'total_layoffs_std_diff',\n",
+       "  'total_layoffs_std_h',\n",
+       "  'total_red_card_match_ref',\n",
+       "  'total_red_card_mean_a',\n",
+       "  'total_red_card_mean_diff',\n",
+       "  'total_red_card_mean_h',\n",
+       "  'total_red_card_std_a',\n",
+       "  'total_red_card_std_diff',\n",
+       "  'total_red_card_std_h',\n",
+       "  'total_second_yel_card_away_ref',\n",
+       "  'total_second_yel_card_home_ref',\n",
+       "  'total_second_yel_card_ref',\n",
+       "  'total_through_ball_mean_a',\n",
+       "  'total_through_ball_mean_diff',\n",
+       "  'total_through_ball_mean_h',\n",
+       "  'total_through_ball_std_a',\n",
+       "  'total_through_ball_std_diff',\n",
+       "  'total_through_ball_std_h',\n",
+       "  'total_throws_std_a',\n",
+       "  'total_throws_std_diff',\n",
+       "  'total_throws_std_h',\n",
+       "  'total_win_pct_over_corners_9.5_a',\n",
+       "  'total_win_pct_over_corners_9.5_diff',\n",
+       "  'total_win_pct_over_corners_9.5_h',\n",
+       "  'total_win_pct_over_goals_2.5_a',\n",
+       "  'total_win_pct_over_goals_2.5_diff',\n",
+       "  'total_win_pct_over_goals_2.5_h',\n",
+       "  'total_win_pct_under_goals_1.5_a',\n",
+       "  'total_win_pct_under_goals_1.5_diff',\n",
+       "  'total_win_pct_under_goals_1.5_h',\n",
+       "  'total_win_pct_under_goals_3.5_a',\n",
+       "  'total_win_pct_under_goals_3.5_diff',\n",
+       "  'total_win_pct_under_goals_3.5_h',\n",
+       "  'total_yel_card_std_a',\n",
+       "  'total_yel_card_std_diff',\n",
+       "  'total_yel_card_std_h',\n",
+       "  'win_pct_away_over_goals_2.5_a',\n",
+       "  'win_pct_away_over_goals_2.5_diff',\n",
+       "  'win_pct_away_over_goals_2.5_h',\n",
+       "  'win_pct_home_over_cards_6.5_a',\n",
+       "  'win_pct_home_over_cards_6.5_diff',\n",
+       "  'win_pct_home_over_cards_6.5_h',\n",
+       "  'win_pct_home_over_goals_2.5_a',\n",
+       "  'win_pct_home_over_goals_2.5_diff',\n",
+       "  'win_pct_home_over_goals_2.5_h',\n",
+       "  'win_pct_home_under_cards_4.5_a',\n",
+       "  'win_pct_home_under_cards_4.5_diff',\n",
+       "  'win_pct_home_under_cards_4.5_h',\n",
+       "  'won_corners_ratio_mean_a',\n",
+       "  'won_corners_ratio_mean_a',\n",
+       "  'won_corners_ratio_mean_diff',\n",
+       "  'won_corners_ratio_mean_diff',\n",
+       "  'won_corners_ratio_mean_h',\n",
+       "  'won_corners_ratio_mean_h',\n",
+       "  'won_corners_ratio_std_a',\n",
+       "  'won_corners_ratio_std_diff',\n",
+       "  'won_corners_ratio_std_h'])"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "common_feat = list(set([\"_\".join(x.split(\"_\")[:-1]) for x in selected_features]))\n",
+    "new_subset = [x for x in train_data.columns for c in common_feat if c in x]\n",
+    "\n",
+    "len(selected_features), len(new_subset), sorted(new_subset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\"\"\"Monkey patch feature_selection module.\"\"\"\n",
+    "from typing import List, Tuple\n",
+    "\n",
+    "# from feature_selection import FeatureSelection\n",
+    "\n",
+    "\n",
+    "def get_root_features(feature_list: List) -> Tuple[List, List]:\n",
+    "    \"\"\"Get the features related to teams, i.e., those ended with \"_a\", \"_h\", \"_diff\".\"\"\"\n",
+    "    # Conditions\n",
+    "    cond1 = lambda x: x.endswith(\"_h\")\n",
+    "    cond2 = lambda x: x.endswith(\"_a\")\n",
+    "    cond3 = lambda x: x.endswith(\"_diff\")\n",
+    "    # Features ended with _h, _a, _diff\n",
+    "    sublist = [c for c in feature_list if (cond1(c) or cond2(c) or cond3(c))]\n",
+    "    return sublist, list(set([\"_\".join(c.split(\"_\")[:-1]) for c in sublist]))\n",
+    "\n",
+    "\n",
+    "def append_list(root_features: List, og_columns: List):\n",
+    "    \"\"\"Append suffixes to the incoming feature labels.\"\"\"\n",
+    "    # Create empty list\n",
+    "    dressed_list = list()\n",
+    "    # Add  suffixes to each root label\n",
+    "    # dressed_list = [c for root in root_features for c in df_columns if c.startswith(root)]\n",
+    "    for c in root_features:\n",
+    "        dressed_list.append(f\"{c}_h\")\n",
+    "        dressed_list.append(f\"{c}_a\")\n",
+    "        dressed_list.append(f\"{c}_diff\")\n",
+    "    dressed_list = [c for c in dressed_list if c in og_columns]\n",
+    "    # return list(set(dressed_list))\n",
+    "    return dressed_list\n",
+    "\n",
+    "\n",
+    "def massage_feat_list(feature_list: List, og_columns: List):\n",
+    "    \"\"\"\n",
+    "    Add team features to the incoming list.\n",
+    "\n",
+    "    Given a feature list, this function adds the full trio of team\n",
+    "    features (*_h, *_a, *_diff) to those already present that are\n",
+    "    related to team statistics.\n",
+    "    \"\"\"\n",
+    "    feature_list = feature_list.copy()\n",
+    "    # Get list of match features and root\n",
+    "    sublist, root = get_root_features(feature_list)\n",
+    "    # Get features not related to teams\n",
+    "    no_team = list(set(feature_list) - set(sublist))\n",
+    "    # Add suffixes to roots\n",
+    "    dressed = append_list(root_features=root, og_columns=og_columns.copy())\n",
+    "    # return list(set(no_team + dressed))\n",
+    "    return no_team + dressed\n",
+    "\n",
+    "\n",
+    "def finalize_list(feature_list: List, df_columns: List):\n",
+    "    \"\"\"\n",
+    "    Complete the feature_list returned by the selection module.\n",
+    "\n",
+    "    Once the number of features has been reduced to a subset of\n",
+    "    relevant features, this function completes the list by adding\n",
+    "    team features to those already present.\n",
+    "\n",
+    "    If a team feature is present, but the set only contains the one\n",
+    "    related to the home or away team, the function adds the corresponding\n",
+    "    feature related to the difference between teams, i.e., team_feat_diff.\n",
+    "    \"\"\"\n",
+    "    feature_list = feature_list.copy()\n",
+    "    # Elements with _h, _a, _diff\n",
+    "    sublist, roots = get_root_features(feature_list)\n",
+    "    # List to be used as building block\n",
+    "    final_list = list(set(feature_list) - set(sublist))\n",
+    "    # Check for team features\n",
+    "    for root in roots:\n",
+    "        triplet = [f\"{root}_h\", f\"{root}_a\", f\"{root}_diff\"]\n",
+    "        subset = [c for tri in triplet for c in sublist if c == tri]\n",
+    "        if len(subset) < 2 and not subset[0].endswith(\"_diff\"):\n",
+    "            subset.append(f\"{root}_diff\")\n",
+    "        final_list = final_list + subset\n",
+    "    final_list = [c for c in final_list if c in df_columns]\n",
+    "    return final_list\n",
+    "\n",
+    "\n",
+    "def patch_repeat_pipeline(self):\n",
+    "    \"\"\"Iterate over the process to create the feature list and repeat it self.repeat times (PATCH).\"\"\"\n",
+    "    i = 0\n",
+    "    while len(self.feature_list) > self.target_features:\n",
+    "        i += 1\n",
+    "        print(f\"{i} iteration\")\n",
+    "        # Call iteration\n",
+    "        self.create_feature_list()\n",
+    "        self.feature_list = massage_feat_list(\n",
+    "            feature_list=self.feature_list, og_columns=self.x_df.columns\n",
+    "        )\n",
+    "        if len(self.feature_list) <= 1:\n",
+    "            break\n",
+    "    return finalize_list(feature_list=self.feature_list, df_columns=self.x_df.columns)\n",
+    "\n",
+    "def patch_train_model(self):\n",
+    "        \"\"\"Preprocess the data and select self.number_models top models.\"\"\"\n",
+    "        # Selected dataset\n",
+    "        selected_cols = self.feature_list + [self.target]\n",
+    "        train_data = self.dataset[selected_cols] if self.x_df.empty else self.x_df[selected_cols]\n",
+    "        # Numeric features\n",
+    "        self.setup_kwargs[\"numeric_features\"] = [\n",
+    "            c for c in self.numeric_features if c in self.feature_list\n",
+    "        ]\n",
+    "        # Ignore features\n",
+    "        self.setup_kwargs[\"ignore_features\"] = [\n",
+    "            c for c in self.ignore_features if c in self.feature_list\n",
+    "        ]\n",
+    "        self.setup_kwargs[\"feature_interaction\"]: False\n",
+    "        # Initialize pycaret setup\n",
+    "        setup(data=train_data, target=self.target, **self.setup_kwargs)\n",
+    "        # Get train dataset and preprocessed dataframe\n",
+    "        self.x_train = get_config(\"X_train\")\n",
+    "        if self.x_df.empty:  # TODO change x_df by dataset and add flag?\n",
+    "            self.x_df = pd.concat([get_config(\"X\"), get_config(\"y\")], axis=1)\n",
+    "            self.setup_kwargs[\"preprocess\"] = False  # Turn off preprocessing\n",
+    "        # Compare models\n",
+    "        self.top_models = self._training_function(**self._args)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def patch_train_model(self):\n",
+    "        \"\"\"Preprocess the data and select self.number_models top models.\"\"\"\n",
+    "        # Selected dataset\n",
+    "        selected_cols = self.feature_list + [self.target]\n",
+    "        train_data = self.dataset[selected_cols] if self.x_df.empty else self.x_df[selected_cols]\n",
+    "        # Numeric features\n",
+    "        self.setup_kwargs[\"numeric_features\"] = [\n",
+    "            c for c in self.numeric_features if c in self.feature_list\n",
+    "        ]\n",
+    "        # Ignore features\n",
+    "        self.setup_kwargs[\"ignore_features\"] = [\n",
+    "            c for c in self.ignore_features if c in self.feature_list\n",
+    "        ]\n",
+    "        self.setup_kwargs[\"feature_interaction\"]: False\n",
+    "        # Initialize pycaret setup\n",
+    "        setup(data=train_data, target=self.target, **self.setup_kwargs)\n",
+    "        # Get train dataset and preprocessed dataframe\n",
+    "        self.x_train = get_config(\"X_train\")\n",
+    "        if self.x_df.empty:  # TODO change x_df by dataset and add flag?\n",
+    "            self.x_df = pd.concat([get_config(\"X\"), get_config(\"y\")], axis=1)\n",
+    "            self.setup_kwargs[\"preprocess\"] = False  # Turn off preprocessing\n",
+    "        # Compare models\n",
+    "        self.top_models = self._training_function(**self._args)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING:param: Setting non-Parameter class attribute FeatureSelection.train_model = <function patch_train_model at 0x000001FF45758310> \n"
+     ]
+    }
+   ],
+   "source": [
+    "FeatureSelection.train_model = patch_train_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING:param: Setting non-Parameter class attribute FeatureSelection.repeat_pipeline = <function patch_repeat_pipeline at 0x000001FF457581F0> \n"
+     ]
+    }
+   ],
+   "source": [
+    "FeatureSelection.repeat_pipeline = patch_repeat_pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<function __main__.patch_repeat_pipeline(self)>"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "FeatureSelection.repeat_pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "FeatureSelection.filter_tuned_duplicate??"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1 iteration\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5186</td>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5299</td>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.5000</td>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5286</td>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5139</td>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0378</td>\n",
+       "                        <td id=\"T_77431f22_8502_11ec_b8d3_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0378</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1ff60920940>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_5ef5dfe9_8505_11ec_8fac_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1ff59c52370>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_5f1cf0ab_8505_11ec_a60d_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1ff60901fd0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_5f4735d9_8505_11ec_9646_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1ff60920c40>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "</style><table id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col1\" class=\"data row0 col1\" >0.5089</td>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col2\" class=\"data row0 col2\" >0.5322</td>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col3\" class=\"data row0 col3\" >0.4892</td>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col4\" class=\"data row0 col4\" >0.5186</td>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col5\" class=\"data row0 col5\" >0.5035</td>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col6\" class=\"data row0 col6\" >0.0186</td>\n",
+       "                        <td id=\"T_5f71f023_8505_11ec_aa29_d8c0a63024d2row0_col7\" class=\"data row0 col7\" >0.0186</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x1ff5e926f10>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "metric_param = {\n",
+    "        \"Accuracy\": 0.1,\n",
+    "        \"AUC\": 0.1,\n",
+    "        \"Recall\": 0.1,\n",
+    "        \"Precision\": 0.1,\n",
+    "        \"F1\": 0.1,\n",
+    "        \"Kappa\": -1.0,\n",
+    "        \"MCC\": -1.0,\n",
+    "    }\n",
+    "feat_sel = FeatureSelection(target=target,\n",
+    "                            dataset=train_data.dropna(),#[list(set(new_subset+new_feat+[target]))],\n",
+    "                            target_features=0.56,\n",
+    "                            filter_metrics=metric_param,\n",
+    "                            include=[\"lr\"],\n",
+    "                            setup_kwargs=setup_kwargs,\n",
+    "                            optimize=True,\n",
+    "                            opt_list=[\"AUC\" , \"Accuracy\" , \"Precision\" , \"Recall\"],\n",
+    "                            number_features = 0.56\n",
+    "                           )\n",
+    "triplet_selected_features = feat_sel.repeat_pipeline()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1445"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(triplet_selected_features)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>win_pct_h</th>\n",
+       "      <th>draw_pct_h</th>\n",
+       "      <th>loss_pct_h</th>\n",
+       "      <th>win_streak_h</th>\n",
+       "      <th>loss_streak_h</th>\n",
+       "      <th>draw_streak_h</th>\n",
+       "      <th>points_h</th>\n",
+       "      <th>ranking_h</th>\n",
+       "      <th>ATT_h</th>\n",
+       "      <th>MED_h</th>\n",
+       "      <th>...</th>\n",
+       "      <th>total_expulsions_ref</th>\n",
+       "      <th>total_tackle_ref</th>\n",
+       "      <th>total_fk_foul_ref</th>\n",
+       "      <th>total_penalty_match_ref</th>\n",
+       "      <th>total_hand_ball_ref</th>\n",
+       "      <th>total_fouls_home_ref</th>\n",
+       "      <th>total_fouls_away_ref</th>\n",
+       "      <th>total_fouls_match_ref</th>\n",
+       "      <th>fk_per_tackle_ref</th>\n",
+       "      <th>goals_2.5</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>match_id</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1069833_german_bundesliga</th>\n",
+       "      <td>-1.643901</td>\n",
+       "      <td>1.533277</td>\n",
+       "      <td>0.214646</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>-0.783440</td>\n",
+       "      <td>1.612666</td>\n",
+       "      <td>0.639803</td>\n",
+       "      <td>-0.179871</td>\n",
+       "      <td>0.315282</td>\n",
+       "      <td>-0.131555</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.985318</td>\n",
+       "      <td>1.101278</td>\n",
+       "      <td>-1.029729</td>\n",
+       "      <td>-0.793801</td>\n",
+       "      <td>0.917055</td>\n",
+       "      <td>-0.732395</td>\n",
+       "      <td>-0.728244</td>\n",
+       "      <td>-0.827393</td>\n",
+       "      <td>-1.218083</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1069835_german_bundesliga</th>\n",
+       "      <td>-0.664834</td>\n",
+       "      <td>-1.429944</td>\n",
+       "      <td>1.660861</td>\n",
+       "      <td>1.123439</td>\n",
+       "      <td>-0.783440</td>\n",
+       "      <td>-0.599329</td>\n",
+       "      <td>0.372292</td>\n",
+       "      <td>0.631282</td>\n",
+       "      <td>-0.405111</td>\n",
+       "      <td>-0.514660</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.369170</td>\n",
+       "      <td>0.356790</td>\n",
+       "      <td>-0.346304</td>\n",
+       "      <td>-1.454542</td>\n",
+       "      <td>0.156403</td>\n",
+       "      <td>0.124371</td>\n",
+       "      <td>-0.627903</td>\n",
+       "      <td>-0.305101</td>\n",
+       "      <td>-0.380686</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1069836_german_bundesliga</th>\n",
+       "      <td>-0.664834</td>\n",
+       "      <td>-0.199662</td>\n",
+       "      <td>0.974545</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>1.710316</td>\n",
+       "      <td>-0.599329</td>\n",
+       "      <td>0.372292</td>\n",
+       "      <td>0.959403</td>\n",
+       "      <td>-1.008551</td>\n",
+       "      <td>-0.857581</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.542920</td>\n",
+       "      <td>0.774597</td>\n",
+       "      <td>1.110759</td>\n",
+       "      <td>-1.454542</td>\n",
+       "      <td>-0.907535</td>\n",
+       "      <td>1.374438</td>\n",
+       "      <td>0.197598</td>\n",
+       "      <td>0.864320</td>\n",
+       "      <td>0.337828</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1069834_german_bundesliga</th>\n",
+       "      <td>-0.664834</td>\n",
+       "      <td>2.212205</td>\n",
+       "      <td>-1.666619</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>-0.783440</td>\n",
+       "      <td>1.869795</td>\n",
+       "      <td>0.058238</td>\n",
+       "      <td>-0.025558</td>\n",
+       "      <td>-0.717569</td>\n",
+       "      <td>-1.320451</td>\n",
+       "      <td>...</td>\n",
+       "      <td>1.406218</td>\n",
+       "      <td>0.391996</td>\n",
+       "      <td>1.565295</td>\n",
+       "      <td>2.265159</td>\n",
+       "      <td>1.301284</td>\n",
+       "      <td>1.168461</td>\n",
+       "      <td>1.635305</td>\n",
+       "      <td>1.640432</td>\n",
+       "      <td>0.764458</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1069842_german_bundesliga</th>\n",
+       "      <td>-0.664834</td>\n",
+       "      <td>0.759340</td>\n",
+       "      <td>0.214646</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>-0.783440</td>\n",
+       "      <td>1.612666</td>\n",
+       "      <td>0.721268</td>\n",
+       "      <td>0.389111</td>\n",
+       "      <td>-0.717569</td>\n",
+       "      <td>-0.514660</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.296511</td>\n",
+       "      <td>0.965913</td>\n",
+       "      <td>-0.156935</td>\n",
+       "      <td>-0.105425</td>\n",
+       "      <td>-0.590373</td>\n",
+       "      <td>0.562788</td>\n",
+       "      <td>-0.929066</td>\n",
+       "      <td>-0.234115</td>\n",
+       "      <td>-0.369954</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2219403_spanish_la_liga</th>\n",
+       "      <td>-1.643901</td>\n",
+       "      <td>0.759340</td>\n",
+       "      <td>0.974545</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>1.710316</td>\n",
+       "      <td>-0.599329</td>\n",
+       "      <td>-0.245043</td>\n",
+       "      <td>0.959403</td>\n",
+       "      <td>-0.238578</td>\n",
+       "      <td>0.080462</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.372171</td>\n",
+       "      <td>0.721725</td>\n",
+       "      <td>0.121370</td>\n",
+       "      <td>1.021130</td>\n",
+       "      <td>-0.051437</td>\n",
+       "      <td>-0.058292</td>\n",
+       "      <td>0.272404</td>\n",
+       "      <td>0.093577</td>\n",
+       "      <td>-0.371811</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2229046_italian_serie_a</th>\n",
+       "      <td>0.203837</td>\n",
+       "      <td>-1.429944</td>\n",
+       "      <td>0.974545</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>0.985387</td>\n",
+       "      <td>-0.599329</td>\n",
+       "      <td>-0.119351</td>\n",
+       "      <td>0.744759</td>\n",
+       "      <td>-0.405111</td>\n",
+       "      <td>0.306435</td>\n",
+       "      <td>...</td>\n",
+       "      <td>1.682687</td>\n",
+       "      <td>-2.489470</td>\n",
+       "      <td>0.417330</td>\n",
+       "      <td>0.763942</td>\n",
+       "      <td>0.568077</td>\n",
+       "      <td>0.583474</td>\n",
+       "      <td>0.272404</td>\n",
+       "      <td>0.460710</td>\n",
+       "      <td>1.857669</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2210408_english_premier_league</th>\n",
+       "      <td>-1.643901</td>\n",
+       "      <td>1.533277</td>\n",
+       "      <td>0.214646</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>0.985387</td>\n",
+       "      <td>-0.599329</td>\n",
+       "      <td>-1.194145</td>\n",
+       "      <td>0.389111</td>\n",
+       "      <td>0.315282</td>\n",
+       "      <td>-0.131555</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.372171</td>\n",
+       "      <td>1.580043</td>\n",
+       "      <td>-0.084230</td>\n",
+       "      <td>-0.105425</td>\n",
+       "      <td>0.685931</td>\n",
+       "      <td>0.208313</td>\n",
+       "      <td>-0.127036</td>\n",
+       "      <td>0.017975</td>\n",
+       "      <td>-1.040655</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2229048_italian_serie_a</th>\n",
+       "      <td>0.203837</td>\n",
+       "      <td>0.759340</td>\n",
+       "      <td>-0.670644</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>0.985387</td>\n",
+       "      <td>-0.599329</td>\n",
+       "      <td>-0.245043</td>\n",
+       "      <td>1.160365</td>\n",
+       "      <td>-0.405111</td>\n",
+       "      <td>-0.514660</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-1.639614</td>\n",
+       "      <td>1.974827</td>\n",
+       "      <td>0.809434</td>\n",
+       "      <td>-2.368850</td>\n",
+       "      <td>-0.590373</td>\n",
+       "      <td>0.275842</td>\n",
+       "      <td>0.869680</td>\n",
+       "      <td>0.638675</td>\n",
+       "      <td>-0.802420</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2210403_english_premier_league</th>\n",
+       "      <td>-0.664834</td>\n",
+       "      <td>1.533277</td>\n",
+       "      <td>-0.670644</td>\n",
+       "      <td>-0.714075</td>\n",
+       "      <td>-0.783440</td>\n",
+       "      <td>1.612666</td>\n",
+       "      <td>-0.728607</td>\n",
+       "      <td>2.103550</td>\n",
+       "      <td>0.121476</td>\n",
+       "      <td>-0.329155</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.022397</td>\n",
+       "      <td>0.586037</td>\n",
+       "      <td>-1.395555</td>\n",
+       "      <td>1.504698</td>\n",
+       "      <td>-1.906150</td>\n",
+       "      <td>-1.713468</td>\n",
+       "      <td>-1.029543</td>\n",
+       "      <td>-1.521511</td>\n",
+       "      <td>-1.583014</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2907 rows × 2200 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                win_pct_h  draw_pct_h  loss_pct_h  \\\n",
+       "match_id                                                            \n",
+       "1069833_german_bundesliga       -1.643901    1.533277    0.214646   \n",
+       "1069835_german_bundesliga       -0.664834   -1.429944    1.660861   \n",
+       "1069836_german_bundesliga       -0.664834   -0.199662    0.974545   \n",
+       "1069834_german_bundesliga       -0.664834    2.212205   -1.666619   \n",
+       "1069842_german_bundesliga       -0.664834    0.759340    0.214646   \n",
+       "...                                   ...         ...         ...   \n",
+       "2219403_spanish_la_liga         -1.643901    0.759340    0.974545   \n",
+       "2229046_italian_serie_a          0.203837   -1.429944    0.974545   \n",
+       "2210408_english_premier_league  -1.643901    1.533277    0.214646   \n",
+       "2229048_italian_serie_a          0.203837    0.759340   -0.670644   \n",
+       "2210403_english_premier_league  -0.664834    1.533277   -0.670644   \n",
+       "\n",
+       "                                win_streak_h  loss_streak_h  draw_streak_h  \\\n",
+       "match_id                                                                     \n",
+       "1069833_german_bundesliga          -0.714075      -0.783440       1.612666   \n",
+       "1069835_german_bundesliga           1.123439      -0.783440      -0.599329   \n",
+       "1069836_german_bundesliga          -0.714075       1.710316      -0.599329   \n",
+       "1069834_german_bundesliga          -0.714075      -0.783440       1.869795   \n",
+       "1069842_german_bundesliga          -0.714075      -0.783440       1.612666   \n",
+       "...                                      ...            ...            ...   \n",
+       "2219403_spanish_la_liga            -0.714075       1.710316      -0.599329   \n",
+       "2229046_italian_serie_a            -0.714075       0.985387      -0.599329   \n",
+       "2210408_english_premier_league     -0.714075       0.985387      -0.599329   \n",
+       "2229048_italian_serie_a            -0.714075       0.985387      -0.599329   \n",
+       "2210403_english_premier_league     -0.714075      -0.783440       1.612666   \n",
+       "\n",
+       "                                points_h  ranking_h     ATT_h     MED_h  ...  \\\n",
+       "match_id                                                                 ...   \n",
+       "1069833_german_bundesliga       0.639803  -0.179871  0.315282 -0.131555  ...   \n",
+       "1069835_german_bundesliga       0.372292   0.631282 -0.405111 -0.514660  ...   \n",
+       "1069836_german_bundesliga       0.372292   0.959403 -1.008551 -0.857581  ...   \n",
+       "1069834_german_bundesliga       0.058238  -0.025558 -0.717569 -1.320451  ...   \n",
+       "1069842_german_bundesliga       0.721268   0.389111 -0.717569 -0.514660  ...   \n",
+       "...                                  ...        ...       ...       ...  ...   \n",
+       "2219403_spanish_la_liga        -0.245043   0.959403 -0.238578  0.080462  ...   \n",
+       "2229046_italian_serie_a        -0.119351   0.744759 -0.405111  0.306435  ...   \n",
+       "2210408_english_premier_league -1.194145   0.389111  0.315282 -0.131555  ...   \n",
+       "2229048_italian_serie_a        -0.245043   1.160365 -0.405111 -0.514660  ...   \n",
+       "2210403_english_premier_league -0.728607   2.103550  0.121476 -0.329155  ...   \n",
+       "\n",
+       "                                total_expulsions_ref  total_tackle_ref  \\\n",
+       "match_id                                                                 \n",
+       "1069833_german_bundesliga                   0.985318          1.101278   \n",
+       "1069835_german_bundesliga                  -0.369170          0.356790   \n",
+       "1069836_german_bundesliga                   0.542920          0.774597   \n",
+       "1069834_german_bundesliga                   1.406218          0.391996   \n",
+       "1069842_german_bundesliga                  -0.296511          0.965913   \n",
+       "...                                              ...               ...   \n",
+       "2219403_spanish_la_liga                     0.372171          0.721725   \n",
+       "2229046_italian_serie_a                     1.682687         -2.489470   \n",
+       "2210408_english_premier_league              0.372171          1.580043   \n",
+       "2229048_italian_serie_a                    -1.639614          1.974827   \n",
+       "2210403_english_premier_league             -0.022397          0.586037   \n",
+       "\n",
+       "                                total_fk_foul_ref  total_penalty_match_ref  \\\n",
+       "match_id                                                                     \n",
+       "1069833_german_bundesliga               -1.029729                -0.793801   \n",
+       "1069835_german_bundesliga               -0.346304                -1.454542   \n",
+       "1069836_german_bundesliga                1.110759                -1.454542   \n",
+       "1069834_german_bundesliga                1.565295                 2.265159   \n",
+       "1069842_german_bundesliga               -0.156935                -0.105425   \n",
+       "...                                           ...                      ...   \n",
+       "2219403_spanish_la_liga                  0.121370                 1.021130   \n",
+       "2229046_italian_serie_a                  0.417330                 0.763942   \n",
+       "2210408_english_premier_league          -0.084230                -0.105425   \n",
+       "2229048_italian_serie_a                  0.809434                -2.368850   \n",
+       "2210403_english_premier_league          -1.395555                 1.504698   \n",
+       "\n",
+       "                                total_hand_ball_ref  total_fouls_home_ref  \\\n",
+       "match_id                                                                    \n",
+       "1069833_german_bundesliga                  0.917055             -0.732395   \n",
+       "1069835_german_bundesliga                  0.156403              0.124371   \n",
+       "1069836_german_bundesliga                 -0.907535              1.374438   \n",
+       "1069834_german_bundesliga                  1.301284              1.168461   \n",
+       "1069842_german_bundesliga                 -0.590373              0.562788   \n",
+       "...                                             ...                   ...   \n",
+       "2219403_spanish_la_liga                   -0.051437             -0.058292   \n",
+       "2229046_italian_serie_a                    0.568077              0.583474   \n",
+       "2210408_english_premier_league             0.685931              0.208313   \n",
+       "2229048_italian_serie_a                   -0.590373              0.275842   \n",
+       "2210403_english_premier_league            -1.906150             -1.713468   \n",
+       "\n",
+       "                                total_fouls_away_ref  total_fouls_match_ref  \\\n",
+       "match_id                                                                      \n",
+       "1069833_german_bundesliga                  -0.728244              -0.827393   \n",
+       "1069835_german_bundesliga                  -0.627903              -0.305101   \n",
+       "1069836_german_bundesliga                   0.197598               0.864320   \n",
+       "1069834_german_bundesliga                   1.635305               1.640432   \n",
+       "1069842_german_bundesliga                  -0.929066              -0.234115   \n",
+       "...                                              ...                    ...   \n",
+       "2219403_spanish_la_liga                     0.272404               0.093577   \n",
+       "2229046_italian_serie_a                     0.272404               0.460710   \n",
+       "2210408_english_premier_league             -0.127036               0.017975   \n",
+       "2229048_italian_serie_a                     0.869680               0.638675   \n",
+       "2210403_english_premier_league             -1.029543              -1.521511   \n",
+       "\n",
+       "                                fk_per_tackle_ref  goals_2.5  \n",
+       "match_id                                                      \n",
+       "1069833_german_bundesliga               -1.218083          1  \n",
+       "1069835_german_bundesliga               -0.380686          0  \n",
+       "1069836_german_bundesliga                0.337828          0  \n",
+       "1069834_german_bundesliga                0.764458          1  \n",
+       "1069842_german_bundesliga               -0.369954          0  \n",
+       "...                                           ...        ...  \n",
+       "2219403_spanish_la_liga                 -0.371811          0  \n",
+       "2229046_italian_serie_a                  1.857669          1  \n",
+       "2210408_english_premier_league          -1.040655          0  \n",
+       "2229048_italian_serie_a                 -0.802420          0  \n",
+       "2210403_english_premier_league          -1.583014          0  \n",
+       "\n",
+       "[2907 rows x 2200 columns]"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "feat_sel.x_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "feat_sel.dataset.to_csv('Vicente_Dataset.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "feat_sel.x_df.to_csv('Vicente_x_df.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4c7541287bfa4753b41e5968084b85b4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluate_model(feat_sel.top_models[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "error = ['win_pct_home_over_corners_11.5_h', 'ranking_a', 'odds_home_over_corners_6.5_diff', 'total_bets_under_corners_6.5_h', 'win_pct_away_over_cards_6.5_diff', 'ratio_under_cards_2.5_a', 'total_clearance_mean_a', 'red_card_1t_mean_h', 'total_bets_over_cards_5.5_a', 'head_clearance_std_diff', 'att_obx_left_mean_a', 'MED_mean_h', 'pts_dropped_winning_pos_mean_h', 'win_pct_home_under_corners_11.5_diff', 'fifty_fifty_std_h', 'total_bets_under_cards_4.5_a', 'red_card_2t_std_h', 'total_bets_over_cards_4.5_h', 'ranking_mean_h', 'penalty_save_std_h', 'red_card_std_h', 'att_cmiss_high_left_mean_h', 'att_cmiss_high_left_mean_a', 'att_post_right_mean_a', 'total_bets_over_corners_9.5_h', 'win_pct_home_under_corners_11.5_a', 'head_clearance_mean_h', 'effective_head_clearance_mean_a', 'odds_home_under_goals_0.5_a', 'ATT_h_mul_shots_mul_goals_mean_a', 'total_bets_under_cards_4.5_diff', 'red_card_2t_std_a', 'accurate_keeper_throws_mean_a', 'ratio_over_cards_2.5_a', 'ranking_diff', 'att_lg_centre_std_h', 'corners_std_diff', 'effective_clearance_mean_h', 'att_obx_left_std_h', 'att_freekick_goal_std_h', 'ratio_over_cards_2.5_h', 'odds_away_over_corners_9.5_a', 'att_goal_high_centre_std_h', 'total_red_card_std_a', 'clearance_off_line_mean_h', 'head_clearance_mean_diff', 'odds_away_over_both_score_a', 'own_goals_mean_h', 'ratio_over_cards_6.5_h', 'att_obxd_right_std_h', 'att_lg_centre_mean_a', 'pts_dropped_winning_pos_mean_diff', 'att_freekick_goal_std_a', 'fifty_fifty_std_a', 'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_a', 'ATT_h_mul_shots_mul_goals_mean_h', 'total_bets_over_corners_9.5_diff', 'shots_mul_goals_mean_diff', 'DEF_mean_diff', 'win_pct_home_over_cards_6.5_diff', 'total_bets_over_cards_2.5_h', 'total_bets_under_cards_2.5_a', 'own_goals_std_a', 'ratio_under_corners_6.5_a', 'last_man_tackle_std_h', 'att_goal_high_centre_std_a', 'att_ibox_own_goal_mean_a', 'ratio_under_cards_2.5_h', 'att_ibox_own_goal_std_h', 'penalty_save_mean_a', 'att_post_right_std_h', 'att_hd_post_std_a', 'att_obxd_right_mean_a', 'ratio_under_corners_6.5_h', 'total_bets_under_goals_2.5_diff', 'ratio_over_cards_6.5_a', 'last_man_tackle_mean_a', 'att_obx_right_mean_h', 'total_bets_under_cards_2.5_diff', 'att_pen_target_mean_a', 'total_bets_under_corners_6.5_a', 'total_clearance_mean_diff', 'att_cmiss_high_right_std_a', 'total_bets_under_corners_9.5_a', 'goal_fastbreak_mean_a', 'odds_home_under_goals_0.5_h', 'att_obx_right_mean_a', 'att_cmiss_high_right_mean_h', 'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_h'] "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['att_lg_centre_std_h']"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "[c for c in feat_sel.dataset.columns if 'att_lg_centre_std_h' in c]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array(['ATT_h_mul_shots_mul_goals_mean_a',\n",
+       "       'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_a',\n",
+       "       'ATT_h_mul_shots_mul_goals_mean_h',\n",
+       "       'total_fwd_zone_pass_mean_diff_mul_goal_kicks_per_shot_mean_h'],\n",
+       "      dtype='<U60')"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.array(error)[~np.array([c in feat_sel.dataset.columns for c in error])]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True, False,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
+       "       False, False,  True,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
+       "        True,  True,  True,  True,  True,  True,  True, False])"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.array([c in feat_sel.dataset.columns for c in error])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(145,\n",
+       " 276,\n",
+       " ['ATT_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'ATT_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'DEF_h_mul_shots_mul_goals_mean_diff',\n",
+       "  'accurate_cross_nocorner_std_a',\n",
+       "  'accurate_cross_nocorner_std_diff',\n",
+       "  'accurate_cross_nocorner_std_h',\n",
+       "  'accurate_cross_std_a',\n",
+       "  'accurate_cross_std_diff',\n",
+       "  'accurate_cross_std_h',\n",
+       "  'accurate_goal_kicks_std_a',\n",
+       "  'accurate_goal_kicks_std_diff',\n",
+       "  'accurate_goal_kicks_std_h',\n",
+       "  'accurate_keeper_sweeper_mean_a',\n",
+       "  'accurate_keeper_sweeper_mean_diff',\n",
+       "  'accurate_keeper_sweeper_mean_h',\n",
+       "  'accurate_keeper_sweeper_std_a',\n",
+       "  'accurate_keeper_sweeper_std_diff',\n",
+       "  'accurate_keeper_sweeper_std_h',\n",
+       "  'accurate_launches_mean_a',\n",
+       "  'accurate_launches_mean_diff',\n",
+       "  'accurate_launches_mean_h',\n",
+       "  'accurate_layoffs_mean_a',\n",
+       "  'accurate_layoffs_mean_diff',\n",
+       "  'accurate_layoffs_mean_h',\n",
+       "  'accurate_layoffs_std_a',\n",
+       "  'accurate_layoffs_std_diff',\n",
+       "  'accurate_layoffs_std_h',\n",
+       "  'accurate_through_ball_mean_a',\n",
+       "  'accurate_through_ball_mean_diff',\n",
+       "  'accurate_through_ball_mean_h',\n",
+       "  'accurate_throws_std_a',\n",
+       "  'accurate_throws_std_diff',\n",
+       "  'accurate_throws_std_h',\n",
+       "  'att_freekick_miss_mean_a',\n",
+       "  'att_freekick_miss_mean_diff',\n",
+       "  'att_freekick_miss_mean_h',\n",
+       "  'att_freekick_miss_std_a',\n",
+       "  'att_freekick_miss_std_diff',\n",
+       "  'att_freekick_miss_std_h',\n",
+       "  'att_goal_low_centre_mean_a',\n",
+       "  'att_goal_low_centre_mean_diff',\n",
+       "  'att_goal_low_centre_mean_h',\n",
+       "  'att_goal_low_centre_std_a',\n",
+       "  'att_goal_low_centre_std_diff',\n",
+       "  'att_goal_low_centre_std_h',\n",
+       "  'att_goal_low_right_mean_a',\n",
+       "  'att_goal_low_right_mean_diff',\n",
+       "  'att_goal_low_right_mean_h',\n",
+       "  'att_hd_goal_mean_a',\n",
+       "  'att_hd_goal_mean_diff',\n",
+       "  'att_hd_goal_mean_h',\n",
+       "  'att_hd_target_mean_a',\n",
+       "  'att_hd_target_mean_diff',\n",
+       "  'att_hd_target_mean_h',\n",
+       "  'att_lf_goal_mean_a',\n",
+       "  'att_lf_goal_mean_diff',\n",
+       "  'att_lf_goal_mean_h',\n",
+       "  'att_lf_goal_std_a',\n",
+       "  'att_lf_goal_std_diff',\n",
+       "  'att_lf_goal_std_h',\n",
+       "  'att_miss_high_left_std_a',\n",
+       "  'att_miss_high_left_std_diff',\n",
+       "  'att_miss_high_left_std_h',\n",
+       "  'att_obox_blocked_mean_a',\n",
+       "  'att_obox_blocked_mean_diff',\n",
+       "  'att_obox_blocked_mean_h',\n",
+       "  'att_pen_goal_mean_a',\n",
+       "  'att_pen_goal_mean_diff',\n",
+       "  'att_pen_goal_mean_h',\n",
+       "  'attempts_conceded_ibox_std_a',\n",
+       "  'attempts_conceded_ibox_std_diff',\n",
+       "  'attempts_conceded_ibox_std_h',\n",
+       "  'attempts_conceded_obox_std_a',\n",
+       "  'attempts_conceded_obox_std_diff',\n",
+       "  'attempts_conceded_obox_std_h',\n",
+       "  'attempts_obox_mean_a',\n",
+       "  'attempts_obox_mean_diff',\n",
+       "  'attempts_obox_mean_h',\n",
+       "  'big_chance_created_mean_a',\n",
+       "  'big_chance_created_mean_diff',\n",
+       "  'big_chance_created_mean_h',\n",
+       "  'big_chance_created_std_a',\n",
+       "  'big_chance_created_std_diff',\n",
+       "  'big_chance_created_std_h',\n",
+       "  'big_chance_missed_std_a',\n",
+       "  'big_chance_missed_std_diff',\n",
+       "  'big_chance_missed_std_h',\n",
+       "  'blocked_cross_mean_a',\n",
+       "  'blocked_cross_mean_diff',\n",
+       "  'blocked_cross_mean_h',\n",
+       "  'contentious_decision_mean_a',\n",
+       "  'contentious_decision_mean_diff',\n",
+       "  'contentious_decision_mean_h',\n",
+       "  'contentious_decision_std_a',\n",
+       "  'contentious_decision_std_diff',\n",
+       "  'contentious_decision_std_h',\n",
+       "  'defender_goals_mean_a',\n",
+       "  'defender_goals_mean_a',\n",
+       "  'defender_goals_mean_diff',\n",
+       "  'defender_goals_mean_diff',\n",
+       "  'defender_goals_mean_h',\n",
+       "  'defender_goals_mean_h',\n",
+       "  'diving_save_std_a',\n",
+       "  'diving_save_std_diff',\n",
+       "  'diving_save_std_h',\n",
+       "  'effective_head_clearance_mean_a',\n",
+       "  'effective_head_clearance_mean_diff',\n",
+       "  'effective_head_clearance_mean_h',\n",
+       "  'forward_goals_mean_a',\n",
+       "  'forward_goals_mean_diff',\n",
+       "  'forward_goals_mean_h',\n",
+       "  'goal_assist_deadball_std_a',\n",
+       "  'goal_assist_deadball_std_diff',\n",
+       "  'goal_assist_deadball_std_h',\n",
+       "  'goal_assist_intent_norm_mean_a',\n",
+       "  'goal_assist_intent_norm_mean_diff',\n",
+       "  'goal_assist_intent_norm_mean_h',\n",
+       "  'goal_assist_intentional_mean_a',\n",
+       "  'goal_assist_intentional_mean_diff',\n",
+       "  'goal_assist_intentional_mean_h',\n",
+       "  'goal_assist_intentional_std_a',\n",
+       "  'goal_assist_intentional_std_diff',\n",
+       "  'goal_assist_intentional_std_h',\n",
+       "  'goal_assist_mean_a',\n",
+       "  'goal_assist_mean_diff',\n",
+       "  'goal_assist_mean_h',\n",
+       "  'goal_assist_openplay_std_a',\n",
+       "  'goal_assist_openplay_std_diff',\n",
+       "  'goal_assist_openplay_std_h',\n",
+       "  'goal_assist_setplay_mean_a',\n",
+       "  'goal_assist_setplay_mean_diff',\n",
+       "  'goal_assist_setplay_mean_h',\n",
+       "  'goals_2t_mean_a',\n",
+       "  'goals_2t_mean_diff',\n",
+       "  'goals_2t_mean_h',\n",
+       "  'goals_mean_a',\n",
+       "  'goals_mean_diff',\n",
+       "  'goals_mean_h',\n",
+       "  'goals_openplay_mean_a',\n",
+       "  'goals_openplay_mean_diff',\n",
+       "  'goals_openplay_mean_h',\n",
+       "  'high_to_low_goals_mean_a',\n",
+       "  'high_to_low_goals_mean_diff',\n",
+       "  'high_to_low_goals_mean_h',\n",
+       "  'interceptions_in_box_std_a',\n",
+       "  'interceptions_in_box_std_diff',\n",
+       "  'interceptions_in_box_std_h',\n",
+       "  'left_div_right_foot_goals_mean_a',\n",
+       "  'left_div_right_foot_goals_mean_a',\n",
+       "  'left_div_right_foot_goals_mean_diff',\n",
+       "  'left_div_right_foot_goals_mean_diff',\n",
+       "  'left_div_right_foot_goals_mean_h',\n",
+       "  'left_div_right_foot_goals_mean_h',\n",
+       "  'lost_corners_mean_a',\n",
+       "  'lost_corners_mean_diff',\n",
+       "  'lost_corners_mean_h',\n",
+       "  'midfielder_goals_mean_a',\n",
+       "  'midfielder_goals_mean_diff',\n",
+       "  'midfielder_goals_mean_h',\n",
+       "  'no_foot_goals_ratio_mean_a',\n",
+       "  'no_foot_goals_ratio_mean_diff',\n",
+       "  'no_foot_goals_ratio_mean_h',\n",
+       "  'no_foot_goals_ratio_std_a',\n",
+       "  'no_foot_goals_ratio_std_diff',\n",
+       "  'no_foot_goals_ratio_std_h',\n",
+       "  'odd_ratio_over_corners_11.5',\n",
+       "  'odd_ratio_under_corners_10.5',\n",
+       "  'odds_away_over_cards_5.5_a',\n",
+       "  'odds_away_over_cards_5.5_diff',\n",
+       "  'odds_away_over_cards_5.5_h',\n",
+       "  'odds_home_under_cards_5.5_a',\n",
+       "  'odds_home_under_cards_5.5_diff',\n",
+       "  'odds_home_under_cards_5.5_h',\n",
+       "  'odds_home_under_goals_0.5_a',\n",
+       "  'odds_home_under_goals_0.5_diff',\n",
+       "  'odds_home_under_goals_0.5_h',\n",
+       "  'ontarget_att_assist_mean_a',\n",
+       "  'ontarget_att_assist_mean_diff',\n",
+       "  'ontarget_att_assist_mean_h',\n",
+       "  'own_goals_mean_a',\n",
+       "  'own_goals_mean_diff',\n",
+       "  'own_goals_mean_h',\n",
+       "  'penalty_conceded_mean_a',\n",
+       "  'penalty_conceded_mean_diff',\n",
+       "  'penalty_conceded_mean_h',\n",
+       "  'penalty_faced_std_a',\n",
+       "  'penalty_faced_std_diff',\n",
+       "  'penalty_faced_std_h',\n",
+       "  'penalty_save_std_a',\n",
+       "  'penalty_save_std_diff',\n",
+       "  'penalty_save_std_h',\n",
+       "  'poss_won_att_3rd_std_a',\n",
+       "  'poss_won_att_3rd_std_diff',\n",
+       "  'poss_won_att_3rd_std_h',\n",
+       "  'ratio_over_corners_11.5_a',\n",
+       "  'ratio_over_corners_11.5_diff',\n",
+       "  'ratio_over_corners_11.5_h',\n",
+       "  'ratio_under_corners_10.5_a',\n",
+       "  'ratio_under_corners_10.5_diff',\n",
+       "  'ratio_under_corners_10.5_h',\n",
+       "  'red_card_std_a',\n",
+       "  'red_card_std_diff',\n",
+       "  'red_card_std_h',\n",
+       "  'rescinded_red_card_std_a',\n",
+       "  'rescinded_red_card_std_diff',\n",
+       "  'rescinded_red_card_std_h',\n",
+       "  'right_to_left_goals_mean_a',\n",
+       "  'right_to_left_goals_mean_diff',\n",
+       "  'right_to_left_goals_mean_h',\n",
+       "  'saves_std_a',\n",
+       "  'saves_std_diff',\n",
+       "  'saves_std_h',\n",
+       "  'shots_mul_goals_mean_a',\n",
+       "  'shots_mul_goals_mean_a',\n",
+       "  'shots_mul_goals_mean_diff',\n",
+       "  'shots_mul_goals_mean_diff',\n",
+       "  'shots_mul_goals_mean_h',\n",
+       "  'shots_mul_goals_mean_h',\n",
+       "  'shots_std_a',\n",
+       "  'shots_std_diff',\n",
+       "  'shots_std_h',\n",
+       "  'total_fastbreak_std_a',\n",
+       "  'total_fastbreak_std_diff',\n",
+       "  'total_fastbreak_std_h',\n",
+       "  'total_keeper_sweeper_mean_a',\n",
+       "  'total_keeper_sweeper_mean_diff',\n",
+       "  'total_keeper_sweeper_mean_h',\n",
+       "  'total_keeper_sweeper_std_a',\n",
+       "  'total_keeper_sweeper_std_diff',\n",
+       "  'total_keeper_sweeper_std_h',\n",
+       "  'total_launches_mean_a',\n",
+       "  'total_launches_mean_diff',\n",
+       "  'total_launches_mean_h',\n",
+       "  'total_launches_std_a',\n",
+       "  'total_launches_std_diff',\n",
+       "  'total_launches_std_h',\n",
+       "  'total_layoffs_std_a',\n",
+       "  'total_layoffs_std_diff',\n",
+       "  'total_layoffs_std_h',\n",
+       "  'total_red_card_std_a',\n",
+       "  'total_red_card_std_diff',\n",
+       "  'total_red_card_std_h',\n",
+       "  'total_through_ball_mean_a',\n",
+       "  'total_through_ball_mean_diff',\n",
+       "  'total_through_ball_mean_h',\n",
+       "  'total_through_ball_std_a',\n",
+       "  'total_through_ball_std_diff',\n",
+       "  'total_through_ball_std_h',\n",
+       "  'total_throws_std_a',\n",
+       "  'total_throws_std_diff',\n",
+       "  'total_throws_std_h',\n",
+       "  'total_win_pct_over_goals_2.5_a',\n",
+       "  'total_win_pct_over_goals_2.5_diff',\n",
+       "  'total_win_pct_over_goals_2.5_h',\n",
+       "  'total_win_pct_under_goals_1.5_a',\n",
+       "  'total_win_pct_under_goals_1.5_diff',\n",
+       "  'total_win_pct_under_goals_1.5_h',\n",
+       "  'total_win_pct_under_goals_3.5_a',\n",
+       "  'total_win_pct_under_goals_3.5_diff',\n",
+       "  'total_win_pct_under_goals_3.5_h',\n",
+       "  'win_pct_away_over_goals_2.5_a',\n",
+       "  'win_pct_away_over_goals_2.5_diff',\n",
+       "  'win_pct_away_over_goals_2.5_h',\n",
+       "  'win_pct_home_over_cards_6.5_a',\n",
+       "  'win_pct_home_over_cards_6.5_diff',\n",
+       "  'win_pct_home_over_cards_6.5_h',\n",
+       "  'win_pct_home_over_goals_2.5_a',\n",
+       "  'win_pct_home_over_goals_2.5_diff',\n",
+       "  'win_pct_home_over_goals_2.5_h',\n",
+       "  'win_pct_home_under_cards_4.5_a',\n",
+       "  'win_pct_home_under_cards_4.5_diff',\n",
+       "  'win_pct_home_under_cards_4.5_h',\n",
+       "  'won_corners_ratio_std_a',\n",
+       "  'won_corners_ratio_std_diff',\n",
+       "  'won_corners_ratio_std_h'])"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "common_feat = list(set([\"_\".join(x.split(\"_\")[:-1]) for x in selected_features]))\n",
+    "new_subset = [x for x in train_data.columns for c in common_feat if c in x]\n",
+    "\n",
+    "len(selected_features), len(new_subset), sorted(new_subset)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### new imports "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "import os\n",
+    "\n",
+    "from ml_bets.constants import FEATURES_PATH\n",
+    "from ml_bets.features.features import Features\n",
+    "from ml_bets.modeling.match_model import PipelineDatasets, run_pycaret_setup\n",
+    "#from featsel.feature_selection import FeatureSelection\n",
+    "\n",
+    "from pycaret.utils import check_metric\n",
+    "from pycaret.classification import (add_metric, calibrate_model, optimize_threshold,\n",
+    "    create_model,\n",
+    "    finalize_model,\n",
+    "    optimize_threshold,              \n",
+    "    save_model,\n",
+    "    compare_models, \n",
+    "    evaluate_model,\n",
+    "    get_config,\n",
+    "    setup,\n",
+    "    tune_model,\n",
+    "    predict_model,\n",
+    ")\n",
+    "\n",
+    "from ml_bets.supplementary.functions import IGNORE_FEATURES\n",
+    "from ml_bets.research.datasets import create_dataset\n",
+    "from ml_bets.features.names.goals import GOALS_FEATURES, NEW_FEATURES"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ml_bets.research.tips import calibrate_tips, tips_from_model, combine_tips, predict_dataset, compose_tips, get_tip_probs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ix = np.logical_and(feats.matches[\"date\"].dt.month > 10,\n",
+    "                    feats.matches[\"competition\"].isin({\"mexican_primera\", 'us_major_league_soccer'}))\n",
+    "index = feats.matches[~ix].index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_feats = ['acc_cross_nocorner_pct_std_a',\n",
+    " 'acc_cross_nocorner_pct_std_diff',\n",
+    " 'acc_cross_nocorner_pct_std_h',\n",
+    " 'accurate_flick_on_mean_h',\n",
+    " 'accurate_flick_on_mean_a',\n",
+    " 'accurate_freekick_cross_mean_a',\n",
+    " 'accurate_freekick_cross_mean_diff',\n",
+    " 'accurate_freekick_cross_mean_h',\n",
+    " 'accurate_goal_kicks_mean_diff',\n",
+    " 'accurate_goal_kicks_std_a',\n",
+    " 'accurate_goal_kicks_std_diff',\n",
+    " 'accurate_goal_kicks_std_h',\n",
+    " 'accurate_keeper_sweeper_mean_a',\n",
+    " 'accurate_keeper_sweeper_std_diff',\n",
+    " 'accurate_keeper_sweeper_std_h',\n",
+    " 'accurate_pass_std_a',\n",
+    " 'accurate_pass_std_h',\n",
+    " 'accurate_through_ball_mean_diff',\n",
+    " 'att_assist_openplay_std_a',\n",
+    " 'att_assist_openplay_std_h',\n",
+    " 'att_cmiss_left_mean_a',\n",
+    " 'att_cmiss_left_mean_h',\n",
+    " 'att_cmiss_left_std_a',\n",
+    " 'att_cmiss_left_std_diff',\n",
+    " 'att_cmiss_left_std_h',\n",
+    " 'att_goal_high_centre_mean_diff',\n",
+    " 'att_goal_high_centre_std_diff',\n",
+    " 'att_goal_high_right_std_a',\n",
+    " 'att_goal_low_centre_mean_diff',\n",
+    " 'att_goal_low_centre_mean_h',\n",
+    " 'att_goal_low_centre_std_a',\n",
+    " 'att_goal_low_centre_std_h',\n",
+    " 'att_hd_goal_mean_a',\n",
+    " 'att_hd_goal_mean_h',\n",
+    " 'att_hd_target_mean_a',\n",
+    " 'att_hd_target_mean_h',\n",
+    " 'att_ibox_own_goal_mean_diff',\n",
+    " 'att_ibox_own_goal_std_h',\n",
+    " 'att_ibox_own_goal_std_a',\n",
+    " 'att_miss_high_mean_a',\n",
+    " 'att_miss_high_mean_h',\n",
+    " 'att_miss_high_right_mean_diff',\n",
+    " 'att_obxd_right_mean_diff',\n",
+    " 'att_obxd_right_std_diff',\n",
+    " 'att_one_on_one_mean_h',\n",
+    " 'att_one_on_one_mean_a',\n",
+    " 'att_one_on_one_std_diff',\n",
+    " 'att_one_on_one_std_h',\n",
+    " 'att_post_high_std_a',\n",
+    " 'att_post_high_std_h',\n",
+    " 'att_post_right_mean_a',\n",
+    " 'att_post_right_mean_h',\n",
+    " 'attempts_ibox_std_a',\n",
+    " 'attempts_ibox_std_h',\n",
+    " 'backward_pass_mean_diff',\n",
+    " 'backward_pass_mean_h',\n",
+    " 'big_chance_created_mean_h',\n",
+    " 'big_chance_created_std_diff',\n",
+    " 'clean_sheet_std_diff',\n",
+    " 'contentious_decision_mean_a',\n",
+    " 'contentious_decision_mean_diff',\n",
+    " 'contentious_decision_std_a',\n",
+    " 'contentious_decision_std_h',\n",
+    " 'duel_won_pct_mean_diff',\n",
+    " 'effective_clearance_mean_diff',\n",
+    " 'error_lead_to_goal_mean_diff',\n",
+    " 'error_lead_to_goal_std_diff',\n",
+    " 'first_yellow_card_1t_mean_h',\n",
+    " 'first_yellow_card_1t_mean_a',\n",
+    " 'foul_throw_in_std_diff',\n",
+    " 'fouled_final_third_mean_diff',\n",
+    " 'goal_assist_deadball_mean_diff',\n",
+    " 'goal_assist_openplay_std_diff',\n",
+    " 'goal_assist_openplay_std_h',\n",
+    " 'goal_assist_setplay_std_a',\n",
+    " 'goal_assist_setplay_std_diff',\n",
+    " 'goal_assist_setplay_std_h',\n",
+    " 'goal_assist_std_diff',\n",
+    " 'goals_mean_diff',\n",
+    " 'goals_openplay_std_h',\n",
+    " 'goals_openplay_std_a',\n",
+    " 'good_high_claim_mean_diff',\n",
+    " 'high_to_low_goals_mean_a',\n",
+    " 'high_to_low_goals_mean_diff',\n",
+    " 'high_to_low_goals_std_a',\n",
+    " 'high_to_low_goals_std_h',\n",
+    " 'imp_prob_under_goals_0.5_h',\n",
+    " 'imp_prob_under_goals_2.5_diff',\n",
+    " 'interception_mean_a',\n",
+    " 'interception_mean_diff',\n",
+    " 'interception_mean_h',\n",
+    " 'interceptions_in_box_std_a',\n",
+    " 'interceptions_in_box_std_diff',\n",
+    " 'interceptions_in_box_std_h',\n",
+    " 'last_man_tackle_mean_a',\n",
+    " 'last_man_tackle_mean_diff',\n",
+    " 'left_div_right_foot_goals_std_diff',\n",
+    " 'leftside_pass_mean_a',\n",
+    " 'leftside_pass_mean_h',\n",
+    " 'leftside_pass_std_a',\n",
+    " 'long_pass_own_to_opp_mean_diff',\n",
+    " 'long_pass_own_to_opp_mean_h',\n",
+    " 'no_foot_goals_ratio_std_a',\n",
+    " 'no_foot_goals_ratio_std_diff',\n",
+    " 'no_foot_goals_ratio_std_h',\n",
+    " 'odd_ratio_under_corners_10.5',\n",
+    " 'odd_ratio_under_corners_8.5',\n",
+    " 'odds_home_under_both_score_h',\n",
+    " 'odds_away_under_both_score_a',\n",
+    " 'odds_away_under_goals_4.5_a',\n",
+    " 'odds_home_under_goals_4.5_h',\n",
+    " 'odds_away_over_goals_4.5_a',\n",
+    " 'odds_home_over_goals_4.5_h',\n",
+    " 'odds_home_under_goals_0.5_diff',\n",
+    " 'odds_home_under_goals_0.5_h',\n",
+    " 'own_goals_std_h',\n",
+    " 'own_goals_std_a',\n",
+    " 'pen_goals_conceded_mean_diff',\n",
+    " 'penalty_faced_std_a',\n",
+    " 'penalty_won_std_a',\n",
+    " 'penalty_faced_std_h',\n",
+    " 'penalty_won_std_h',\n",
+    " 'poss_won_att_3rd_std_a',\n",
+    " 'poss_won_att_3rd_std_diff',\n",
+    " 'poss_won_att_3rd_std_h',\n",
+    " 'post_scoring_att_std_a',\n",
+    " 'post_scoring_att_std_h',\n",
+    " 'prob_squared_under_goals_2.5',\n",
+    " 'pts_dropped_winning_pos_mean_a',\n",
+    " 'pts_dropped_winning_pos_std_diff',\n",
+    " 'pts_dropped_winning_pos_std_h',\n",
+    " 'ratio_over_goals_2.5_a',\n",
+    " 'ratio_over_goals_2.5_h',\n",
+    " 'ratio_under_goals_2.5_a',\n",
+    " 'ratio_under_goals_2.5_diff',\n",
+    " 'raw_prob_over_goals_1.5',\n",
+    " 'raw_prob_over_goals_2.5',\n",
+    " 'raw_prob_over_goals_3.5',\n",
+    " 'raw_prob_under_goals_1.5',\n",
+    " 'raw_prob_under_goals_2.5',\n",
+    " 'raw_prob_under_goals_3.5',\n",
+    " 'raw_prob_under_goals_4.5',\n",
+    " 'red_card_1t_mean_a',\n",
+    " 'red_card_1t_mean_diff',\n",
+    " 'red_card_1t_mean_h',\n",
+    " 'red_card_2t_mean_a',\n",
+    " 'red_card_2t_mean_diff',\n",
+    " 'red_card_mean_h',\n",
+    " 'red_card_mean_a',\n",
+    " 'red_card_std_diff',\n",
+    " 'right_to_left_goals_mean_diff',\n",
+    " 'right_to_left_goals_std_diff',\n",
+    " 'rightside_pass_div_leftside_pass_mean_a',\n",
+    " 'rightside_pass_div_leftside_pass_mean_h',\n",
+    " 'second_yellow_mean_h',\n",
+    " 'shots_mul_goals_std_h',\n",
+    " 'second_yellow_mean_a',\n",
+    " 'shots_mul_goals_std_a',\n",
+    " 'successful_final_third_passes_mean_a',\n",
+    " 'successful_final_third_passes_mean_diff',\n",
+    " 'successful_final_third_passes_std_a',\n",
+    " 'successful_final_third_passes_std_diff',\n",
+    " 'successful_put_through_std_a',\n",
+    " 'successful_put_through_std_diff',\n",
+    " 'successful_put_through_std_h',\n",
+    " 'total_clearance_mean_a',\n",
+    " 'total_clearance_mean_h',\n",
+    " 'total_fastbreak_std_diff',\n",
+    " 'total_high_claim_mean_a',\n",
+    " 'total_high_claim_mean_diff',\n",
+    " 'total_high_claim_mean_h',\n",
+    " 'total_keeper_sweeper_mean_a',\n",
+    " 'total_keeper_sweeper_mean_h',\n",
+    " 'total_launches_std_h',\n",
+    " 'total_launches_std_a',\n",
+    " 'total_red_card_mean_h',\n",
+    " 'total_red_card_mean_a',\n",
+    " 'total_red_card_std_diff',\n",
+    " 'total_throws_std_a',\n",
+    " 'total_throws_std_diff',\n",
+    " 'total_win_pct_over_goals_1.5_a',\n",
+    " 'total_win_pct_over_goals_1.5_h',\n",
+    " 'total_win_pct_over_goals_2.5_a',\n",
+    " 'total_win_pct_over_goals_2.5_h',\n",
+    " 'total_win_pct_over_goals_3.5_a',\n",
+    " 'total_win_pct_over_goals_3.5_h',\n",
+    " 'total_win_pct_under_goals_1.5_a',\n",
+    " 'total_win_pct_under_goals_1.5_h',\n",
+    " 'total_win_pct_under_goals_2.5_a',\n",
+    " 'total_win_pct_under_goals_2.5_h',\n",
+    " 'total_win_pct_under_goals_3.5_a',\n",
+    " 'total_yel_card_std_a',\n",
+    " 'total_yel_card_std_diff',\n",
+    " 'total_yel_card_std_h',\n",
+    " 'win_pct_away_over_goals_2.5_a',\n",
+    " 'win_pct_away_over_goals_2.5_diff',\n",
+    " 'win_pct_away_under_goals_2.5_a',\n",
+    " 'win_pct_away_under_goals_2.5_diff',\n",
+    " 'win_pct_home_over_goals_2.5_a',\n",
+    " 'win_pct_home_over_goals_2.5_diff',\n",
+    " 'win_pct_home_over_goals_2.5_h',\n",
+    " 'win_pct_home_under_goals_2.5_diff',\n",
+    " 'win_pct_home_under_goals_2.5_h',\n",
+    " 'winner_mean_diff']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def setup_dataset(test_date):\n",
+    "    ds = create_dataset(target=target,\n",
+    "                        test_date=test_date,\n",
+    "                        features=feats,\n",
+    "                        odds_features=True,\n",
+    "                        include_std=True,\n",
+    "                        test_weeks=6,\n",
+    "                        ignore_features=IGNORE_FEATURES,\n",
+    "                        drop_future_matches=False,\n",
+    "                       )\n",
+    "    train_data = ds.train_data.copy()#[ds.train_data.index.map(lambda x: \"us_major_league_soccer\" not in x and \"mexican\" not in x)]\n",
+    "    train_data.drop(columns=[\"hour_rank\", \"hour_before_16\", \"is_weekend\"], inplace=True)\n",
+    "    train_data = train_data[train_data.index.isin(index)][list(set(test_feats)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    test_data = ds.test_set[ds.test_set.index.isin(index)][list(set(test_feats)) + [target]].copy()#.reset_index(drop=True)\n",
+    "    val_data = ds.val_set[ds.val_set.index.isin(index)].copy()\n",
+    "    setup_kwargs = dict(\n",
+    "        preprocess=True,\n",
+    "        test_data=test_data[train_data.columns.tolist()],#.dropna(),\n",
+    "        #numeric_features=[x for x in train_data.columns.tolist() if x != target],\n",
+    "        #custom_pipeline=loaded,\n",
+    "        #train_size=0.75,\n",
+    "        session_id=123,\n",
+    "        normalize=True,\n",
+    "        normalize_method=\"robust\",\n",
+    "        transformation=True,\n",
+    "        ignore_low_variance=True,\n",
+    "        remove_multicollinearity=False,\n",
+    "        multicollinearity_threshold=0.8,\n",
+    "        n_jobs=-1,\n",
+    "        use_gpu=False,\n",
+    "        profile=False,\n",
+    "        #ignore_features=ignore_features,\n",
+    "        fold_strategy=\"stratifiedkfold\",#\"timeseries\",\n",
+    "        remove_perfect_collinearity=True,\n",
+    "        create_clusters=False,\n",
+    "        fold=4,\n",
+    "        feature_selection=False,\n",
+    "        # you can use this to keep the 95 % most relevant features (fat_sel_threshold)\n",
+    "        feature_selection_threshold=0.5,\n",
+    "        combine_rare_levels=False,\n",
+    "        rare_level_threshold=0.02,\n",
+    "        pca=False,\n",
+    "        pca_method=\"linear\",\n",
+    "        pca_components=30,\n",
+    "        polynomial_features=False,\n",
+    "        polynomial_degree=2,\n",
+    "        polynomial_threshold=0.05,\n",
+    "        trigonometry_features=False,\n",
+    "        remove_outliers=True,\n",
+    "        outliers_threshold=0.01,\n",
+    "        feature_ratio=False,\n",
+    "        feature_interaction=False,\n",
+    "        # Makes everything slow AF. use to find out possibly interesting features\n",
+    "        interaction_threshold=0.05,\n",
+    "        fix_imbalance=True,\n",
+    "        log_experiment=False,\n",
+    "        verbose=False,\n",
+    "        silent=True,\n",
+    "        experiment_name=\"lagstest\",\n",
+    "    )\n",
+    "    _ = setup(data=train_data, target=target, **setup_kwargs)\n",
+    "    return train_data, test_data, val_data, ds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data, test_data, val_data, ds = setup_dataset(\"4-Dec-2021\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
+    "def train_ensemble():\n",
+    "    top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"et\", \"rf\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )\n",
+    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
+    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
+    "    blend = blend_models(cali)\n",
+    "    opti = tune_model(blend, optimize=\"Precision\", choose_better=True, n_iter=50, search_library=\"optuna\")\n",
+    "    return opti, cali, tuned_models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pycaret.classification import stack_models, ensemble_model, blend_models\n",
+    "def train_linear_models():\n",
+    "    top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )\n",
+    "    tuned_models = [tune_model(model, optimize=\"MCC\", choose_better=True, n_iter=50, search_library=\"optuna\") for model in top_models]\n",
+    "    cali = [calibrate_model(tuned, method=\"sigmoid\", calibrate_fold=4) for tuned in tuned_models]\n",
+    "    return tuned_models, cali"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col0,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col1,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col2,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col3,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col4,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col5,#T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col6{\n",
+       "            background:  yellow;\n",
+       "        }</style><table id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Accuracy</th>        <th class=\"col_heading level0 col1\" >AUC</th>        <th class=\"col_heading level0 col2\" >Recall</th>        <th class=\"col_heading level0 col3\" >Prec.</th>        <th class=\"col_heading level0 col4\" >F1</th>        <th class=\"col_heading level0 col5\" >Kappa</th>        <th class=\"col_heading level0 col6\" >MCC</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >0.6414</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.6893</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6716</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.6507</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6610</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.2806</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.2807</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >0.5997</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.6586</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.6321</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.6124</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6221</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.1969</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.1970</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >0.6216</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.6679</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.6123</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.6442</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6278</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.2435</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.2439</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >0.6126</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.6475</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.6015</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.6345</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6175</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.2257</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.2260</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row4\" class=\"row_heading level0 row4\" >Mean</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col0\" class=\"data row4 col0\" >0.6188</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col1\" class=\"data row4 col1\" >0.6658</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col2\" class=\"data row4 col2\" >0.6294</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col3\" class=\"data row4 col3\" >0.6354</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col4\" class=\"data row4 col4\" >0.6321</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col5\" class=\"data row4 col5\" >0.2367</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow4_col6\" class=\"data row4 col6\" >0.2369</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdlevel0_row5\" class=\"row_heading level0 row5\" >SD</th>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col0\" class=\"data row5 col0\" >0.0152</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col1\" class=\"data row5 col1\" >0.0154</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col2\" class=\"data row5 col2\" >0.0267</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col3\" class=\"data row5 col3\" >0.0145</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col4\" class=\"data row5 col4\" >0.0171</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col5\" class=\"data row5 col5\" >0.0303</td>\n",
+       "                        <td id=\"T_dd83b6a8_82b5_11ec_86da_31d766f3ddbdrow5_col6\" class=\"data row5 col6\" >0.0303</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7f44e43353d0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "tuned_linear, cali_linear = train_linear_models()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style  type=\"text/css\" >\n",
+       "    #T_08d08ce2_82b5_11ec_86da_31d766f3ddbd th {\n",
+       "          text-align: left;\n",
+       "    }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col0,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col7{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col2,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col4,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col6,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col7,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col5,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col1,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col3,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col5{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col8,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col8,#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  lightgrey;\n",
+       "        }#T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col8{\n",
+       "            text-align:  left;\n",
+       "            text-align:  left;\n",
+       "            background-color:  yellow;\n",
+       "            background-color:  lightgrey;\n",
+       "        }</style><table id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbd\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>        <th class=\"col_heading level0 col8\" >TT (Sec)</th>    </tr></thead><tbody>\n",
+       "                <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row0\" class=\"row_heading level0 row0\" >lr</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col0\" class=\"data row0 col0\" >Logistic Regression</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col1\" class=\"data row0 col1\" >0.6166</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col2\" class=\"data row0 col2\" >0.6734</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col3\" class=\"data row0 col3\" >0.5836</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col4\" class=\"data row0 col4\" >0.6500</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col5\" class=\"data row0 col5\" >0.6081</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col6\" class=\"data row0 col6\" >0.2351</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col7\" class=\"data row0 col7\" >0.2400</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow0_col8\" class=\"data row0 col8\" >0.3000</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row1\" class=\"row_heading level0 row1\" >lda</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col0\" class=\"data row1 col0\" >Linear Discriminant Analysis</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col1\" class=\"data row1 col1\" >0.6166</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col2\" class=\"data row1 col2\" >0.6705</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col3\" class=\"data row1 col3\" >0.5892</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col4\" class=\"data row1 col4\" >0.6475</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col5\" class=\"data row1 col5\" >0.6115</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col6\" class=\"data row1 col6\" >0.2347</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col7\" class=\"data row1 col7\" >0.2387</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow1_col8\" class=\"data row1 col8\" >0.3025</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row2\" class=\"row_heading level0 row2\" >ridge</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col0\" class=\"data row2 col0\" >Ridge Classifier</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col1\" class=\"data row2 col1\" >0.6166</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col2\" class=\"data row2 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col3\" class=\"data row2 col3\" >0.5892</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col4\" class=\"data row2 col4\" >0.6476</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col5\" class=\"data row2 col5\" >0.6115</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col6\" class=\"data row2 col6\" >0.2347</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col7\" class=\"data row2 col7\" >0.2387</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow2_col8\" class=\"data row2 col8\" >0.2850</td>\n",
+       "            </tr>\n",
+       "            <tr>\n",
+       "                        <th id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdlevel0_row3\" class=\"row_heading level0 row3\" >svm</th>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col0\" class=\"data row3 col0\" >SVM - Linear Kernel</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col1\" class=\"data row3 col1\" >0.5841</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col2\" class=\"data row3 col2\" >0.0000</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col3\" class=\"data row3 col3\" >0.5441</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col4\" class=\"data row3 col4\" >0.6157</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col5\" class=\"data row3 col5\" >0.5734</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col6\" class=\"data row3 col6\" >0.1708</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col7\" class=\"data row3 col7\" >0.1739</td>\n",
+       "                        <td id=\"T_08d08ce2_82b5_11ec_86da_31d766f3ddbdrow3_col8\" class=\"data row3 col8\" >0.3050</td>\n",
+       "            </tr>\n",
+       "    </tbody></table>"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x7fb80cbcb2e0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "top_models = compare_models(\n",
+    "            n_select=8,\n",
+    "            sort='MCC',\n",
+    "            include=[\"lr\", \"lda\", \"ridge\", \"svm\"],\n",
+    "            verbose=True,\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "SGDClassifier(alpha=0.10126692364605047, average=False, class_weight=None,\n",
+       "              early_stopping=False, epsilon=0.1, eta0=0.001022673152073415,\n",
+       "              fit_intercept=False, l1_ratio=0.2203376725525369,\n",
+       "              learning_rate='optimal', loss='hinge', max_iter=1000,\n",
+       "              n_iter_no_change=5, n_jobs=-1, penalty='l1', power_t=0.5,\n",
+       "              random_state=123, shuffle=True, tol=0.001,\n",
+       "              validation_fraction=0.1, verbose=0, warm_start=False)"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tuned_linear[3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6522b430b2a64128ac3789a8d93475f2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluate_model(tuned_linear[3])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a9de03ccce3a4047b615b146297be69e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(ToggleButtons(description='Plot Type:', icons=('',), options=(('Hyperparameters', 'param…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "evaluate_model(cali_linear[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "market=\"goals\"\n",
+    "cutoff = 2.5\n",
+    "tips = tips_from_model(model=cali_linear[0],\n",
+    "                test_data=test_data,\n",
+    "                features=feats,\n",
+    "                market=market,\n",
+    "                cutoff=cutoff,\n",
+    "                      )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>win</th>\n",
+       "      <th>profit</th>\n",
+       "      <th>exp_payoff</th>\n",
+       "      <th>exp_payoff_prec</th>\n",
+       "      <th>odds</th>\n",
+       "      <th>confidence</th>\n",
+       "      <th>count</th>\n",
+       "      <th>buenas</th>\n",
+       "      <th>model</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>bet_type</th>\n",
+       "      <th>match_week</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"7\" valign=\"top\">goals_over_2.5</th>\n",
+       "      <th>2021_19</th>\n",
+       "      <td>0.818182</td>\n",
+       "      <td>0.434545</td>\n",
+       "      <td>1.193888</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.743636</td>\n",
+       "      <td>0.683591</td>\n",
+       "      <td>11</td>\n",
+       "      <td>9</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_20</th>\n",
+       "      <td>0.700000</td>\n",
+       "      <td>0.144000</td>\n",
+       "      <td>1.087417</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.641000</td>\n",
+       "      <td>0.664270</td>\n",
+       "      <td>10</td>\n",
+       "      <td>7</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_21</th>\n",
+       "      <td>0.800000</td>\n",
+       "      <td>0.359000</td>\n",
+       "      <td>1.130386</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.685000</td>\n",
+       "      <td>0.671600</td>\n",
+       "      <td>10</td>\n",
+       "      <td>8</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_22</th>\n",
+       "      <td>0.571429</td>\n",
+       "      <td>0.040000</td>\n",
+       "      <td>1.130597</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.708571</td>\n",
+       "      <td>0.662800</td>\n",
+       "      <td>7</td>\n",
+       "      <td>4</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_23</th>\n",
+       "      <td>0.500000</td>\n",
+       "      <td>-0.140000</td>\n",
+       "      <td>1.216788</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.815000</td>\n",
+       "      <td>0.667750</td>\n",
+       "      <td>4</td>\n",
+       "      <td>2</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_24</th>\n",
+       "      <td>0.625000</td>\n",
+       "      <td>0.058750</td>\n",
+       "      <td>1.168317</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.730000</td>\n",
+       "      <td>0.674975</td>\n",
+       "      <td>8</td>\n",
+       "      <td>5</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_25</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.000000</td>\n",
+       "      <td>1.200240</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.800000</td>\n",
+       "      <td>0.666800</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">goals_under_2.5</th>\n",
+       "      <th>2021_19</th>\n",
+       "      <td>0.750000</td>\n",
+       "      <td>0.255000</td>\n",
+       "      <td>1.034973</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.630000</td>\n",
+       "      <td>0.634150</td>\n",
+       "      <td>4</td>\n",
+       "      <td>3</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_20</th>\n",
+       "      <td>0.800000</td>\n",
+       "      <td>0.372000</td>\n",
+       "      <td>1.142580</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.706000</td>\n",
+       "      <td>0.670480</td>\n",
+       "      <td>5</td>\n",
+       "      <td>4</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_21</th>\n",
+       "      <td>0.333333</td>\n",
+       "      <td>-0.363333</td>\n",
+       "      <td>1.200528</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.746667</td>\n",
+       "      <td>0.686033</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_22</th>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.670000</td>\n",
+       "      <td>1.025046</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.670000</td>\n",
+       "      <td>0.613800</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_23</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-1.000000</td>\n",
+       "      <td>1.011686</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.670000</td>\n",
+       "      <td>0.605800</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2021_24</th>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.755000</td>\n",
+       "      <td>1.155859</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.755000</td>\n",
+       "      <td>0.662375</td>\n",
+       "      <td>4</td>\n",
+       "      <td>4</td>\n",
+       "      <td>model</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                 win    profit  exp_payoff  exp_payoff_prec  \\\n",
+       "bet_type        match_week                                                    \n",
+       "goals_over_2.5  2021_19     0.818182  0.434545    1.193888              NaN   \n",
+       "                2021_20     0.700000  0.144000    1.087417              NaN   \n",
+       "                2021_21     0.800000  0.359000    1.130386              NaN   \n",
+       "                2021_22     0.571429  0.040000    1.130597              NaN   \n",
+       "                2021_23     0.500000 -0.140000    1.216788              NaN   \n",
+       "                2021_24     0.625000  0.058750    1.168317              NaN   \n",
+       "                2021_25     0.000000 -1.000000    1.200240              NaN   \n",
+       "goals_under_2.5 2021_19     0.750000  0.255000    1.034973              NaN   \n",
+       "                2021_20     0.800000  0.372000    1.142580              NaN   \n",
+       "                2021_21     0.333333 -0.363333    1.200528              NaN   \n",
+       "                2021_22     1.000000  0.670000    1.025046              NaN   \n",
+       "                2021_23     0.000000 -1.000000    1.011686              NaN   \n",
+       "                2021_24     1.000000  0.755000    1.155859              NaN   \n",
+       "\n",
+       "                                odds  confidence  count  buenas  model  \n",
+       "bet_type        match_week                                              \n",
+       "goals_over_2.5  2021_19     1.743636    0.683591     11       9  model  \n",
+       "                2021_20     1.641000    0.664270     10       7  model  \n",
+       "                2021_21     1.685000    0.671600     10       8  model  \n",
+       "                2021_22     1.708571    0.662800      7       4  model  \n",
+       "                2021_23     1.815000    0.667750      4       2  model  \n",
+       "                2021_24     1.730000    0.674975      8       5  model  \n",
+       "                2021_25     1.800000    0.666800      1       0  model  \n",
+       "goals_under_2.5 2021_19     1.630000    0.634150      4       3  model  \n",
+       "                2021_20     1.706000    0.670480      5       4  model  \n",
+       "                2021_21     1.746667    0.686033      3       1  model  \n",
+       "                2021_22     1.670000    0.613800      1       1  model  \n",
+       "                2021_23     1.670000    0.605800      1       0  model  \n",
+       "                2021_24     1.755000    0.662375      4       4  model  "
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def show_results(df, groupby=[\"bet_type\", \"match_week\"]):\n",
+    "    ix = np.logical_and(df[\"win\"]>=0, True)# df[\"label\"]>=1)\n",
+    "    #ix = np.logical_and(df[\"consensus\"] >=0.1, ix)\n",
+    "    ix2 = np.logical_and(df[\"odds\"]>=1.5, df[\"odds\"]<2.)\n",
+    "    ix = np.logical_and(ix, ix2)\n",
+    "    #ix = np.logical_and(ix, ~df[\"validation\"])\n",
+    "    #ix = np.logical_and(ix, df[\"exp_payoff_prec\"] >0.88)\n",
+    "    #ix = np.logical_and(ix, df[\"exp_payoff\"] <1.2)\n",
+    "    #ix = np.logical_and(ix, df[\"exp_payoff\"] >0.8)\n",
+    "    ix = np.logical_and(ix, df[\"confidence\"] >=0.60)\n",
+    "    #ix = np.logical_and(ix, df[\"confidence\"] <=0.75)\n",
+    "    x = df[ix].groupby(groupby)[[\"win\", \"profit\", \"exp_payoff\", \"exp_payoff_prec\", \"odds\", \"confidence\"]].mean()\n",
+    "    x[\"count\"] = df[ix].groupby(groupby)[[\"win\"]].count()\n",
+    "    x[\"buenas\"] = df[ix].groupby(groupby)[[\"win\"]].sum().astype(int)\n",
+    "    x[\"model\"] = df[\"model\"].iloc[0]\n",
+    "    return x\n",
+    "show_results(tips)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/setup.py b/setup.py
index 4633506..d175ce8 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@
 
 setup(
     name="feature_selection",
-    description="This module allows the user to select the most relevant features of a dataset in order to use them in a machine learning project",
+    description="Feature-selection finds the most relevant features of a dataset to be used in a classification project.",
     long_description=long_description,
     long_description_content_type="text/markdown",
     packages=find_packages(),