wehub-resource-sync
4610 lines
139 KiB
Plaintext
4610 lines
139 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c59f6ef8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#| eval: false\n",
|
|
"! [ -e /content ] && pip install -Uqq fastai # upgrade fastai on colab"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "866a6043",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| default_exp tabular.core"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7bf25495",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"from fastai.torch_basics import *\n",
|
|
"from fastai.data.all import *"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7bd0e630",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"from nbdev.showdoc import *"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2638ab5a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"pd.set_option('mode.chained_assignment','raise')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e1b505e9",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Tabular core\n",
|
|
"\n",
|
|
"> Basic function to preprocess tabular data before assembling it in a `DataLoaders`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "33f21224",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Initial preprocessing"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0bbabf95",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def make_date(df, date_field):\n",
|
|
" \"Make sure `df[date_field]` is of the right date type.\"\n",
|
|
" field_dtype = df[date_field].dtype\n",
|
|
" if isinstance(field_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):\n",
|
|
" field_dtype = np.datetime64\n",
|
|
" if not isinstance(field_dtype, np.dtype) or not np.issubdtype(field_dtype, np.datetime64):\n",
|
|
" df[date_field] = pd.to_datetime(df[date_field])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fc4c8e6d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df = pd.DataFrame({'date': ['2019-12-04', '2019-11-29', '2019-11-15', '2019-10-24']})\n",
|
|
"make_date(df, 'date')\n",
|
|
"test_eq(df['date'].dtype, np.dtype('datetime64[us]'))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2731eac6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def add_datepart(df, field_name, prefix=None, drop=True, time=False):\n",
|
|
" \"Helper function that adds columns relevant to a date in the column `field_name` of `df`.\"\n",
|
|
" make_date(df, field_name)\n",
|
|
" field = df[field_name]\n",
|
|
" prefix = ifnone(prefix, re.sub('[Dd]ate$', '', field_name))\n",
|
|
" attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear', 'Is_month_end', 'Is_month_start',\n",
|
|
" 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']\n",
|
|
" if time: attr = attr + ['Hour', 'Minute', 'Second']\n",
|
|
" # Pandas removed `dt.week` in v1.1.10\n",
|
|
" week = field.dt.isocalendar().week.astype(field.dt.day.dtype) if hasattr(field.dt, 'isocalendar') else field.dt.week\n",
|
|
" for n in attr: df[prefix + n] = getattr(field.dt, n.lower()) if n != 'Week' else week\n",
|
|
" mask = ~field.isna()\n",
|
|
" df[prefix + 'Elapsed'] = np.where(mask,field.values.astype(np.int64) // 10 ** 9,np.nan)\n",
|
|
" if drop: df.drop(field_name, axis=1, inplace=True)\n",
|
|
" return df"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "3d9ae5e0",
|
|
"metadata": {},
|
|
"source": [
|
|
"For example if we have a series of dates we can then generate features such as `Year`, `Month`, `Day`, `Dayofweek`, `Is_month_start`, etc as shown below:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "f24190a5",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>Year</th>\n",
|
|
" <th>Month</th>\n",
|
|
" <th>Week</th>\n",
|
|
" <th>Day</th>\n",
|
|
" <th>Dayofweek</th>\n",
|
|
" <th>Dayofyear</th>\n",
|
|
" <th>Is_month_end</th>\n",
|
|
" <th>Is_month_start</th>\n",
|
|
" <th>Is_quarter_end</th>\n",
|
|
" <th>Is_quarter_start</th>\n",
|
|
" <th>Is_year_end</th>\n",
|
|
" <th>Is_year_start</th>\n",
|
|
" <th>Elapsed</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>2019.0</td>\n",
|
|
" <td>12.0</td>\n",
|
|
" <td>49.0</td>\n",
|
|
" <td>4.0</td>\n",
|
|
" <td>2.0</td>\n",
|
|
" <td>338.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>1575417.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>2019.0</td>\n",
|
|
" <td>11.0</td>\n",
|
|
" <td>46.0</td>\n",
|
|
" <td>15.0</td>\n",
|
|
" <td>4.0</td>\n",
|
|
" <td>319.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>1573776.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>2019.0</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td>43.0</td>\n",
|
|
" <td>24.0</td>\n",
|
|
" <td>3.0</td>\n",
|
|
" <td>297.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>1571875.0</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" Year Month Week ... Is_year_end Is_year_start Elapsed\n",
|
|
"0 2019.0 12.0 49.0 ... False False 1575417.0\n",
|
|
"1 NaN NaN NaN ... False False NaN\n",
|
|
"2 2019.0 11.0 46.0 ... False False 1573776.0\n",
|
|
"3 2019.0 10.0 43.0 ... False False 1571875.0\n",
|
|
"\n",
|
|
"[4 rows x 13 columns]"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df = pd.DataFrame({'date': ['2019-12-04', None, '2019-11-15', '2019-10-24']})\n",
|
|
"df = add_datepart(df, 'date')\n",
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ebceaa3d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"test_eq(df.columns, ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear', 'Is_month_end', 'Is_month_start',\n",
|
|
" 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start', 'Elapsed'])\n",
|
|
"test_eq(df[df.Elapsed.isna()].shape,(1, 13))\n",
|
|
"\n",
|
|
"# Test that week dtype is consistent with other datepart fields\n",
|
|
"test_eq(df['Year'].dtype, df['Week'].dtype)\n",
|
|
"\n",
|
|
"test_eq(pd.api.types.is_numeric_dtype(df['Elapsed']), True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0f87626f",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>f1</th>\n",
|
|
" <th>f2</th>\n",
|
|
" <th>f3</th>\n",
|
|
" <th>f4</th>\n",
|
|
" <th>Year</th>\n",
|
|
" <th>Month</th>\n",
|
|
" <th>Week</th>\n",
|
|
" <th>Day</th>\n",
|
|
" <th>Dayofweek</th>\n",
|
|
" <th>Dayofyear</th>\n",
|
|
" <th>Is_month_end</th>\n",
|
|
" <th>Is_month_start</th>\n",
|
|
" <th>Is_quarter_end</th>\n",
|
|
" <th>Is_quarter_start</th>\n",
|
|
" <th>Is_year_end</th>\n",
|
|
" <th>Is_year_start</th>\n",
|
|
" <th>Elapsed</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>1.0</td>\n",
|
|
" <td>2.0</td>\n",
|
|
" <td>3.0</td>\n",
|
|
" <td>4.0</td>\n",
|
|
" <td>2019</td>\n",
|
|
" <td>12</td>\n",
|
|
" <td>49</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>338</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>1575417.0</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" f1 f2 f3 f4 ... Is_quarter_start Is_year_end Is_year_start Elapsed\n",
|
|
"0 1.0 2.0 3.0 4.0 ... False False False 1575417.0\n",
|
|
"\n",
|
|
"[1 rows x 17 columns]"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"df = pd.DataFrame({'f1': [1.],'f2': [2.],'f3': [3.],'f4': [4.],'date':['2019-12-04']})\n",
|
|
"df = add_datepart(df, 'date')\n",
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "edcd1107",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"# Test Order of columns when date isn't in first position\n",
|
|
"test_eq(df.columns, ['f1', 'f2', 'f3', 'f4', 'Year', 'Month', 'Week', 'Day',\n",
|
|
" 'Dayofweek', 'Dayofyear', 'Is_month_end', 'Is_month_start',\n",
|
|
" 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start', 'Elapsed'])\n",
|
|
"\n",
|
|
"# Test that week dtype is consistent with other datepart fields\n",
|
|
"test_eq(df['Year'].dtype, df['Week'].dtype)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4d78de59",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def _get_elapsed(df,field_names, date_field, base_field, prefix):\n",
|
|
" for f in field_names:\n",
|
|
" day1 = np.timedelta64(1, 'D')\n",
|
|
" last_date,last_base,res = np.datetime64(),None,[]\n",
|
|
" for b,v,d in zip(df[base_field].values, df[f].values, df[date_field].values):\n",
|
|
" if last_base is None or b != last_base:\n",
|
|
" last_date,last_base = np.datetime64(),b\n",
|
|
" if v: last_date = d\n",
|
|
" res.append(((d-last_date).astype('timedelta64[D]') / day1))\n",
|
|
" df[prefix + f] = res\n",
|
|
" return df"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e3901ffe",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def add_elapsed_times(df, field_names, date_field, base_field):\n",
|
|
" \"Add in `df` for each event in `field_names` the elapsed time according to `date_field` grouped by `base_field`\"\n",
|
|
" field_names = list(L(field_names))\n",
|
|
" #Make sure date_field is a date and base_field a bool\n",
|
|
" df[field_names] = df[field_names].astype('bool')\n",
|
|
" make_date(df, date_field)\n",
|
|
"\n",
|
|
" work_df = df[field_names + [date_field, base_field]]\n",
|
|
" work_df = work_df.sort_values([base_field, date_field])\n",
|
|
" work_df = _get_elapsed(work_df, field_names, date_field, base_field, 'After')\n",
|
|
" work_df = work_df.sort_values([base_field, date_field], ascending=[True, False])\n",
|
|
" work_df = _get_elapsed(work_df, field_names, date_field, base_field, 'Before')\n",
|
|
"\n",
|
|
" for a in ['After' + f for f in field_names] + ['Before' + f for f in field_names]:\n",
|
|
" work_df[a] = work_df[a].fillna(0).astype(int)\n",
|
|
"\n",
|
|
" for a,s in zip([True, False], ['_bw', '_fw']):\n",
|
|
" work_df = work_df.set_index(date_field)\n",
|
|
" tmp = (work_df[[base_field] + field_names].sort_index(ascending=a)\n",
|
|
" .groupby(base_field).rolling(7, min_periods=1).sum())\n",
|
|
" if base_field in tmp: tmp.drop(base_field, axis=1,inplace=True)\n",
|
|
" tmp.reset_index(inplace=True)\n",
|
|
" work_df.reset_index(inplace=True)\n",
|
|
" work_df = work_df.merge(tmp, 'left', [date_field, base_field], suffixes=['', s])\n",
|
|
" work_df.drop(field_names, axis=1, inplace=True)\n",
|
|
" return df.merge(work_df, 'left', [date_field, base_field])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a8cf138e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>date</th>\n",
|
|
" <th>event</th>\n",
|
|
" <th>base</th>\n",
|
|
" <th>Afterevent</th>\n",
|
|
" <th>Beforeevent</th>\n",
|
|
" <th>event_bw</th>\n",
|
|
" <th>event_fw</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>2019-12-04</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" <td>0.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>2019-11-29</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>2019-11-15</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>22</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" <td>0.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>2019-10-24</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" <td>1.0</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" date event base Afterevent Beforeevent event_bw event_fw\n",
|
|
"0 2019-12-04 False 1 5 0 1.0 0.0\n",
|
|
"1 2019-11-29 True 1 0 0 1.0 1.0\n",
|
|
"2 2019-11-15 False 2 22 0 1.0 0.0\n",
|
|
"3 2019-10-24 True 2 0 0 1.0 1.0"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df = pd.DataFrame({'date': ['2019-12-04', '2019-11-29', '2019-11-15', '2019-10-24'],\n",
|
|
" 'event': [False, True, False, True], 'base': [1,1,2,2]})\n",
|
|
"df = add_elapsed_times(df, ['event'], 'date', 'base')\n",
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6086b66b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def cont_cat_split(df, max_card=20, dep_var=None):\n",
|
|
" \"Helper function that returns column names of cont and cat variables from given `df`.\"\n",
|
|
" cont_names, cat_names = [], []\n",
|
|
" for label in df:\n",
|
|
" if label in L(dep_var): continue\n",
|
|
" if ((pd.api.types.is_integer_dtype(df[label].dtype) and\n",
|
|
" df[label].unique().shape[0] > max_card) or\n",
|
|
" pd.api.types.is_float_dtype(df[label].dtype)):\n",
|
|
" cont_names.append(label)\n",
|
|
" else: cat_names.append(label)\n",
|
|
" return cont_names, cat_names"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "03b5fefd",
|
|
"metadata": {},
|
|
"source": [
|
|
"This function works by determining if a column is continuous or categorical based on the cardinality of its values. If it is above the `max_card` parameter (or a `float` datatype) then it will be added to the `cont_names` else `cat_names`. An example is below:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0b252f06",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Example with simple numpy types\n",
|
|
"df = pd.DataFrame({'cat1': [1, 2, 3, 4], 'cont1': [1., 2., 3., 2.], 'cat2': ['a', 'b', 'b', 'a'],\n",
|
|
" 'i8': pd.Series([1, 2, 3, 4], dtype='int8'),\n",
|
|
" 'u8': pd.Series([1, 2, 3, 4], dtype='uint8'),\n",
|
|
" 'f16': pd.Series([1, 2, 3, 4], dtype='float16'),\n",
|
|
" 'y1': [1, 0, 1, 0], 'y2': [2, 1, 1, 0]})\n",
|
|
"cont_names, cat_names = cont_cat_split(df)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e573e687",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"cont_names: ['cont1', 'f16']\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"cat_names: ['cat1', 'cat2', 'i8', 'u8', 'y1', 'y2']`\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| echo: false\n",
|
|
"print(f'cont_names: {cont_names}\\ncat_names: {cat_names}`')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2ce5de4d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"# Test all columns\n",
|
|
"cont, cat = cont_cat_split(df)\n",
|
|
"test_eq((cont, cat), (['cont1', 'f16'], ['cat1', 'cat2', 'i8', 'u8', 'y1', 'y2']))\n",
|
|
"\n",
|
|
"# Test exclusion of dependent variable\n",
|
|
"cont, cat = cont_cat_split(df, dep_var='y1')\n",
|
|
"test_eq((cont, cat), (['cont1', 'f16'], ['cat1', 'cat2', 'i8', 'u8', 'y2']))\n",
|
|
"\n",
|
|
"# Test exclusion of multi-label dependent variables\n",
|
|
"cont, cat = cont_cat_split(df, dep_var=['y1', 'y2'])\n",
|
|
"test_eq((cont, cat), (['cont1', 'f16'], ['cat1', 'cat2', 'i8', 'u8']))\n",
|
|
"\n",
|
|
"# Test maximal cardinality bound for int variable\n",
|
|
"cont, cat = cont_cat_split(df, max_card=3)\n",
|
|
"test_eq((cont, cat), (['cat1', 'cont1', 'i8', 'u8', 'f16'], ['cat2', 'y1', 'y2']))\n",
|
|
"cont, cat = cont_cat_split(df, max_card=2)\n",
|
|
"test_eq((cont, cat), (['cat1', 'cont1', 'i8', 'u8', 'f16', 'y2'], ['cat2', 'y1']))\n",
|
|
"cont, cat = cont_cat_split(df, max_card=1)\n",
|
|
"test_eq((cont, cat), (['cat1', 'cont1', 'i8', 'u8', 'f16', 'y1', 'y2'], ['cat2']))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bcde5c9a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Example with pandas types and generated columns\n",
|
|
"df = pd.DataFrame({'cat1': pd.Series(['l','xs','xl','s'], dtype='category'),\n",
|
|
" 'ui32': pd.Series([1, 2, 3, 4], dtype='UInt32'),\n",
|
|
" 'i64': pd.Series([1, 2, 3, 4], dtype='Int64'),\n",
|
|
" 'f16': pd.Series([1, 2, 3, 4], dtype='Float64'),\n",
|
|
" 'd1_date': ['2021-02-09', None, '2020-05-12', '2020-08-14'],\n",
|
|
" })\n",
|
|
"df = add_datepart(df, 'd1_date', drop=False)\n",
|
|
"df['cat1'] = df['cat1'].cat.set_categories(['xl','l','m','s','xs'], ordered=True)\n",
|
|
"cont_names, cat_names = cont_cat_split(df, max_card=0)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e5cdebe7",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"cont_names: ['ui32', 'i64', 'f16', 'd1_Year', 'd1_Month', 'd1_Week', 'd1_Day', 'd1_Dayofweek', 'd1_Dayofyear', 'd1_Elapsed']\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"cat_names: ['cat1', 'd1_date', 'd1_Is_month_end', 'd1_Is_month_start', 'd1_Is_quarter_end', 'd1_Is_quarter_start', 'd1_Is_year_end', 'd1_Is_year_start']\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| echo: false\n",
|
|
"print(f'cont_names: {cont_names}\\ncat_names: {cat_names}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8fcdf5b4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"cont, cat = cont_cat_split(df, max_card=0)\n",
|
|
"test_eq((cont, cat), (\n",
|
|
" ['ui32', 'i64', 'f16', 'd1_Year', 'd1_Month', 'd1_Week', 'd1_Day', 'd1_Dayofweek', 'd1_Dayofyear', 'd1_Elapsed'],\n",
|
|
" ['cat1', 'd1_date', 'd1_Is_month_end', 'd1_Is_month_start', 'd1_Is_quarter_end', 'd1_Is_quarter_start', 'd1_Is_year_end', 'd1_Is_year_start']\n",
|
|
" ))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9a3b6c97",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def df_shrink_dtypes(df, skip=[], obj2cat=True, int2uint=False):\n",
|
|
" \"Return any possible smaller data types for DataFrame columns. Allows `object`->`category`, `int`->`uint`, and exclusion.\"\n",
|
|
"\n",
|
|
" # 1: Build column filter and typemap\n",
|
|
" excl_types, skip = {'category','datetime64[ns]','bool'}, set(skip)\n",
|
|
"\n",
|
|
" typemap = {'int' : [(np.dtype(x), np.iinfo(x).min, np.iinfo(x).max) for x in (np.int8, np.int16, np.int32, np.int64)],\n",
|
|
" 'uint' : [(np.dtype(x), np.iinfo(x).min, np.iinfo(x).max) for x in (np.uint8, np.uint16, np.uint32, np.uint64)],\n",
|
|
" 'float' : [(np.dtype(x), np.finfo(x).min, np.finfo(x).max) for x in (np.float32, np.float64, np.longdouble)]\n",
|
|
" }\n",
|
|
" if obj2cat: typemap['object'] = typemap['str'] = 'category'\n",
|
|
" else: excl_types.update({'object', 'str'})\n",
|
|
"\n",
|
|
" new_dtypes = {}\n",
|
|
" exclude = lambda dt: dt[1].name not in excl_types and dt[0] not in skip\n",
|
|
"\n",
|
|
" for c, old_t in filter(exclude, df.dtypes.items()):\n",
|
|
" t = next((v for k,v in typemap.items() if old_t.name.startswith(k)), None)\n",
|
|
"\n",
|
|
" if isinstance(t, list): # Find the smallest type that fits\n",
|
|
" if int2uint and t==typemap['int'] and df[c].min() >= 0: t=typemap['uint']\n",
|
|
" new_t = next((r[0] for r in t if r[1]<=df[c].min() and r[2]>=df[c].max()), None)\n",
|
|
" if new_t and new_t == old_t: new_t = None\n",
|
|
" else: new_t = t if isinstance(t, str) else None\n",
|
|
"\n",
|
|
" if new_t: new_dtypes[c] = new_t\n",
|
|
" return new_dtypes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2cf725eb",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/markdown": [
|
|
"---\n",
|
|
"\n",
|
|
"[source](https://github.com/fastai/fastai/blob/main/fastai/tabular/core.py#L99){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
|
|
"\n",
|
|
"### df_shrink_dtypes\n",
|
|
"\n",
|
|
"```python\n",
|
|
"\n",
|
|
"def df_shrink_dtypes(\n",
|
|
" df, skip:list=[], obj2cat:bool=True, int2uint:bool=False\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Return any possible smaller data types for DataFrame columns. Allows `object`->`category`, `int`->`uint`, and exclusion.*"
|
|
],
|
|
"text/plain": [
|
|
"```python\n",
|
|
"\n",
|
|
"def df_shrink_dtypes(\n",
|
|
" df, skip:list=[], obj2cat:bool=True, int2uint:bool=False\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Return any possible smaller data types for DataFrame columns. Allows `object`->`category`, `int`->`uint`, and exclusion.*"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"show_doc(df_shrink_dtypes, title_level=3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "eb2e1776",
|
|
"metadata": {},
|
|
"source": [
|
|
"For example we will make a sample `DataFrame` with `int`, `float`, `bool`, and `object` datatypes:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "83c7879a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"i int64\n",
|
|
"f float64\n",
|
|
"e bool\n",
|
|
"date str\n",
|
|
"dtype: object"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df = pd.DataFrame({'i': [-100, 0, 100], 'f': [-100.0, 0.0, 100.0], 'e': [True, False, True],\n",
|
|
" 'date':['2019-12-04','2019-11-29','2019-11-15',]})\n",
|
|
"df.dtypes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "fdf8e5cb",
|
|
"metadata": {},
|
|
"source": [
|
|
"We can then call `df_shrink_dtypes` to find the smallest possible datatype that can support the data:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "aae29084",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"{'i': dtype('int8'), 'f': dtype('float32'), 'date': 'category'}"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"dt = df_shrink_dtypes(df)\n",
|
|
"dt"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "1d33d1d8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"test_eq(df['i'].dtype, 'int64')\n",
|
|
"test_eq(dt['i'], 'int8')\n",
|
|
"\n",
|
|
"test_eq(df['f'].dtype, 'float64')\n",
|
|
"test_eq(dt['f'], 'float32')\n",
|
|
"\n",
|
|
"test_eq(df['date'].dtype, 'str')\n",
|
|
"test_eq(dt['date'], 'category')\n",
|
|
"\n",
|
|
"dt2 = df_shrink_dtypes(df, obj2cat=False)\n",
|
|
"test_eq('date' not in dt2, True)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a97f5143",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def df_shrink(df, skip=[], obj2cat=True, int2uint=False):\n",
|
|
" \"Reduce DataFrame memory usage, by casting to smaller types returned by `df_shrink_dtypes()`.\"\n",
|
|
" dt = df_shrink_dtypes(df, skip, obj2cat=obj2cat, int2uint=int2uint)\n",
|
|
" return df.astype(dt)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "663c5991",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/markdown": [
|
|
"---\n",
|
|
"\n",
|
|
"[source](https://github.com/fastai/fastai/blob/main/fastai/tabular/core.py#L128){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
|
|
"\n",
|
|
"### df_shrink\n",
|
|
"\n",
|
|
"```python\n",
|
|
"\n",
|
|
"def df_shrink(\n",
|
|
" df, skip:list=[], obj2cat:bool=True, int2uint:bool=False\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Reduce DataFrame memory usage, by casting to smaller types returned by `df_shrink_dtypes()`.*"
|
|
],
|
|
"text/plain": [
|
|
"```python\n",
|
|
"\n",
|
|
"def df_shrink(\n",
|
|
" df, skip:list=[], obj2cat:bool=True, int2uint:bool=False\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Reduce DataFrame memory usage, by casting to smaller types returned by `df_shrink_dtypes()`.*"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"show_doc(df_shrink, title_level=3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b57d6f90",
|
|
"metadata": {},
|
|
"source": [
|
|
"`df_shrink(df)` attempts to make a DataFrame uses less memory, by fit numeric columns into smallest datatypes. In addition:\n",
|
|
"\n",
|
|
" * `boolean`, `category`, `datetime64[ns]` dtype columns are ignored.\n",
|
|
" * 'object' type columns are categorified, which can save a lot of memory in large dataset. It can be turned off by `obj2cat=False`.\n",
|
|
" * `int2uint=True`, to fit `int` types to `uint` types, if all data in the column is >= 0.\n",
|
|
" * columns can be excluded by name using `excl_cols=['col1','col2']`.\n",
|
|
"\n",
|
|
"To get only new column data types without actually casting a DataFrame,\n",
|
|
"use `df_shrink_dtypes()` with all the same parameters for `df_shrink()`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "03986bdf",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df = pd.DataFrame({'i': [-100, 0, 100], 'f': [-100.0, 0.0, 100.0], 'u':[0, 10,254],\n",
|
|
" 'date':['2019-12-04','2019-11-29','2019-11-15']})\n",
|
|
"df2 = df_shrink(df, skip=['date'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "60ce3613",
|
|
"metadata": {},
|
|
"source": [
|
|
"Let's compare the two:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "de4b9958",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"i int64\n",
|
|
"f float64\n",
|
|
"u int64\n",
|
|
"date str\n",
|
|
"dtype: object"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df.dtypes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4cd57e5b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"i int8\n",
|
|
"f float32\n",
|
|
"u int16\n",
|
|
"date str\n",
|
|
"dtype: object"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df2.dtypes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b8999949",
|
|
"metadata": {},
|
|
"source": [
|
|
"We can see that the datatypes changed, and even further we can look at their relative memory usages:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "470b8e60",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Initial Dataframe: 228 bytes\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Reduced Dataframe: 177 bytes\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| echo: false\n",
|
|
"print(f'Initial Dataframe: {df.memory_usage().sum()} bytes')\n",
|
|
"print(f'Reduced Dataframe: {df2.memory_usage().sum()} bytes')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "caa0a883",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"test_eq(df['i'].dtype=='int64' and df2['i'].dtype=='int8', True)\n",
|
|
"test_eq(df['f'].dtype=='float64' and df2['f'].dtype=='float32', True)\n",
|
|
"test_eq(df['u'].dtype=='int64' and df2['u'].dtype=='int16', True)\n",
|
|
"test_eq(df2['date'].dtype, 'str')\n",
|
|
"\n",
|
|
"test_eq(df2.memory_usage().sum() < df.memory_usage().sum(), True)\n",
|
|
"\n",
|
|
"# Test int => uint (when col.min() >= 0)\n",
|
|
"df3 = df_shrink(df, int2uint=True)\n",
|
|
"test_eq(df3['u'].dtype, 'uint8') # int64 -> uint8 instead of int16\n",
|
|
"\n",
|
|
"# Test excluding columns\n",
|
|
"df4 = df_shrink(df, skip=['i','u'])\n",
|
|
"test_eq(df['i'].dtype, df4['i'].dtype)\n",
|
|
"test_eq(df4['u'].dtype, 'int64')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f17bcf7d",
|
|
"metadata": {},
|
|
"source": [
|
|
"Here's another example using the `ADULT_SAMPLE` dataset:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d770bbbd",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"path = untar_data(URLs.ADULT_SAMPLE)\n",
|
|
"df = pd.read_csv(path/'adult.csv')\n",
|
|
"new_df = df_shrink(df, int2uint=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "92caa755",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Initial Dataframe: 3.907452 megabytes\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Reduced Dataframe: 0.814989 megabytes\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| echo: false\n",
|
|
"print(f'Initial Dataframe: {df.memory_usage().sum() / 1000000} megabytes')\n",
|
|
"print(f'Reduced Dataframe: {new_df.memory_usage().sum() / 1000000} megabytes')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "471a2a21",
|
|
"metadata": {},
|
|
"source": [
|
|
"We reduced the overall memory used by 79%!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "21299434",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Tabular -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d3046984",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class _TabIloc:\n",
|
|
" \"Get/set rows by iloc and cols by name\"\n",
|
|
" def __init__(self,to): self.to = to\n",
|
|
" def __getitem__(self, idxs):\n",
|
|
" df = self.to.items\n",
|
|
" if isinstance(idxs,tuple):\n",
|
|
" rows,cols = idxs\n",
|
|
" cols = df.columns.isin(cols) if is_listy(cols) else df.columns.get_loc(cols)\n",
|
|
" else: rows,cols = idxs,slice(None)\n",
|
|
" return self.to.new(df.iloc[rows, cols])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "498b3f92",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class Tabular(CollBase, GetAttr, FilteredBase):\n",
|
|
" \"A `DataFrame` wrapper that knows which cols are cont/cat/y, and returns rows in `__getitem__`\"\n",
|
|
" _default,with_cont='procs',True\n",
|
|
" def __init__(self, df, procs=None, cat_names=None, cont_names=None, y_names=None, y_block=None, splits=None,\n",
|
|
" do_setup=True, device=None, inplace=False, reduce_memory=True):\n",
|
|
" if inplace and splits is not None and pd.options.mode.chained_assignment is not None:\n",
|
|
" warn(\"Using inplace with splits will trigger a pandas error. Set `pd.options.mode.chained_assignment=None` to avoid it.\")\n",
|
|
" if not inplace: df = df.copy()\n",
|
|
" if reduce_memory: df = df_shrink(df)\n",
|
|
" if splits is not None: df = df.iloc[sum(splits, [])]\n",
|
|
" self.dataloaders = delegates(self._dl_type.__init__)(self.dataloaders)\n",
|
|
" super().__init__(df)\n",
|
|
"\n",
|
|
" self.y_names,self.device = L(y_names),device\n",
|
|
" if y_block is None and self.y_names:\n",
|
|
" # Make ys categorical if they're not numeric\n",
|
|
" ys = df[self.y_names]\n",
|
|
" if len(ys.select_dtypes(include='number').columns)!=len(ys.columns): y_block = CategoryBlock()\n",
|
|
" else: y_block = RegressionBlock()\n",
|
|
" if y_block is not None and do_setup:\n",
|
|
" if callable(y_block): y_block = y_block()\n",
|
|
" procs = L(procs) + y_block.type_tfms\n",
|
|
" self.cat_names,self.cont_names,self.procs = L(cat_names),L(cont_names),Pipeline(procs)\n",
|
|
" self.split = len(df) if splits is None else len(splits[0])\n",
|
|
" if do_setup: self.setup()\n",
|
|
"\n",
|
|
" def new(self, df, inplace=False):\n",
|
|
" return type(self)(df, do_setup=False, reduce_memory=False, y_block=TransformBlock(), inplace=inplace,\n",
|
|
" **attrdict(self, 'procs','cat_names','cont_names','y_names', 'device'))\n",
|
|
"\n",
|
|
" def subset(self, i): return self.new(self.items[slice(0,self.split) if i==0 else slice(self.split,len(self))])\n",
|
|
" def copy(self): self.items = self.items.copy(); return self\n",
|
|
" def decode(self): return self.procs.decode(self)\n",
|
|
" def decode_row(self, row): return self.new(pd.DataFrame(row).T).decode().items.iloc[0]\n",
|
|
" def show(self, max_n=10, **kwargs): display_df(self.new(self.all_cols[:max_n]).decode().items)\n",
|
|
" def setup(self): self.procs.setup(self)\n",
|
|
" def process(self): self.procs(self)\n",
|
|
" def loc(self): return self.items.loc\n",
|
|
" def iloc(self): return _TabIloc(self)\n",
|
|
" def targ(self): return self.items[self.y_names]\n",
|
|
" def x_names (self): return self.cat_names + self.cont_names\n",
|
|
" def n_subsets(self): return 2\n",
|
|
" def y(self): return self[self.y_names[0]]\n",
|
|
" def new_empty(self): return self.new(pd.DataFrame({}, columns=self.items.columns))\n",
|
|
" def to_device(self, d=None):\n",
|
|
" self.device = d\n",
|
|
" return self\n",
|
|
"\n",
|
|
" def all_col_names (self):\n",
|
|
" ys = [n for n in self.y_names if n in self.items.columns]\n",
|
|
" return self.x_names + self.y_names if len(ys) == len(self.y_names) else self.x_names\n",
|
|
"\n",
|
|
"properties(Tabular,'loc','iloc','targ','all_col_names','n_subsets','x_names','y')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c5a5cf34",
|
|
"metadata": {},
|
|
"source": [
|
|
"* `df`: A `DataFrame` of your data\n",
|
|
"* `cat_names`: Your categorical `x` variables\n",
|
|
"* `cont_names`: Your continuous `x` variables\n",
|
|
"* `y_names`: Your dependent `y` variables\n",
|
|
" * Note: Mixed y's such as Regression and Classification is not currently supported, however multiple regression or classification outputs is\n",
|
|
"* `y_block`: How to sub-categorize the type of `y_names` (`CategoryBlock` or `RegressionBlock`)\n",
|
|
"* `splits`: How to split your data\n",
|
|
"* `do_setup`: A parameter for if `Tabular` will run the data through the `procs` upon initialization\n",
|
|
"* `device`: `cuda` or `cpu`\n",
|
|
"* `inplace`: If `True`, `Tabular` will not keep a separate copy of your original `DataFrame` in memory. You should ensure `pd.options.mode.chained_assignment` is `None` before setting this\n",
|
|
"* `reduce_memory`: `fastai` will attempt to reduce the overall memory usage by the inputted `DataFrame` with `df_shrink`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e23698c9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class TabularPandas(Tabular):\n",
|
|
" \"A `Tabular` object with transforms\"\n",
|
|
" def transform(self, cols, f, all_col=True):\n",
|
|
" if not all_col: cols = [c for c in cols if c in self.items.columns]\n",
|
|
" if len(cols) > 0: self[cols] = self[cols].transform(f)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "efdb998b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def _add_prop(cls, nm):\n",
|
|
" @property\n",
|
|
" def f(o): return o[list(getattr(o,nm+'_names'))]\n",
|
|
" @f.setter\n",
|
|
" def fset(o, v): o[getattr(o,nm+'_names')] = v\n",
|
|
" setattr(cls, nm+'s', f)\n",
|
|
" setattr(cls, nm+'s', fset)\n",
|
|
"\n",
|
|
"_add_prop(Tabular, 'cat')\n",
|
|
"_add_prop(Tabular, 'cont')\n",
|
|
"_add_prop(Tabular, 'y')\n",
|
|
"_add_prop(Tabular, 'x')\n",
|
|
"_add_prop(Tabular, 'all_col')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "372c322d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,0,2], 'b':[0,0,0,0,1]})\n",
|
|
"to = TabularPandas(df, cat_names='a')\n",
|
|
"t = pickle.loads(pickle.dumps(to))\n",
|
|
"test_eq(t.items,to.items)\n",
|
|
"test_eq(to.all_cols,to[['a']])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "089ce33e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"import gc"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ff01aaac",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipymini_23754/1389180172.py:6: FutureWarning: `torch.distributed.reduce_op` is deprecated, please use `torch.distributed.ReduceOp` instead\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" return len([x for x in objs if isinstance(x, pd.DataFrame)])\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"def _count_objs(o):\n",
|
|
" \"Counts number of instanes of class `o`\"\n",
|
|
" gc.collect() # Force garbage collection\n",
|
|
" objs = gc.get_objects()\n",
|
|
" return len([x for x in objs if isinstance(x, pd.DataFrame)])\n",
|
|
"\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,0,2], 'b':[0,0,0,0,1]})\n",
|
|
"df_b = pd.DataFrame({'a':[1,2,0,0,2], 'b':[1,0,3,0,1]})\n",
|
|
"\n",
|
|
"to = TabularPandas(df, cat_names='a', inplace=True)\n",
|
|
"\n",
|
|
"_init_count = _count_objs(pd.DataFrame)\n",
|
|
"to_new = to.new(df_b, inplace=True)\n",
|
|
"test_eq(_init_count, _count_objs(pd.DataFrame))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5c7cd5ef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class TabularProc(InplaceTransform):\n",
|
|
" \"Base class to write a non-lazy tabular processor for dataframes\"\n",
|
|
" def setup(self, items=None, train_setup=False): #TODO: properly deal with train_setup\n",
|
|
" super().setup(getattr(items,'train',items), train_setup=False)\n",
|
|
" # Procs are called as soon as data is available\n",
|
|
" return self(items.items if isinstance(items,Datasets) else items)\n",
|
|
"\n",
|
|
" @property\n",
|
|
" def name(self): return f\"{super().name} -- {getattr(self,'__stored_args__',{})}\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "80609ba3",
|
|
"metadata": {},
|
|
"source": [
|
|
"These transforms are applied as soon as the data is available rather than as data is called from the `DataLoader`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "87389d74",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def _apply_cats (voc, add, c):\n",
|
|
" if not (hasattr(c, 'dtype') and isinstance(c.dtype, CategoricalDtype)):\n",
|
|
" return pd.Categorical(c, categories=voc[c.name][add:]).codes+add\n",
|
|
" return c.cat.codes+add #if is_categorical_dtype(c) else c.map(voc[c.name].o2i)\n",
|
|
"def _decode_cats(voc, c): return c.map(dict(enumerate(voc[c.name].items)))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "958b9760",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class Categorify(TabularProc):\n",
|
|
" \"Transform the categorical variables to something similar to `pd.Categorical`\"\n",
|
|
" order = 1\n",
|
|
" def setups(self, to):\n",
|
|
" store_attr(classes={n:CategoryMap(to.iloc[:,n].items, add_na=(n in to.cat_names)) for n in to.cat_names}, but='to')\n",
|
|
"\n",
|
|
" def encodes(self, to): to.transform(to.cat_names, partial(_apply_cats, self.classes, 1))\n",
|
|
" def decodes(self, to): to.transform(to.cat_names, partial(_decode_cats, self.classes))\n",
|
|
" def __getitem__(self,k): return self.classes[k]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "1a858dde",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| exporti\n",
|
|
"@Categorize\n",
|
|
"def setups(self, to:Tabular):\n",
|
|
" if len(to.y_names) > 0:\n",
|
|
" if self.vocab is None:\n",
|
|
" self.vocab = CategoryMap(getattr(to, 'train', to).iloc[:,to.y_names[0]].items, strict=True)\n",
|
|
" else:\n",
|
|
" self.vocab = CategoryMap(self.vocab, sort=False, add_na=self.add_na)\n",
|
|
" self.c = len(self.vocab)\n",
|
|
" return self(to)\n",
|
|
"\n",
|
|
"@Categorize\n",
|
|
"def encodes(self, to:Tabular):\n",
|
|
" to.transform(to.y_names, partial(_apply_cats, {n: self.vocab for n in to.y_names}, 0), all_col=False)\n",
|
|
" return to\n",
|
|
"\n",
|
|
"@Categorize\n",
|
|
"def decodes(self, to:Tabular):\n",
|
|
" to.transform(to.y_names, partial(_decode_cats, {n: self.vocab for n in to.y_names}), all_col=False)\n",
|
|
" return to"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "81779533",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/markdown": [
|
|
"---\n",
|
|
"\n",
|
|
"[source](https://github.com/fastai/fastai/blob/main/fastai/tabular/core.py#L241){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
|
|
"\n",
|
|
"### Categorify\n",
|
|
"\n",
|
|
"```python\n",
|
|
"\n",
|
|
"def Categorify(\n",
|
|
" enc:NoneType=None, dec:NoneType=None, split_idx:NoneType=None, order:NoneType=None\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Transform the categorical variables to something similar to `pd.Categorical`*"
|
|
],
|
|
"text/plain": [
|
|
"```python\n",
|
|
"\n",
|
|
"def Categorify(\n",
|
|
" enc:NoneType=None, dec:NoneType=None, split_idx:NoneType=None, order:NoneType=None\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Transform the categorical variables to something similar to `pd.Categorical`*"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"show_doc(Categorify, title_level=3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "3e434510",
|
|
"metadata": {},
|
|
"source": [
|
|
"While visually in the `DataFrame` you will not see a change, the classes are stored in `to.procs.categorify` as we can see below on a dummy `DataFrame`:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6b1d8488",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>a</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>2</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>2</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"df = pd.DataFrame({'a':[0,1,2,0,2]})\n",
|
|
"to = TabularPandas(df, Categorify, 'a')\n",
|
|
"to.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "33dec0c8",
|
|
"metadata": {},
|
|
"source": [
|
|
"Each column's unique values are stored in a dictionary of `column:[values]`:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3ab3e3cb",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"{'a': ['#na#', np.int8(0), np.int8(1), np.int8(2)]}"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"cat = to.procs.categorify\n",
|
|
"cat.classes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6fdc3a77",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"def test_series(a,b): return test_eq(list(a), b)\n",
|
|
"test_series(cat['a'], ['#na#',0,1,2])\n",
|
|
"test_series(to['a'], [1,2,3,1,3])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7d7de6b0",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipymini_23754/3600165379.py:3: Pandas4Warning: Constructing a Categorical with a dtype and values containing non-null entries not in that dtype's categories is deprecated and will raise in a future version.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" return pd.Categorical(c, categories=voc[c.name][add:]).codes+add\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"df1 = pd.DataFrame({'a':[1,0,3,-1,2]})\n",
|
|
"to1 = to.new(df1)\n",
|
|
"to1.process()\n",
|
|
"#Values that weren't in the training df are sent to 0 (na)\n",
|
|
"test_series(to1['a'], [2,1,0,0,3])\n",
|
|
"to2 = cat.decode(to1)\n",
|
|
"test_series(to2['a'], [1,0,'#na#','#na#',2])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4890eef4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipymini_23754/3600165379.py:3: Pandas4Warning: Constructing a Categorical with a dtype and values containing non-null entries not in that dtype's categories is deprecated and will raise in a future version.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" return pd.Categorical(c, categories=voc[c.name][add:]).codes+add\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#test with splits\n",
|
|
"cat = Categorify()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,2]})\n",
|
|
"to = TabularPandas(df, cat, 'a', splits=[[0,1,2],[3,4]])\n",
|
|
"test_series(cat['a'], ['#na#',0,1,2])\n",
|
|
"test_series(to['a'], [1,2,3,0,3])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "939d92fb",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"df = pd.DataFrame({'a':pd.Categorical(['M','H','L','M'], categories=['H','M','L'], ordered=True)})\n",
|
|
"to = TabularPandas(df, Categorify, 'a')\n",
|
|
"cat = to.procs.categorify\n",
|
|
"test_series(cat['a'], ['#na#','H','M','L'])\n",
|
|
"test_series(to.items.a, [2,1,3,2])\n",
|
|
"to2 = cat.decode(to)\n",
|
|
"test_series(to2['a'], ['M','H','L','M'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "21145135",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipymini_23754/3600165379.py:3: Pandas4Warning: Constructing a Categorical with a dtype and values containing non-null entries not in that dtype's categories is deprecated and will raise in a future version.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" return pd.Categorical(c, categories=voc[c.name][add:]).codes+add\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#test with targets\n",
|
|
"cat = Categorify()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,2], 'b': ['a', 'b', 'a', 'b', 'b']})\n",
|
|
"to = TabularPandas(df, cat, 'a', splits=[[0,1,2],[3,4]], y_names='b')\n",
|
|
"test_series(to.vocab, ['a', 'b'])\n",
|
|
"test_series(to['b'], [0,1,0,1,1])\n",
|
|
"to2 = to.procs.decode(to)\n",
|
|
"test_series(to2['b'], ['a', 'b', 'a', 'b', 'b'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "47daf9db",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipymini_23754/3600165379.py:3: Pandas4Warning: Constructing a Categorical with a dtype and values containing non-null entries not in that dtype's categories is deprecated and will raise in a future version.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" return pd.Categorical(c, categories=voc[c.name][add:]).codes+add\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"cat = Categorify()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,2], 'b': ['a', 'b', 'a', 'b', 'b']})\n",
|
|
"to = TabularPandas(df, cat, 'a', splits=[[0,1,2],[3,4]], y_names='b')\n",
|
|
"test_series(to.vocab, ['a', 'b'])\n",
|
|
"test_series(to['b'], [0,1,0,1,1])\n",
|
|
"to2 = to.procs.decode(to)\n",
|
|
"test_series(to2['b'], ['a', 'b', 'a', 'b', 'b'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "1b6bed8e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipymini_23754/3600165379.py:3: Pandas4Warning: Constructing a Categorical with a dtype and values containing non-null entries not in that dtype's categories is deprecated and will raise in a future version.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" return pd.Categorical(c, categories=voc[c.name][add:]).codes+add\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#test with targets and train\n",
|
|
"cat = Categorify()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,2], 'b': ['a', 'b', 'a', 'c', 'b']})\n",
|
|
"to = TabularPandas(df, cat, 'a', splits=[[0,1,2],[3,4]], y_names='b')\n",
|
|
"test_series(to.vocab, ['a', 'b'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "87ad8404",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#test to ensure no copies of the dataframe are stored\n",
|
|
"cat = Categorify()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,4]})\n",
|
|
"to = TabularPandas(df, cat, cont_names='a', splits=[[0,1,2],[3,4]])\n",
|
|
"test_eq(hasattr(to.categorify, 'to'), False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c786be37",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| exporti\n",
|
|
"@Normalize\n",
|
|
"def setups(self, to:Tabular):\n",
|
|
" store_attr(but='to', means=dict(getattr(to, 'train', to).conts.mean()),\n",
|
|
" stds=dict(getattr(to, 'train', to).conts.std(ddof=0)+1e-7))\n",
|
|
" return self(to)\n",
|
|
"\n",
|
|
"@Normalize\n",
|
|
"def encodes(self, to:Tabular):\n",
|
|
" to.conts = (to.conts-self.means) / self.stds\n",
|
|
" return to\n",
|
|
"\n",
|
|
"@Normalize\n",
|
|
"def decodes(self, to:Tabular):\n",
|
|
" to.conts = (to.conts*self.stds ) + self.means\n",
|
|
" return to"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "242a0cd4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"norm = Normalize()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,4]})\n",
|
|
"to = TabularPandas(df, norm, cont_names='a')\n",
|
|
"x = np.array([0,1,2,3,4])\n",
|
|
"m,s = x.mean(),x.std()\n",
|
|
"test_eq(norm.means['a'], m)\n",
|
|
"test_close(norm.stds['a'], s)\n",
|
|
"test_close(to['a'].values, (x-m)/s)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "87cb469f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"df1 = pd.DataFrame({'a':[5,6,7]})\n",
|
|
"to1 = to.new(df1)\n",
|
|
"to1.process()\n",
|
|
"test_close(to1['a'].values, (np.array([5,6,7])-m)/s)\n",
|
|
"to2 = norm.decode(to1)\n",
|
|
"test_close(to2['a'].values, [5,6,7])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3b539113",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"norm = Normalize()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,4]})\n",
|
|
"to = TabularPandas(df, norm, cont_names='a', splits=[[0,1,2],[3,4]])\n",
|
|
"x = np.array([0,1,2])\n",
|
|
"m,s = x.mean(),x.std()\n",
|
|
"test_eq(norm.means['a'], m)\n",
|
|
"test_close(norm.stds['a'], s)\n",
|
|
"test_close(to['a'].values, (np.array([0,1,2,3,4])-m)/s)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8dd74ab5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"norm = Normalize()\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,3,4]})\n",
|
|
"to = TabularPandas(df, norm, cont_names='a', splits=[[0,1,2],[3,4]])\n",
|
|
"test_eq(hasattr(to.procs.normalize, 'to'), False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "76a90230",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class FillStrategy:\n",
|
|
" \"Namespace containing the various filling strategies.\"\n",
|
|
" def median (c,fill): return c.median()\n",
|
|
" def constant(c,fill): return fill\n",
|
|
" def mode (c,fill): return c.dropna().value_counts().idxmax()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b18e77f9",
|
|
"metadata": {},
|
|
"source": [
|
|
"Currently, filling with the `median`, a `constant`, and the `mode` are supported."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e67b4128",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class FillMissing(TabularProc):\n",
|
|
" \"Fill the missing values in continuous columns.\"\n",
|
|
" def __init__(self, fill_strategy=FillStrategy.median, add_col=True, fill_vals=None):\n",
|
|
" if fill_vals is None: fill_vals = defaultdict(int)\n",
|
|
" store_attr()\n",
|
|
"\n",
|
|
" def setups(self, to):\n",
|
|
" missing = pd.isnull(to.conts).any()\n",
|
|
" store_attr(but='to', na_dict={n:self.fill_strategy(to[n], self.fill_vals[n])\n",
|
|
" for n in missing[missing].keys()})\n",
|
|
" self.fill_strategy = self.fill_strategy.__name__\n",
|
|
"\n",
|
|
" def encodes(self, to):\n",
|
|
" missing = pd.isnull(to.conts)\n",
|
|
" for n in missing.any()[missing.any()].keys():\n",
|
|
" assert n in self.na_dict, f\"nan values in `{n}` but not in setup training set\"\n",
|
|
" for n in self.na_dict.keys():\n",
|
|
" to[n] = to[n].fillna(self.na_dict[n])\n",
|
|
" if self.add_col:\n",
|
|
" to.loc[:,n+'_na'] = missing[n]\n",
|
|
" if n+'_na' not in to.cat_names: to.cat_names.append(n+'_na')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c3778c41",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/markdown": [
|
|
"---\n",
|
|
"\n",
|
|
"[source](https://github.com/fastai/fastai/blob/main/fastai/tabular/core.py#L297){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
|
|
"\n",
|
|
"### FillMissing\n",
|
|
"\n",
|
|
"```python\n",
|
|
"\n",
|
|
"def FillMissing(\n",
|
|
" fill_strategy:function=median, add_col:bool=True, fill_vals:NoneType=None\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Fill the missing values in continuous columns.*"
|
|
],
|
|
"text/plain": [
|
|
"```python\n",
|
|
"\n",
|
|
"def FillMissing(\n",
|
|
" fill_strategy:function=median, add_col:bool=True, fill_vals:NoneType=None\n",
|
|
"):\n",
|
|
"\n",
|
|
"\n",
|
|
"```\n",
|
|
"\n",
|
|
"*Fill the missing values in continuous columns.*"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"show_doc(FillMissing, title_level=3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0a2c1296",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"fill1,fill2,fill3 = (FillMissing(fill_strategy=s)\n",
|
|
" for s in [FillStrategy.median, FillStrategy.constant, FillStrategy.mode])\n",
|
|
"df = pd.DataFrame({'a':[0,1,np.nan,1,2,3,4]})\n",
|
|
"df1 = df.copy(); df2 = df.copy()\n",
|
|
"tos = (TabularPandas(df, fill1, cont_names='a'),\n",
|
|
" TabularPandas(df1, fill2, cont_names='a'),\n",
|
|
" TabularPandas(df2, fill3, cont_names='a'))\n",
|
|
"test_eq(fill1.na_dict, {'a': 1.5})\n",
|
|
"test_eq(fill2.na_dict, {'a': 0})\n",
|
|
"test_eq(fill3.na_dict, {'a': 1.0})\n",
|
|
"\n",
|
|
"for t in tos: test_eq(t.cat_names, ['a_na'])\n",
|
|
"\n",
|
|
"for to_,v in zip(tos, [1.5, 0., 1.]):\n",
|
|
" test_eq(to_['a'].values, np.array([0, 1, v, 1, 2, 3, 4]))\n",
|
|
" test_eq(to_['a_na'].values, np.array([0, 0, 1, 0, 0, 0, 0]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bf0b9e5a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"fill = FillMissing()\n",
|
|
"df = pd.DataFrame({'a':[0,1,np.nan,1,2,3,4], 'b': [0,1,2,3,4,5,6]})\n",
|
|
"to = TabularPandas(df, fill, cont_names=['a', 'b'])\n",
|
|
"test_eq(fill.na_dict, {'a': 1.5})\n",
|
|
"test_eq(to.cat_names, ['a_na'])\n",
|
|
"test_eq(to['a'].values, np.array([0, 1, 1.5, 1, 2, 3, 4]))\n",
|
|
"test_eq(to['a_na'].values, np.array([0, 0, 1, 0, 0, 0, 0]))\n",
|
|
"test_eq(to['b'].values, np.array([0,1,2,3,4,5,6]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "65c7be5e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"fill = FillMissing()\n",
|
|
"df = pd.DataFrame({'a':[0,1,np.nan,1,2,3,4], 'b': [0,1,2,3,4,5,6]})\n",
|
|
"to = TabularPandas(df, fill, cont_names=['a', 'b'])\n",
|
|
"test_eq(hasattr(to.procs.fill_missing, 'to'), False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "46e09f6c",
|
|
"metadata": {},
|
|
"source": [
|
|
"## TabularPandas Pipelines -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d15969b2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"procs = [Normalize, Categorify, FillMissing, noop]\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,1,np.nan,1,2,3,4]})\n",
|
|
"to = TabularPandas(df, procs, cat_names='a', cont_names='b')\n",
|
|
"\n",
|
|
"#Test setup and apply on df_main\n",
|
|
"test_series(to.cat_names, ['a', 'b_na'])\n",
|
|
"test_series(to['a'], [1,2,3,2,2,3,1])\n",
|
|
"test_series(to['b_na'], [1,1,2,1,1,1,1])\n",
|
|
"x = np.array([0,1,1.5,1,2,3,4])\n",
|
|
"m,s = x.mean(),x.std()\n",
|
|
"test_close(to['b'].values, (x-m)/s)\n",
|
|
"test_eq(to.classes, {'a': ['#na#',0,1,2], 'b_na': ['#na#',False,True]})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fc5497b0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#Test apply on y_names\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,1,np.nan,1,2,3,4], 'c': ['b','a','b','a','a','b','a']})\n",
|
|
"to = TabularPandas(df, procs, 'a', 'b', y_names='c')\n",
|
|
"\n",
|
|
"test_series(to.cat_names, ['a', 'b_na'])\n",
|
|
"test_series(to['a'], [1,2,3,2,2,3,1])\n",
|
|
"test_series(to['b_na'], [1,1,2,1,1,1,1])\n",
|
|
"test_series(to['c'], [1,0,1,0,0,1,0])\n",
|
|
"x = np.array([0,1,1.5,1,2,3,4])\n",
|
|
"m,s = x.mean(),x.std()\n",
|
|
"test_close(to['b'].values, (x-m)/s)\n",
|
|
"test_eq(to.classes, {'a': ['#na#',0,1,2], 'b_na': ['#na#',False,True]})\n",
|
|
"test_eq(to.vocab, ['a','b'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "02ee4f62",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,1,np.nan,1,2,3,4], 'c': ['b','a','b','a','a','b','a']})\n",
|
|
"to = TabularPandas(df, procs, 'a', 'b', y_names='c')\n",
|
|
"\n",
|
|
"test_series(to.cat_names, ['a', 'b_na'])\n",
|
|
"test_series(to['a'], [1,2,3,2,2,3,1])\n",
|
|
"test_eq(df.a.dtype, np.int64 if sys.platform == \"win32\" else int)\n",
|
|
"test_series(to['b_na'], [1,1,2,1,1,1,1])\n",
|
|
"test_series(to['c'], [1,0,1,0,0,1,0])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d2884d57",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/var/folders/51/b2_szf2945n072c0vj2cyty40000gn/T/ipymini_23754/3600165379.py:3: Pandas4Warning: Constructing a Categorical with a dtype and values containing non-null entries not in that dtype's categories is deprecated and will raise in a future version.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" return pd.Categorical(c, categories=voc[c.name][add:]).codes+add\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"df = pd.DataFrame({'a':[0,1,2,1,1,2,0], 'b':[0,np.nan,1,1,2,3,4], 'c': ['b','a','b','a','a','b','a']})\n",
|
|
"to = TabularPandas(df, procs, cat_names='a', cont_names='b', y_names='c', splits=[[0,1,4,6], [2,3,5]])\n",
|
|
"\n",
|
|
"test_series(to.cat_names, ['a', 'b_na'])\n",
|
|
"test_series(to['a'], [1,2,2,1,0,2,0])\n",
|
|
"test_eq(df.a.dtype, np.int64 if sys.platform == \"win32\" else int)\n",
|
|
"test_series(to['b_na'], [1,2,1,1,1,1,1])\n",
|
|
"test_series(to['c'], [1,0,0,0,1,0,1])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9be310e6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def _maybe_expand(o): return o[:,None] if o.ndim==1 else o"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "774acf79",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class ReadTabBatch(ItemTransform):\n",
|
|
" \"Transform `TabularPandas` values into a `Tensor` with the ability to decode\"\n",
|
|
" def __init__(self, to): self.to = to.new_empty()\n",
|
|
"\n",
|
|
" def encodes(self, to):\n",
|
|
" if not to.with_cont: res = (tensor(to.cats).long(),)\n",
|
|
" else: res = (tensor(to.cats).long(),tensor(to.conts).float())\n",
|
|
" ys = [n for n in to.y_names if n in to.items.columns]\n",
|
|
" if len(ys) == len(to.y_names): res = res + (tensor(to.targ),)\n",
|
|
" if to.device is not None: res = to_device(res, to.device)\n",
|
|
" return res\n",
|
|
"\n",
|
|
" def decodes(self, o):\n",
|
|
" o = [_maybe_expand(o_) for o_ in to_np(o) if o_.size != 0]\n",
|
|
" vals = np.concatenate(o, axis=1)\n",
|
|
" try: df = pd.DataFrame(vals, columns=self.to.all_col_names)\n",
|
|
" except: df = pd.DataFrame(vals, columns=self.to.x_names)\n",
|
|
" to = self.to.new(df)\n",
|
|
" return to"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "063a6e8e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"@dispatch\n",
|
|
"def show_batch(x: Tabular, y, its, max_n=10, ctxs=None):\n",
|
|
" x.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bcf9ce05",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"@delegates()\n",
|
|
"class TabDataLoader(TfmdDL):\n",
|
|
" \"A transformed `DataLoader` for Tabular data\"\n",
|
|
" def __init__(self, dataset, bs=16, shuffle=False, after_batch=None, num_workers=0, **kwargs):\n",
|
|
" if after_batch is None: after_batch = L(TransformBlock().batch_tfms)+ReadTabBatch(dataset)\n",
|
|
" super().__init__(dataset, bs=bs, shuffle=shuffle, after_batch=after_batch, num_workers=num_workers, **kwargs)\n",
|
|
"\n",
|
|
" def create_item(self, s): return self.dataset.iloc[s or 0]\n",
|
|
" def create_batch(self, b): return self.dataset.iloc[b]\n",
|
|
" def do_item(self, s): return 0 if s is None else s\n",
|
|
"\n",
|
|
"TabularPandas._dl_type = TabDataLoader"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "253b8d64",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"@delegates()\n",
|
|
"class TabWeightedDL(TabDataLoader):\n",
|
|
" \"A transformed `DataLoader` for Tabular Weighted data\"\n",
|
|
" def __init__(self, dataset, bs=16, wgts=None, shuffle=False, after_batch=None, num_workers=0, **kwargs):\n",
|
|
" wgts = np.array([1.]*len(dataset) if wgts is None else wgts)\n",
|
|
" self.wgts = wgts / wgts.sum()\n",
|
|
" super().__init__(dataset, bs=bs, shuffle=shuffle, after_batch=after_batch, num_workers=num_workers, **kwargs)\n",
|
|
" self.idxs = self.get_idxs()\n",
|
|
"\n",
|
|
" def get_idxs(self):\n",
|
|
" if self.n == 0: return []\n",
|
|
" if not self.shuffle: return super().get_idxs()\n",
|
|
" return list(np.random.choice(self.n, self.n, p=self.wgts))\n",
|
|
"\n",
|
|
"TabularPandas._dl_type = TabWeightedDL"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "62c9fd30",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Integration example\n",
|
|
"\n",
|
|
"For a more in-depth explanation, see the [tabular tutorial](http://docs.fast.ai/tutorial.tabular.html)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "774aa6d2",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>sex</th>\n",
|
|
" <th>capital-gain</th>\n",
|
|
" <th>capital-loss</th>\n",
|
|
" <th>hours-per-week</th>\n",
|
|
" <th>native-country</th>\n",
|
|
" <th>salary</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>49</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>101320</td>\n",
|
|
" <td>Assoc-acdm</td>\n",
|
|
" <td>12.0</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Wife</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1902</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>44</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>236746</td>\n",
|
|
" <td>Masters</td>\n",
|
|
" <td>14.0</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>10520</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>45</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>38</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>96185</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>32</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>38</td>\n",
|
|
" <td>Self-emp-inc</td>\n",
|
|
" <td>112847</td>\n",
|
|
" <td>Prof-school</td>\n",
|
|
" <td>15.0</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>42</td>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>82297</td>\n",
|
|
" <td>7th-8th</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Wife</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>50</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" age workclass fnlwgt ... hours-per-week native-country salary\n",
|
|
"0 49 Private 101320 ... 40 United-States >=50k\n",
|
|
"1 44 Private 236746 ... 45 United-States >=50k\n",
|
|
"2 38 Private 96185 ... 32 United-States <50k\n",
|
|
"3 38 Self-emp-inc 112847 ... 40 United-States >=50k\n",
|
|
"4 42 Self-emp-not-inc 82297 ... 50 United-States <50k\n",
|
|
"\n",
|
|
"[5 rows x 15 columns]"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"path = untar_data(URLs.ADULT_SAMPLE)\n",
|
|
"df = pd.read_csv(path/'adult.csv')\n",
|
|
"df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()\n",
|
|
"df_test.drop('salary', axis=1, inplace=True)\n",
|
|
"df_main.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "21345619",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']\n",
|
|
"cont_names = ['age', 'fnlwgt', 'education-num']\n",
|
|
"procs = [Categorify, FillMissing, Normalize]\n",
|
|
"splits = RandomSplitter()(range_of(df_main))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5e2ff96b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"to = TabularPandas(df_main, procs, cat_names, cont_names, y_names=\"salary\", splits=splits)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "57eb78ab",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/Users/jhoward/aai-ws/fastai/fastai/torch_core.py:154: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/utils/tensor_numpy.cpp:212.)\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
" else as_tensor(x.values, **kwargs) if isinstance(x, (pd.Series, pd.DataFrame))\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>education-num_na</th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>salary</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>State-gov</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>23.000000</td>\n",
|
|
" <td>287987.999279</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>State-gov</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Tech-support</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>32.000000</td>\n",
|
|
" <td>131587.998019</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>58.000000</td>\n",
|
|
" <td>183810.000021</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>Self-emp-inc</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Sales</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>83.999998</td>\n",
|
|
" <td>172906.999298</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>Local-gov</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Sales</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>31.000000</td>\n",
|
|
" <td>128016.002079</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>22.000001</td>\n",
|
|
" <td>58915.997617</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>?</td>\n",
|
|
" <td>11th</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>?</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>26.000000</td>\n",
|
|
" <td>176966.999519</td>\n",
|
|
" <td>7.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Other-relative</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>23.999999</td>\n",
|
|
" <td>86744.998003</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>38.000000</td>\n",
|
|
" <td>123833.001583</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>11th</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Farming-fishing</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>35.000000</td>\n",
|
|
" <td>168322.000103</td>\n",
|
|
" <td>7.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"dls = to.dataloaders()\n",
|
|
"dls.valid.show_batch()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b44bbe41",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>education-num_na</th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>salary</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>3564</th>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>50.0</td>\n",
|
|
" <td>124793.0</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5802</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>30.0</td>\n",
|
|
" <td>205204.0</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1087</th>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Farming-fishing</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>55.0</td>\n",
|
|
" <td>149168.0</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1239</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>25.0</td>\n",
|
|
" <td>104193.0</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2234</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Handlers-cleaners</td>\n",
|
|
" <td>Other-relative</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>22.0</td>\n",
|
|
" <td>361138.0</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8032</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Sales</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>54.0</td>\n",
|
|
" <td>391016.0</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4364</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>45.0</td>\n",
|
|
" <td>380922.0</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5404</th>\n",
|
|
" <td>Self-emp-inc</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Farming-fishing</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>60.0</td>\n",
|
|
" <td>160062.0</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>693</th>\n",
|
|
" <td>State-gov</td>\n",
|
|
" <td>Masters</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>27.0</td>\n",
|
|
" <td>315640.0</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2684</th>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Farming-fishing</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>84.0</td>\n",
|
|
" <td>155057.0</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"to.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "d553e81a",
|
|
"metadata": {},
|
|
"source": [
|
|
"We can decode any set of transformed data by calling `to.decode_row` with our raw data:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2583c751",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"age 50.0\n",
|
|
"workclass Self-emp-not-inc\n",
|
|
"fnlwgt 124793.0\n",
|
|
"education HS-grad\n",
|
|
"education-num 9.0\n",
|
|
"marital-status Married-civ-spouse\n",
|
|
"occupation Craft-repair\n",
|
|
"relationship Husband\n",
|
|
"race White\n",
|
|
"sex Male\n",
|
|
"capital-gain 0\n",
|
|
"capital-loss 0\n",
|
|
"hours-per-week 30\n",
|
|
"native-country United-States\n",
|
|
"salary <50k\n",
|
|
"education-num_na False\n",
|
|
"Name: 3564, dtype: object"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"row = to.items.iloc[0]\n",
|
|
"to.decode_row(row)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f3c28624",
|
|
"metadata": {},
|
|
"source": [
|
|
"We can make new test datasets based on the training data with the `to.new()`\n",
|
|
"\n",
|
|
":::{.callout-note}\n",
|
|
"\n",
|
|
"Since machine learning models can't magically understand categories it was never trained on, the data should reflect this. If there are different missing values in your test data you should address this before training\n",
|
|
"\n",
|
|
":::"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "aeaea7ed",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>sex</th>\n",
|
|
" <th>capital-gain</th>\n",
|
|
" <th>capital-loss</th>\n",
|
|
" <th>hours-per-week</th>\n",
|
|
" <th>native-country</th>\n",
|
|
" <th>education-num_na</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>10000</th>\n",
|
|
" <td>0.470185</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>1.346422</td>\n",
|
|
" <td>10</td>\n",
|
|
" <td>1.165893</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>Philippines</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>10001</th>\n",
|
|
" <td>-0.923825</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>1.258980</td>\n",
|
|
" <td>12</td>\n",
|
|
" <td>-0.424752</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>15</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>10002</th>\n",
|
|
" <td>1.057137</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>0.150986</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>-1.220074</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>9</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>37</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>10003</th>\n",
|
|
" <td>0.543554</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>-0.284206</td>\n",
|
|
" <td>12</td>\n",
|
|
" <td>-0.424752</td>\n",
|
|
" <td>7</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>43</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>10004</th>\n",
|
|
" <td>0.763661</td>\n",
|
|
" <td>6</td>\n",
|
|
" <td>1.449451</td>\n",
|
|
" <td>9</td>\n",
|
|
" <td>0.370571</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>60</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" age workclass ... native-country education-num_na\n",
|
|
"10000 0.470185 5 ... Philippines 1\n",
|
|
"10001 -0.923825 5 ... United-States 1\n",
|
|
"10002 1.057137 5 ... United-States 1\n",
|
|
"10003 0.543554 5 ... United-States 1\n",
|
|
"10004 0.763661 6 ... United-States 1\n",
|
|
"\n",
|
|
"[5 rows x 15 columns]"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"to_tst = to.new(df_test)\n",
|
|
"to_tst.process()\n",
|
|
"to_tst.items.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8d7c445c",
|
|
"metadata": {},
|
|
"source": [
|
|
"We can then convert it to a `DataLoader`:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6a85b936",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>education-num_na</th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>45.0</td>\n",
|
|
" <td>338104.995686</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Transport-moving</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>Other</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>26.0</td>\n",
|
|
" <td>328662.994908</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>11th</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>53.0</td>\n",
|
|
" <td>209021.999974</td>\n",
|
|
" <td>7.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Widowed</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>46.0</td>\n",
|
|
" <td>162029.999608</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>Self-emp-inc</td>\n",
|
|
" <td>Assoc-voc</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>49.0</td>\n",
|
|
" <td>349229.997552</td>\n",
|
|
" <td>11.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>Local-gov</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>34.0</td>\n",
|
|
" <td>124826.999041</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>Self-emp-inc</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Sales</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>53.0</td>\n",
|
|
" <td>290640.000020</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Sales</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>19.0</td>\n",
|
|
" <td>106273.001990</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Protective-serv</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>72.0</td>\n",
|
|
" <td>53684.003204</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Sales</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>20.0</td>\n",
|
|
" <td>505979.987402</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"tst_dl = dls.valid.new(to_tst)\n",
|
|
"tst_dl.show_batch()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3473df5e",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>education-num_na</th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>salary</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Widowed</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>58.000000</td>\n",
|
|
" <td>178644.000556</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Handlers-cleaners</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>21.000000</td>\n",
|
|
" <td>131811.002389</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>10th</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>#na#</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>16.999999</td>\n",
|
|
" <td>294485.002377</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Widowed</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>66.000001</td>\n",
|
|
" <td>98836.997894</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>28.000000</td>\n",
|
|
" <td>420054.007486</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>Federal-gov</td>\n",
|
|
" <td>Assoc-voc</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>43.000000</td>\n",
|
|
" <td>92774.997808</td>\n",
|
|
" <td>11.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>?</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>?</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>19.000000</td>\n",
|
|
" <td>234518.999208</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>35.000000</td>\n",
|
|
" <td>186009.000006</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>Federal-gov</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>25.000000</td>\n",
|
|
" <td>144258.998968</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>?</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>?</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>20.000000</td>\n",
|
|
" <td>117788.996828</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td><50k</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Create a TabWeightedDL\n",
|
|
"train_ds = to.train\n",
|
|
"weights = np.random.random(len(train_ds))\n",
|
|
"train_dl = TabWeightedDL(train_ds, wgts=weights, bs=64, shuffle=True)\n",
|
|
"\n",
|
|
"train_dl.show_batch()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "49343d89",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"batch = next(iter(train_dl))\n",
|
|
"\n",
|
|
"x, y = batch[0].shape\n",
|
|
"\n",
|
|
"test_eq(x, 64)\n",
|
|
"test_eq(y, 7)\n",
|
|
"\n",
|
|
"assert hasattr(train_dl, 'wgts'), \"Weights attribute missing in DataLoader\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "2e8f4cfd",
|
|
"metadata": {},
|
|
"source": [
|
|
"TabDataLoader's create_item method"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "665057bd",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"age 35\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Name: 0, dtype: int8\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"df = pd.DataFrame([{'age': 35}])\n",
|
|
"to = TabularPandas(df)\n",
|
|
"dls = to.dataloaders()\n",
|
|
"print(dls.create_item(0))\n",
|
|
"# test_eq(dls.create_item(0).items.to_dict(), {'age': 0.5330614747286777, 'workclass': 5, 'fnlwgt': -0.26305443080666174, 'education': 10, 'education-num': 1.169790230219763, 'marital-status': 1, 'occupation': 13, 'relationship': 5, 'race': 3, 'sex': ' Female', 'capital-gain': 0, 'capital-loss': 0, 'hours-per-week': 35, 'native-country': 'United-States', 'salary': 1, 'education-num_na': 1})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "b2f17bbd",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Other target types"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "be0b3f63",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Multi-label categories"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "035e521c",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### one-hot encoded label"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3ce5308e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def _mock_multi_label(df):\n",
|
|
" sal,sex,white = [],[],[]\n",
|
|
" for row in df.itertuples():\n",
|
|
" sal.append(row.salary == '>=50k')\n",
|
|
" sex.append(row.sex == ' Male')\n",
|
|
" white.append(row.race == ' White')\n",
|
|
" df['salary'] = np.array(sal)\n",
|
|
" df['male'] = np.array(sex)\n",
|
|
" df['white'] = np.array(white)\n",
|
|
" return df"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c81c8277",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"path = untar_data(URLs.ADULT_SAMPLE)\n",
|
|
"df = pd.read_csv(path/'adult.csv')\n",
|
|
"df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()\n",
|
|
"df_main = _mock_multi_label(df_main)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "17c80776",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>sex</th>\n",
|
|
" <th>capital-gain</th>\n",
|
|
" <th>capital-loss</th>\n",
|
|
" <th>hours-per-week</th>\n",
|
|
" <th>native-country</th>\n",
|
|
" <th>salary</th>\n",
|
|
" <th>male</th>\n",
|
|
" <th>white</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>49</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>101320</td>\n",
|
|
" <td>Assoc-acdm</td>\n",
|
|
" <td>12.0</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Wife</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1902</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>44</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>236746</td>\n",
|
|
" <td>Masters</td>\n",
|
|
" <td>14.0</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>10520</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>45</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>38</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>96185</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>32</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>38</td>\n",
|
|
" <td>Self-emp-inc</td>\n",
|
|
" <td>112847</td>\n",
|
|
" <td>Prof-school</td>\n",
|
|
" <td>15.0</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>42</td>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>82297</td>\n",
|
|
" <td>7th-8th</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Wife</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>50</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" age workclass fnlwgt ... salary male white\n",
|
|
"0 49 Private 101320 ... True False True\n",
|
|
"1 44 Private 236746 ... True True True\n",
|
|
"2 38 Private 96185 ... False False False\n",
|
|
"3 38 Self-emp-inc 112847 ... True True False\n",
|
|
"4 42 Self-emp-not-inc 82297 ... False False False\n",
|
|
"\n",
|
|
"[5 rows x 17 columns]"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df_main.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "67f710b2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| exporti\n",
|
|
"@EncodedMultiCategorize\n",
|
|
"def setups(self, to:Tabular):\n",
|
|
" self.c = len(self.vocab)\n",
|
|
" return self(to)\n",
|
|
"\n",
|
|
"@EncodedMultiCategorize\n",
|
|
"def encodes(self, to:Tabular): return to\n",
|
|
"\n",
|
|
"@EncodedMultiCategorize\n",
|
|
"def decodes(self, to:Tabular):\n",
|
|
" to.transform(to.y_names, lambda c: c==1)\n",
|
|
" return to"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "947ce02a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']\n",
|
|
"cont_names = ['age', 'fnlwgt', 'education-num']\n",
|
|
"procs = [Categorify, FillMissing, Normalize]\n",
|
|
"splits = RandomSplitter()(range_of(df_main))\n",
|
|
"y_names=[\"salary\", \"male\", \"white\"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b7e076f4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"CPU times: user 30.8 ms, sys: 774 us, total: 31.6 ms\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Wall time: 31.4 ms\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"%time to = TabularPandas(df_main, procs, cat_names, cont_names, y_names=y_names, y_block=MultiCategoryBlock(encoded=True, vocab=y_names), splits=splits)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "04eecd3c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>education-num_na</th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>salary</th>\n",
|
|
" <th>male</th>\n",
|
|
" <th>white</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Machine-op-inspct</td>\n",
|
|
" <td>Wife</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>35.000000</td>\n",
|
|
" <td>110668.001907</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>51.000000</td>\n",
|
|
" <td>33303.995791</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>37.000000</td>\n",
|
|
" <td>38468.006454</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>?</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>?</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>63.999999</td>\n",
|
|
" <td>108082.000741</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>38.000000</td>\n",
|
|
" <td>187747.999948</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Tech-support</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>28.000000</td>\n",
|
|
" <td>375313.002386</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>#na#</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>36.000000</td>\n",
|
|
" <td>130199.998135</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>11th</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>36.000000</td>\n",
|
|
" <td>123151.001427</td>\n",
|
|
" <td>7.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Adm-clerical</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>32.000000</td>\n",
|
|
" <td>107218.002435</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>11th</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>32.000000</td>\n",
|
|
" <td>185732.000241</td>\n",
|
|
" <td>7.0</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"dls = to.dataloaders()\n",
|
|
"dls.valid.show_batch()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "31084bed",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Not one-hot encoded"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6d5344d1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def _mock_multi_label(df):\n",
|
|
" targ = []\n",
|
|
" for row in df.itertuples():\n",
|
|
" labels = []\n",
|
|
" if row.salary == '>=50k': labels.append('>50k')\n",
|
|
" if row.sex == ' Male': labels.append('male')\n",
|
|
" if row.race == ' White': labels.append('white')\n",
|
|
" targ.append(' '.join(labels))\n",
|
|
" df['target'] = np.array(targ)\n",
|
|
" return df"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "deaa4ec3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"path = untar_data(URLs.ADULT_SAMPLE)\n",
|
|
"df = pd.read_csv(path/'adult.csv')\n",
|
|
"df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()\n",
|
|
"df_main = _mock_multi_label(df_main)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "eff30354",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>age</th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>sex</th>\n",
|
|
" <th>capital-gain</th>\n",
|
|
" <th>capital-loss</th>\n",
|
|
" <th>hours-per-week</th>\n",
|
|
" <th>native-country</th>\n",
|
|
" <th>salary</th>\n",
|
|
" <th>target</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>49</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>101320</td>\n",
|
|
" <td>Assoc-acdm</td>\n",
|
|
" <td>12.0</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Wife</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1902</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" <td>>50k white</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>44</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>236746</td>\n",
|
|
" <td>Masters</td>\n",
|
|
" <td>14.0</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>10520</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>45</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" <td>>50k male white</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>38</td>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>96185</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>32</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td><50k</td>\n",
|
|
" <td></td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>38</td>\n",
|
|
" <td>Self-emp-inc</td>\n",
|
|
" <td>112847</td>\n",
|
|
" <td>Prof-school</td>\n",
|
|
" <td>15.0</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>Asian-Pac-Islander</td>\n",
|
|
" <td>Male</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>40</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td>>=50k</td>\n",
|
|
" <td>>50k male</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>42</td>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>82297</td>\n",
|
|
" <td>7th-8th</td>\n",
|
|
" <td>NaN</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Wife</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>Female</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>50</td>\n",
|
|
" <td>United-States</td>\n",
|
|
" <td><50k</td>\n",
|
|
" <td></td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" age workclass fnlwgt ... native-country salary target\n",
|
|
"0 49 Private 101320 ... United-States >=50k >50k white\n",
|
|
"1 44 Private 236746 ... United-States >=50k >50k male white\n",
|
|
"2 38 Private 96185 ... United-States <50k \n",
|
|
"3 38 Self-emp-inc 112847 ... United-States >=50k >50k male\n",
|
|
"4 42 Self-emp-not-inc 82297 ... United-States <50k \n",
|
|
"\n",
|
|
"[5 rows x 16 columns]"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"df_main.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d9766eec",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"@MultiCategorize\n",
|
|
"def encodes(self, to:Tabular):\n",
|
|
" #to.transform(to.y_names, partial(_apply_cats, {n: self.vocab for n in to.y_names}, 0))\n",
|
|
" return to\n",
|
|
"\n",
|
|
"@MultiCategorize\n",
|
|
"def decodes(self, to:Tabular):\n",
|
|
" #to.transform(to.y_names, partial(_decode_cats, {n: self.vocab for n in to.y_names}))\n",
|
|
" return to"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fd5f38e5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']\n",
|
|
"cont_names = ['age', 'fnlwgt', 'education-num']\n",
|
|
"procs = [Categorify, FillMissing, Normalize]\n",
|
|
"splits = RandomSplitter()(range_of(df_main))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3cd349cb",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"CPU times: user 10.5 ms, sys: 201 us, total: 10.7 ms\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Wall time: 10.6 ms\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"%time to = TabularPandas(df_main, procs, cat_names, cont_names, y_names=\"target\", y_block=MultiCategoryBlock(), splits=splits)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cdd1400c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"['-', '_', 'a', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y']"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"to.procs[2].vocab"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "ededa205",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Regression"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d6b3e995",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| exporti\n",
|
|
"@RegressionSetup\n",
|
|
"def setups(self, to:Tabular):\n",
|
|
" if self.c is not None: return\n",
|
|
" self.c = len(to.y_names)\n",
|
|
" return to\n",
|
|
"\n",
|
|
"@RegressionSetup\n",
|
|
"def encodes(self, to:Tabular): return to\n",
|
|
"\n",
|
|
"@RegressionSetup\n",
|
|
"def decodes(self, to:Tabular): return to"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0ab00801",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"path = untar_data(URLs.ADULT_SAMPLE)\n",
|
|
"df = pd.read_csv(path/'adult.csv')\n",
|
|
"df_main,df_test = df.iloc[:10000].copy(),df.iloc[10000:].copy()\n",
|
|
"df_main = _mock_multi_label(df_main)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "36253610",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']\n",
|
|
"cont_names = ['fnlwgt', 'education-num']\n",
|
|
"procs = [Categorify, FillMissing, Normalize]\n",
|
|
"splits = RandomSplitter()(range_of(df_main))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7d05a460",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"CPU times: user 21.8 ms, sys: 969 us, total: 22.7 ms\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Wall time: 21.9 ms\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"%time to = TabularPandas(df_main, procs, cat_names, cont_names, y_names='age', splits=splits)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "76c0315b",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"{'fnlwgt': np.float64(192511.077125),\n",
|
|
" 'education-num': np.float64(10.076749801635742)}"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"to.procs[-1].means"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3ebfddc8",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>workclass</th>\n",
|
|
" <th>education</th>\n",
|
|
" <th>marital-status</th>\n",
|
|
" <th>occupation</th>\n",
|
|
" <th>relationship</th>\n",
|
|
" <th>race</th>\n",
|
|
" <th>education-num_na</th>\n",
|
|
" <th>fnlwgt</th>\n",
|
|
" <th>education-num</th>\n",
|
|
" <th>age</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>Local-gov</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Married-spouse-absent</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>216129.000726</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td>38.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Protective-serv</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>Black</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>162814.000297</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>34.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>193881.999975</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td>44.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>Local-gov</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Exec-managerial</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>144778.000789</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>43.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>HS-grad</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>304385.993937</td>\n",
|
|
" <td>9.0</td>\n",
|
|
" <td>24.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>5th-6th</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Craft-repair</td>\n",
|
|
" <td>Not-in-family</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>155621.001484</td>\n",
|
|
" <td>3.0</td>\n",
|
|
" <td>28.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>Self-emp-not-inc</td>\n",
|
|
" <td>7th-8th</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Other-service</td>\n",
|
|
" <td>Unmarried</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>385632.000374</td>\n",
|
|
" <td>4.0</td>\n",
|
|
" <td>56.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>Bachelors</td>\n",
|
|
" <td>Never-married</td>\n",
|
|
" <td>Prof-specialty</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>244365.998597</td>\n",
|
|
" <td>13.0</td>\n",
|
|
" <td>22.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>Private</td>\n",
|
|
" <td>11th</td>\n",
|
|
" <td>Divorced</td>\n",
|
|
" <td>Handlers-cleaners</td>\n",
|
|
" <td>Own-child</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>112262.998271</td>\n",
|
|
" <td>7.0</td>\n",
|
|
" <td>30.0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>Local-gov</td>\n",
|
|
" <td>Some-college</td>\n",
|
|
" <td>Married-civ-spouse</td>\n",
|
|
" <td>Protective-serv</td>\n",
|
|
" <td>Husband</td>\n",
|
|
" <td>White</td>\n",
|
|
" <td>False</td>\n",
|
|
" <td>195258.000082</td>\n",
|
|
" <td>10.0</td>\n",
|
|
" <td>41.0</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"dls = to.dataloaders()\n",
|
|
"dls.valid.show_batch()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c94a9701",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Not being used now - for multi-modal"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "181bc611",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class TensorTabular(fastuple):\n",
|
|
" def get_ctxs(self, max_n=10, **kwargs):\n",
|
|
" n_samples = min(self[0].shape[0], max_n)\n",
|
|
" df = pd.DataFrame(index = range(n_samples))\n",
|
|
" return [df.iloc[i] for i in range(n_samples)]\n",
|
|
"\n",
|
|
" def display(self, ctxs): display_df(pd.DataFrame(ctxs))\n",
|
|
"\n",
|
|
"class TabularLine(pd.Series):\n",
|
|
" \"A line of a dataframe that knows how to show itself\"\n",
|
|
" def show(self, ctx=None, **kwargs): return self if ctx is None else ctx.append(self)\n",
|
|
"\n",
|
|
"class ReadTabLine(ItemTransform):\n",
|
|
" def __init__(self, proc): self.proc = proc\n",
|
|
"\n",
|
|
" def encodes(self, row):\n",
|
|
" cats,conts = (o.map(row.__getitem__) for o in (self.proc.cat_names,self.proc.cont_names))\n",
|
|
" return TensorTabular(tensor(cats).long(),tensor(conts).float())\n",
|
|
"\n",
|
|
" def decodes(self, o):\n",
|
|
" to = TabularPandas(o, self.proc.cat_names, self.proc.cont_names, self.proc.y_names)\n",
|
|
" to = self.proc.decode(to)\n",
|
|
" return TabularLine(pd.Series({c: v for v,c in zip(to.items[0]+to.items[1], self.proc.cat_names+self.proc.cont_names)}))\n",
|
|
"\n",
|
|
"class ReadTabTarget(ItemTransform):\n",
|
|
" def __init__(self, proc): self.proc = proc\n",
|
|
" def encodes(self, row): return row[self.proc.y_names].astype(np.int64)\n",
|
|
" def decodes(self, o): return Category(self.proc.classes[self.proc.y_names][o])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "463c71e3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# tds = TfmdDS(to.items, tfms=[[ReadTabLine(proc)], ReadTabTarget(proc)])\n",
|
|
"# enc = tds[1]\n",
|
|
"# test_eq(enc[0][0], tensor([2,1]))\n",
|
|
"# test_close(enc[0][1], tensor([-0.628828]))\n",
|
|
"# test_eq(enc[1], 1)\n",
|
|
"\n",
|
|
"# dec = tds.decode(enc)\n",
|
|
"# assert isinstance(dec[0], TabularLine)\n",
|
|
"# test_close(dec[0], pd.Series({'a': 1, 'b_na': False, 'b': 1}))\n",
|
|
"# test_eq(dec[1], 'a')\n",
|
|
"\n",
|
|
"# test_stdout(lambda: print(show_at(tds, 1)), \"\"\"a 1\n",
|
|
"# b_na False\n",
|
|
"# b 1\n",
|
|
"# category a\n",
|
|
"# dtype: object\"\"\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f64980dc",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Export -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9b8563f9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"from nbdev import nbdev_export\n",
|
|
"nbdev_export()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|