wehub-resource-sync
263 lines
7.9 KiB
Plaintext
263 lines
7.9 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "23bed9de",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#| eval: false\n",
|
|
"! [ -e /content ] && pip install -Uqq fastai # upgrade fastai on colab"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b8fb8b4d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"from fastai.torch_basics import *\n",
|
|
"from fastai.tabular.core import *"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a1649af3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"from nbdev.showdoc import *"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d4d6193b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| default_exp tabular.model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "5c3f5da9",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Tabular model\n",
|
|
"\n",
|
|
"> A basic model that can be used on tabular data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cbe4b173",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Embeddings"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "393080e2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def emb_sz_rule(\n",
|
|
" n_cat:int # Cardinality of a category\n",
|
|
") -> int:\n",
|
|
" \"Rule of thumb to pick embedding size corresponding to `n_cat`\"\n",
|
|
" return min(600, round(1.6 * n_cat**0.56))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "aaf47c66",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def _one_emb_sz(classes, n, sz_dict=None):\n",
|
|
" \"Pick an embedding size for `n` depending on `classes` if not given in `sz_dict`.\"\n",
|
|
" sz_dict = ifnone(sz_dict, {})\n",
|
|
" n_cat = len(classes[n])\n",
|
|
" sz = sz_dict.get(n, int(emb_sz_rule(n_cat))) # rule of thumb\n",
|
|
" return n_cat,sz"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "061c60f8",
|
|
"metadata": {},
|
|
"source": [
|
|
"Through trial and error, this general rule takes the lower of two values:\n",
|
|
"\n",
|
|
"* A dimension space of 600\n",
|
|
"* A dimension space equal to 1.6 times the cardinality of the variable to 0.56.\n",
|
|
"\n",
|
|
"This provides a good starter for a good embedding space for your variables. For more advanced users who wish to lean into this practice, you can tweak these values to your discretion. It is not uncommon for slight adjustments to this general formula to provide more success."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "777807fb",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"def get_emb_sz(\n",
|
|
" to:Tabular|TabularPandas, \n",
|
|
" sz_dict:dict=None # Dictionary of {'class_name' : size, ...} to override default `emb_sz_rule` \n",
|
|
") -> list: # List of embedding sizes for each category\n",
|
|
" \"Get embedding size for each cat_name in `Tabular` or `TabularPandas`, or populate embedding size manually using sz_dict\"\n",
|
|
" return [_one_emb_sz(to.classes, n, sz_dict) for n in to.cat_names]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "75c40873",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class TabularModel(Module):\n",
|
|
" \"Basic model for tabular data.\"\n",
|
|
" def __init__(self, \n",
|
|
" emb_szs:list, # Sequence of (num_embeddings, embedding_dim) for each categorical variable\n",
|
|
" n_cont:int, # Number of continuous variables\n",
|
|
" out_sz:int, # Number of outputs for final `LinBnDrop` layer\n",
|
|
" layers:list, # Sequence of ints used to specify the input and output size of each `LinBnDrop` layer\n",
|
|
" ps:float|MutableSequence=None, # Sequence of dropout probabilities for `LinBnDrop`\n",
|
|
" embed_p:float=0., # Dropout probability for `Embedding` layer\n",
|
|
" y_range=None, # Low and high for `SigmoidRange` activation \n",
|
|
" use_bn:bool=True, # Use `BatchNorm1d` in `LinBnDrop` layers\n",
|
|
" bn_final:bool=False, # Use `BatchNorm1d` on final layer\n",
|
|
" bn_cont:bool=True, # Use `BatchNorm1d` on continuous variables\n",
|
|
" act_cls=nn.ReLU(inplace=True), # Activation type for `LinBnDrop` layers\n",
|
|
" lin_first:bool=True # Linear layer is first or last in `LinBnDrop` layers\n",
|
|
" ):\n",
|
|
" ps = ifnone(ps, [0]*len(layers))\n",
|
|
" if not is_listy(ps): ps = [ps]*len(layers)\n",
|
|
" self.embeds = nn.ModuleList([Embedding(ni, nf) for ni,nf in emb_szs])\n",
|
|
" self.emb_drop = nn.Dropout(embed_p)\n",
|
|
" self.bn_cont = nn.BatchNorm1d(n_cont) if bn_cont else None\n",
|
|
" n_emb = sum(e.embedding_dim for e in self.embeds)\n",
|
|
" self.n_emb,self.n_cont = n_emb,n_cont\n",
|
|
" sizes = [n_emb + n_cont] + layers + [out_sz]\n",
|
|
" actns = [act_cls for _ in range(len(sizes)-2)] + [None]\n",
|
|
" _layers = [LinBnDrop(sizes[i], sizes[i+1], bn=use_bn and (i!=len(actns)-1 or bn_final), p=p, act=a, lin_first=lin_first)\n",
|
|
" for i,(p,a) in enumerate(zip(ps+[0.],actns))]\n",
|
|
" if y_range is not None: _layers.append(SigmoidRange(*y_range))\n",
|
|
" self.layers = nn.Sequential(*_layers)\n",
|
|
"\n",
|
|
" def forward(self, x_cat, x_cont=None):\n",
|
|
" if self.n_emb != 0:\n",
|
|
" x = [e(x_cat[:,i]) for i,e in enumerate(self.embeds)]\n",
|
|
" x = torch.cat(x, 1)\n",
|
|
" x = self.emb_drop(x)\n",
|
|
" if self.n_cont != 0:\n",
|
|
" if self.bn_cont is not None: x_cont = self.bn_cont(x_cont)\n",
|
|
" x = torch.cat([x, x_cont], 1) if self.n_emb != 0 else x_cont\n",
|
|
" return self.layers(x)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "fd761044",
|
|
"metadata": {},
|
|
"source": [
|
|
"This model expects your `cat` and `cont` variables seperated. `cat` is passed through an `Embedding` layer and potential `Dropout`, while `cont` is passed though potential `BatchNorm1d`. Afterwards both are concatenated and passed through a series of `LinBnDrop`, before a final `Linear` layer corresponding to the expected outputs. "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "1d09a4dc",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"emb_szs = [(4,2), (17,8)]\n",
|
|
"m = TabularModel(emb_szs, n_cont=2, out_sz=2, layers=[200,100]).eval()\n",
|
|
"x_cat = torch.tensor([[2,12]]).long()\n",
|
|
"x_cont = torch.tensor([[0.7633, -0.1887]]).float()\n",
|
|
"out = m(x_cat, x_cont)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b7283da3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"@delegates(TabularModel.__init__)\n",
|
|
"def tabular_config(**kwargs):\n",
|
|
" \"Convenience function to easily create a config for `TabularModel`\"\n",
|
|
" return kwargs"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "70e6e59e",
|
|
"metadata": {},
|
|
"source": [
|
|
"Any direct setup of `TabularModel`'s internals should be passed through here:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "dbb0306d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"config = tabular_config(embed_p=0.6, use_bn=False); config"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "6939423f",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Export -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "46818389",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"from nbdev import nbdev_export\n",
|
|
"nbdev_export()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"jupytext": {
|
|
"split_at_heading": true
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "python3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|