wehub/fastai--fastai
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
main
fastai--fastai/nbs/13a_learner.ipynb
T
wehub-resource-sync 844360907f chore: import upstream snapshot with attribution
2026-07-13 13:21:43 +08:00

4909 lines
175 KiB
Plaintext
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "4d4e22ab",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#| eval: false\n",
"! [ -e /content ] && pip install -Uqq fastai # upgrade fastai on colab"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6a4f0e50",
"metadata": {},
"outputs": [],
"source": [
"#| default_exp learner"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f4ade127",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"from fastai.data.all import *\n",
"from fastai.optimizer import *\n",
"from fastai.callback.core import *\n",
"from contextlib import nullcontext\n",
"import cloudpickle,pickle,threading\n",
"from collections.abc import MutableSequence"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "46b3b8b4",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"from nbdev.showdoc import *"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c92ce8d1",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"_all_ = ['CancelBackwardException', 'CancelStepException','CancelFitException','CancelEpochException',\n",
" 'CancelTrainException','CancelValidException','CancelBatchException']"
]
},
{
"cell_type": "markdown",
"id": "07eb5555",
"metadata": {},
"source": [
"# Learner, Metrics, Callbacks\n",
"\n",
"> Basic class for handling the training loop"
]
},
{
"cell_type": "markdown",
"id": "27488c17",
"metadata": {},
"source": [
"You probably want to jump directly to the definition of `Learner`."
]
},
{
"cell_type": "markdown",
"id": "f897f1f2",
"metadata": {},
"source": [
"## Utils function"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bd19c703",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#For tests\n",
"from torch.utils.data import TensorDataset, DataLoader as TorchDL"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2fb896f5",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"def synth_dbunch(a=2, b=3, bs=16, n_train=10, n_valid=2, cuda=False, tfmdDL=True):\n",
" \"A simple dataset where `x` is random and `y = a*x + b` plus some noise.\"\n",
" def get_data(n):\n",
" x = torch.randn(int(bs*n))\n",
" return TensorDataset(x, a*x + b + 0.1*torch.randn(int(bs*n)))\n",
" train_ds = get_data(n_train)\n",
" valid_ds = get_data(n_valid)\n",
" device = default_device() if cuda else None\n",
" if tfmdDL:\n",
" train_dl = TfmdDL(train_ds, bs=bs, shuffle=True, num_workers=0, drop_last=True)\n",
" valid_dl = TfmdDL(valid_ds, bs=bs, num_workers=0)\n",
" else:\n",
" train_dl = TorchDL(train_ds, batch_size=bs, shuffle=True, num_workers=0, drop_last=True)\n",
" valid_dl = TorchDL(valid_ds, batch_size=bs, num_workers=0)\n",
" device = None\n",
" return DataLoaders(train_dl, valid_dl, device=device)\n",
"\n",
"class RegModel(Module):\n",
" \"A r\"\n",
" def __init__(self): self.a,self.b = nn.Parameter(torch.randn(1)),nn.Parameter(torch.randn(1))\n",
" def forward(self, x): return x*self.a + self.b"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7ccd2b79",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"defaults.lr = 1e-3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e614ffdf",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def replacing_yield(o, attr, val):\n",
" \"Context manager to temporarily replace an attribute\"\n",
" old = getattr(o,attr)\n",
" try: yield setattr(o,attr,val)\n",
" finally: setattr(o,attr,old)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "54592812",
"metadata": {},
"outputs": [],
"source": [
"class _A:\n",
" def __init__(self, a): self.a = a\n",
" @contextmanager\n",
" def a_changed(self, v): return replacing_yield(self, 'a', v)\n",
"\n",
"a = _A(42)\n",
"with a.a_changed(32):\n",
" test_eq(a.a, 32)\n",
"test_eq(a.a, 42)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e1ff5be1",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def mk_metric(m):\n",
" \"Convert `m` to an `AvgMetric`, unless it's already a `Metric`\"\n",
" if isinstance(m,type): m = m()\n",
" return m if isinstance(m, Metric) else AvgMetric(m)"
]
},
{
"cell_type": "markdown",
"id": "d98107b2",
"metadata": {},
"source": [
"See the class `Metric` below for more information."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c628b757",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def save_model(file, model, opt, with_opt=True, pickle_protocol=2, **torch_save_kwargs):\n",
" \"Save `model` to `file` along with `opt` (if available, and if `with_opt`)\"\n",
" if rank_distrib(): return # don't save if child proc\n",
" if opt is None: with_opt=False\n",
" state = get_model(model).state_dict()\n",
" if with_opt: state = {'model': state, 'opt':opt.state_dict()}\n",
" torch.save(state, file, pickle_protocol=pickle_protocol, **torch_save_kwargs)"
]
},
{
"cell_type": "markdown",
"id": "6dfca263",
"metadata": {},
"source": [
"`file` can be a `Path` object, a string or an opened file object. `pickle_protocol` and `torch_save_kwargs` is passed along to `torch.save`"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a1d380b2",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def load_model(file, model, opt, with_opt=True, device=None, strict=True, **torch_load_kwargs):\n",
" \"Load `model` from `file` along with `opt` (if available, and if `with_opt`)\"\n",
" if isinstance(device, int): device = torch.device('cuda', device)\n",
" elif device is None: device = 'cpu'\n",
" if ismin_torch(\"2.5\"):\n",
" context = torch.serialization.safe_globals([L, np.core.multiarray.scalar, np.dtype, *[getattr(np.dtypes, dt) for dt in np.dtypes.__all__]])\n",
" torch_load_kwargs.setdefault(\"weights_only\", True)\n",
" else: context = nullcontext()\n",
" with context:\n",
" state = torch.load(file, map_location=device, **torch_load_kwargs)\n",
" hasopt = set(state)=={'model', 'opt'}\n",
" model_state = state['model'] if hasopt else state\n",
" get_model(model).load_state_dict(model_state, strict=strict)\n",
" if hasopt and with_opt:\n",
" try: opt.load_state_dict(state['opt'])\n",
" except:\n",
" if with_opt: warn(\"Could not load the optimizer state.\")\n",
" elif with_opt: warn(\"Saved file doesn't contain an optimizer state.\")"
]
},
{
"cell_type": "markdown",
"id": "dc9b455e",
"metadata": {},
"source": [
"`file` can be a `Path` object, a string or an opened file object. If a `device` is passed, the model is loaded on it, otherwise it's loaded on the CPU.\n",
"\n",
"If `strict` is `True`, the file must exactly contain weights for every parameter key in `model`, if `strict` is `False`, only the keys that are in the saved model are loaded in `model`.\n",
"\n",
"You can pass in other kwargs to `torch.load` through `torch_load_kwargs`.\n",
"\n",
"As of PyTorch 2.5 and fastai 2.7.19, `load_model` uses PyTorch's safe serialization for model loading."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e0150dae",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def _try_concat(o):\n",
" try: return torch.cat(o)\n",
" except: return sum([L(o_[i,:] for i in range_of(o_)) for o_ in o], L())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "986f89f1",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"_before_epoch = [event.before_fit, event.before_epoch]\n",
"_after_epoch = [event.after_epoch, event.after_fit]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "985a1076",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class _ConstantFunc():\n",
" \"Returns a function that returns `o`\"\n",
" def __init__(self, o): self.o = o\n",
" def __call__(self, *args, **kwargs): return self.o"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d40584de",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class SkipToEpoch(Callback):\n",
" \"Skip training up to `epoch`\"\n",
" order = 70\n",
" \n",
" def __init__(self, epoch:int):\n",
" self._skip_to = epoch\n",
"\n",
" def before_epoch(self):\n",
" if self.epoch < self._skip_to:\n",
" raise CancelEpochException"
]
},
{
"cell_type": "markdown",
"id": "dd68f807",
"metadata": {},
"source": [
"## Learner -"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2b74fcb3",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"_loop = ['Start Fit', 'before_fit', 'Start Epoch Loop', 'before_epoch', 'Start Train', 'before_train',\n",
" 'Start Batch Loop', 'before_batch', 'after_pred', 'after_loss', 'before_backward', 'before_step',\n",
" 'after_step', 'after_cancel_batch', 'after_batch','End Batch Loop','End Train',\n",
" 'after_cancel_train', 'after_train', 'Start Valid', 'before_validate','Start Batch Loop',\n",
" '**CBs same as train batch**', 'End Batch Loop', 'End Valid', 'after_cancel_validate',\n",
" 'after_validate', 'End Epoch Loop', 'after_cancel_epoch', 'after_epoch', 'End Fit',\n",
" 'after_cancel_fit', 'after_fit']"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b50c27de",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class Learner(GetAttr):\n",
" _default='model'\n",
" def __init__(self,\n",
" dls:DataLoaders, # `DataLoaders` containing fastai or PyTorch `DataLoader`s\n",
" model:Callable, # PyTorch model for training or inference\n",
" loss_func:Callable|None=None, # Loss function. Defaults to `dls` loss\n",
" opt_func:Optimizer|OptimWrapper=Adam, # Optimization function for training\n",
" lr:float|slice=defaults.lr, # Default learning rate\n",
" splitter:Callable=trainable_params, # Split model into parameter groups. Defaults to one parameter group\n",
" cbs:Callback|MutableSequence|None=None, # `Callback`s to add to `Learner`\n",
" metrics:Callable|MutableSequence|None=None, # `Metric`s to calculate on validation set\n",
" path:str|Path|None=None, # Parent directory to save, load, and export models. Defaults to `dls` `path`\n",
" model_dir:str|Path='models', # Subdirectory to save and load models\n",
" wd:float|int|None=None, # Default weight decay\n",
" wd_bn_bias:bool=False, # Apply weight decay to normalization and bias parameters\n",
" train_bn:bool=True, # Train frozen normalization layers\n",
" moms:tuple=(0.95,0.85,0.95), # Default momentum for schedulers\n",
" default_cbs:bool=True # Include default `Callback`s\n",
" ):\n",
" path = Path(path) if path is not None else getattr(dls, 'path', Path('.'))\n",
" if loss_func is None:\n",
" loss_func = getattr(dls.train_ds, 'loss_func', None)\n",
" assert loss_func is not None, \"Could not infer loss function from the data, please pass a loss function.\"\n",
" self.dls,self.model = dls,model\n",
" store_attr(but='dls,model,cbs')\n",
" self.training,self.create_mbar,self.logger,self.opt,self.cbs = False,True,print,None,L()\n",
" if default_cbs: self.add_cbs(L(defaults.callbacks))\n",
" self.add_cbs(cbs)\n",
" self.lock = threading.Lock()\n",
" self(\"after_create\")\n",
"\n",
" @property\n",
" def metrics(self): return self._metrics\n",
" @metrics.setter\n",
" def metrics(self,v): self._metrics = L(v).map(mk_metric)\n",
"\n",
" def _grab_cbs(self, cb_cls): return L(cb for cb in self.cbs if isinstance(cb, cb_cls))\n",
"\n",
" def add_cbs(self, cbs):\n",
" L(cbs).map(self.add_cb)\n",
" return self\n",
"\n",
" def remove_cbs(self, cbs):\n",
" L(cbs).map(self.remove_cb)\n",
" return self\n",
"\n",
" def add_cb(self, cb):\n",
" if isinstance(cb, type): cb = cb()\n",
" cb.learn = self\n",
" setattr(self, cb.name, cb)\n",
" self.cbs.append(cb)\n",
" return self\n",
"\n",
" def remove_cb(self, cb):\n",
" if isinstance(cb, type): self.remove_cbs(self._grab_cbs(cb))\n",
" else:\n",
" cb.learn = None\n",
" if hasattr(self, cb.name): delattr(self, cb.name)\n",
" if cb in self.cbs: self.cbs.remove(cb)\n",
" return self\n",
"\n",
" @contextmanager\n",
" def added_cbs(self, cbs):\n",
" self.add_cbs(cbs)\n",
" try: yield\n",
" finally: self.remove_cbs(cbs)\n",
"\n",
" @contextmanager\n",
" def removed_cbs(self, cbs):\n",
" self.remove_cbs(cbs)\n",
" try: yield self\n",
" finally: self.add_cbs(cbs)\n",
"\n",
" def ordered_cbs(self, event): return [cb for cb in self.cbs.sorted('order') if hasattr(cb, event)]\n",
" def __call__(self, event_name): L(event_name).map(self._call_one)\n",
"\n",
" def _call_one(self, event_name):\n",
" if not hasattr(event, event_name): raise Exception(f'missing {event_name}')\n",
" for cb in self.cbs.sorted('order'): cb(event_name)\n",
"\n",
" def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)\n",
"\n",
" def create_opt(self):\n",
" if isinstance(self.opt_func, partial):\n",
" if 'lr' in self.opt_func.keywords:\n",
" self.lr = self.opt_func.keywords['lr']\n",
" if isinstance(self.opt_func, OptimWrapper):\n",
" self.opt = self.opt_func\n",
" self.opt.clear_state()\n",
" else:\n",
" self.opt = self.opt_func(self.splitter(self.model), lr=self.lr)\n",
" if not self.wd_bn_bias:\n",
" for p in self._bn_bias_state(True ): p['do_wd'] = False\n",
" if self.train_bn:\n",
" for p in self._bn_bias_state(False): p['force_train'] = True\n",
"\n",
" def _split(self, b):\n",
" i = getattr(self.dls, 'n_inp', 1 if len(b)==1 else len(b)-1)\n",
" self.xb,self.yb = b[:i],b[i:]\n",
"\n",
" def _with_events(self, f, event_type, ex, final=noop):\n",
" try: self(f'before_{event_type}'); f()\n",
" except ex: self(f'after_cancel_{event_type}')\n",
" self(f'after_{event_type}'); final()\n",
"\n",
" def all_batches(self):\n",
" self.n_iter = len(self.dl)\n",
" for o in enumerate(self.dl): self.one_batch(*o)\n",
"\n",
" def _backward(self): self.loss_grad.backward()\n",
" def _step(self): self.opt.step()\n",
"\n",
" def _do_grad_opt(self):\n",
" self._with_events(self._backward, 'backward', CancelBackwardException)\n",
" self._with_events(self._step, 'step', CancelStepException)\n",
" self.opt.zero_grad()\n",
"\n",
" def _do_one_batch(self):\n",
" self.pred = self.model(*self.xb)\n",
" self('after_pred')\n",
" if len(self.yb):\n",
" self.loss_grad = self.loss_func(self.pred, *self.yb)\n",
" self.loss = self.loss_grad.clone()\n",
" self('after_loss')\n",
" if not self.training or not len(self.yb): return\n",
" self._do_grad_opt()\n",
"\n",
" def _set_device(self, b):\n",
" model_device = next(self.model.parameters()).device\n",
" dls_device = getattr(self.dls, 'device', default_device())\n",
" if model_device == dls_device: return to_device(b, dls_device)\n",
" else: return to_device(b, model_device)\n",
"\n",
" def one_batch(self, i, b):\n",
" self.iter = i\n",
" b = self._set_device(b)\n",
" self._split(b)\n",
" self._with_events(self._do_one_batch, 'batch', CancelBatchException)\n",
"\n",
" def _do_epoch_train(self):\n",
" self.dl = self.dls.train\n",
" self._with_events(self.all_batches, 'train', CancelTrainException)\n",
"\n",
" def _do_epoch_validate(self, ds_idx=1, dl=None):\n",
" if dl is None: dl = self.dls[ds_idx]\n",
" self.dl = dl\n",
" with torch.no_grad(): self._with_events(self.all_batches, 'validate', CancelValidException)\n",
"\n",
" def _do_epoch(self):\n",
" self._do_epoch_train()\n",
" self._do_epoch_validate()\n",
"\n",
" def _do_fit(self):\n",
" for epoch in range(self.n_epoch):\n",
" self.epoch=epoch\n",
" self._with_events(self._do_epoch, 'epoch', CancelEpochException)\n",
"\n",
" def fit(self, n_epoch, lr=None, wd=None, cbs=None, reset_opt=False, start_epoch=0):\n",
" if start_epoch != 0:\n",
" cbs = L(cbs) + SkipToEpoch(start_epoch)\n",
" with self.added_cbs(cbs):\n",
" if reset_opt or not self.opt: self.create_opt()\n",
" if wd is None: wd = self.wd\n",
" if wd is not None: self.opt.set_hypers(wd=wd)\n",
" self.opt.set_hypers(lr=self.lr if lr is None else lr)\n",
" self.n_epoch = n_epoch\n",
" self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)\n",
"\n",
" def _end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None\n",
" def __enter__(self): self(_before_epoch); return self\n",
" def __exit__(self, exc_type, exc_value, tb): self(_after_epoch)\n",
"\n",
" def validation_context(self, cbs=None, inner=False):\n",
" cms = [self.no_logging(),self.no_mbar(), self.lock]\n",
" if cbs: cms.append(self.added_cbs(cbs))\n",
" if not inner: cms.append(self)\n",
" return ContextManagers(cms)\n",
"\n",
" def validate(self, ds_idx=1, dl=None, cbs=None):\n",
" if dl is None: dl = self.dls[ds_idx]\n",
" with self.validation_context(cbs=cbs): self._do_epoch_validate(ds_idx, dl)\n",
" return getattr(self, 'final_record', None)\n",
"\n",
" @delegates(GatherPredsCallback.__init__)\n",
" def get_preds(self,\n",
" ds_idx:int=1, # `DataLoader` to use for predictions if `dl` is None. 0: train. 1: valid\n",
" dl=None, # `DataLoader` to use for predictions, defaults to `ds_idx=1` if None\n",
" with_input:bool=False, # Return inputs with predictions\n",
" with_decoded:bool=False, # Return decoded predictions\n",
" with_loss:bool=False, # Return per item loss with predictions\n",
" act=None, # Apply activation to predictions, defaults to `self.loss_func`'s activation\n",
" inner:bool=False, # If False, create progress bar, show logger, use temporary `cbs`\n",
" reorder:bool=True, # Reorder predictions on dataset indicies, if applicable\n",
" cbs:Callback|MutableSequence|None=None, # Temporary `Callback`s to apply during prediction\n",
" **kwargs\n",
" )-> tuple:\n",
" if dl is None: dl = self.dls[ds_idx].new(shuffle=False, drop_last=False)\n",
" else:\n",
" try: len(dl)\n",
" except TypeError as e:\n",
" raise TypeError(f\"`dl` is {type(dl)} and doesn't have len(dl)\")\n",
" if isinstance(dl, DataLoader):\n",
" if dl.drop_last: dl = dl.new(shuffle=False, drop_last=False)\n",
" if reorder and hasattr(dl, 'get_idxs'):\n",
" idxs = dl.get_idxs()\n",
" dl = dl.new(get_idxs = _ConstantFunc(idxs))\n",
" cb = GatherPredsCallback(with_input=with_input, with_loss=with_loss, **kwargs)\n",
" ctx_mgrs = self.validation_context(cbs=L(cbs)+[cb], inner=inner)\n",
" if with_loss: ctx_mgrs.append(self.loss_not_reduced())\n",
" with ContextManagers(ctx_mgrs):\n",
" self._do_epoch_validate(dl=dl)\n",
" if act is None: act = getcallable(self.loss_func, 'activation')\n",
" res = cb.all_tensors()\n",
" pred_i = 1 if with_input else 0\n",
" if res[pred_i] is not None:\n",
" res[pred_i] = act(res[pred_i])\n",
" if with_decoded: res.insert(pred_i+2, getcallable(self.loss_func, 'decodes')(res[pred_i]))\n",
" if reorder and hasattr(dl, 'get_idxs'): res = nested_reorder(res, tensor(idxs).argsort())\n",
" return tuple(res)\n",
" self._end_cleanup()\n",
"\n",
" def predict(self, item, rm_type_tfms=None, with_input=False):\n",
" dl = self.dls.test_dl([item], rm_type_tfms=rm_type_tfms, num_workers=0)\n",
" inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)\n",
" i = getattr(self.dls, 'n_inp', -1)\n",
" inp = (inp,) if i==1 else tuplify(inp)\n",
" dec = self.dls.decode_batch(inp + tuplify(dec_preds))[0]\n",
" dec_inp,dec_targ = map(detuplify, [dec[:i],dec[i:]])\n",
" res = dec_targ,dec_preds[0],preds[0]\n",
" if with_input: res = (dec_inp,) + res\n",
" return res\n",
"\n",
" def show_results(self, ds_idx=1, dl=None, max_n=9, shuffle=True, **kwargs):\n",
" if dl is None: dl = self.dls[ds_idx].new(shuffle=shuffle)\n",
" b = dl.one_batch()\n",
" _,_,preds = self.get_preds(dl=[b], with_decoded=True)\n",
" dl.show_results(b, preds, max_n=max_n, **kwargs)\n",
"\n",
" def show_training_loop(self):\n",
" indent = 0\n",
" for s in _loop:\n",
" if s.startswith('Start'): print(f'{\" \"*indent}{s}'); indent += 2\n",
" elif s.startswith('End'): indent -= 2; print(f'{\" \"*indent}{s}')\n",
" else: print(f'{\" \"*indent} - {s:15}:', self.ordered_cbs(s))\n",
"\n",
" @contextmanager\n",
" def no_logging(self): return replacing_yield(self, 'logger', noop)\n",
" @contextmanager\n",
" def no_mbar(self): return replacing_yield(self, 'create_mbar', False)\n",
"\n",
" @contextmanager\n",
" def loss_not_reduced(self):\n",
" if hasattr(self.loss_func, 'reduction'): return replacing_yield(self.loss_func, 'reduction', 'none')\n",
" else: return replacing_yield(self, 'loss_func', partial(self.loss_func, reduction='none'))\n",
" \n",
" def to_detach(self,b,cpu=True,gather=True):\n",
" return self.dl.to_detach(b,cpu,gather) if hasattr(getattr(self,'dl',None),'to_detach') else to_detach(b,cpu,gather)\n",
" \n",
" def __getstate__(self): return {k:v for k,v in self.__dict__.items() if k!='lock'}\n",
" def __setstate__(self, state):\n",
" self.__dict__.update(state)\n",
" self.lock = threading.Lock()\n",
"\n",
"Learner.x,Learner.y = add_props(lambda i,x: detuplify((x.xb,x.yb)[i]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "67008b26",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"add_docs(Learner, \"Group together a `model`, some `dls` and a `loss_func` to handle training\",\n",
" add_cbs=\"Add `cbs` to the list of `Callback` and register `self` as their learner\",\n",
" add_cb=\"Add `cb` to the list of `Callback` and register `self` as their learner\",\n",
" remove_cbs=\"Remove `cbs` from the list of `Callback` and deregister `self` as their learner\",\n",
" remove_cb=\"Add `cb` from the list of `Callback` and deregister `self` as their learner\",\n",
" added_cbs=\"Context manage that temporarily adds `cbs`\",\n",
" removed_cbs=\"Context manage that temporarily removes `cbs`\",\n",
" ordered_cbs=\"List of `Callback`s, in order, for an `event` in the training loop\",\n",
" create_opt=\"Create an optimizer with default hyper-parameters\",\n",
" one_batch=\"Train or evaluate `self.model` on batch `(xb,yb)`\",\n",
" all_batches=\"Train or evaluate `self.model` on all the batches of `self.dl`\",\n",
" fit=\"Fit `self.model` for `n_epoch` using `cbs`. Optionally `reset_opt`.\",\n",
" validate=\"Validate on `dl` with potential new `cbs`.\",\n",
" get_preds=\"Get the predictions and targets on the `ds_idx`-th dbunchset or `dl`, optionally `with_input` and `with_loss`\",\n",
" predict=\"Prediction on `item`, fully decoded, loss function decoded and probabilities\",\n",
" validation_context=\"A `ContextManagers` suitable for validation, with optional `cbs`\",\n",
" show_results=\"Show some predictions on `ds_idx`-th dataset or `dl`\",\n",
" show_training_loop=\"Show each step in the training loop\",\n",
" no_logging=\"Context manager to temporarily remove `logger`\",\n",
" no_mbar=\"Context manager to temporarily prevent the master progress bar from being created\",\n",
" loss_not_reduced=\"A context manager to evaluate `loss_func` with reduction set to none.\",\n",
" to_detach=\"Calls `to_detach` if `self.dl` provides a `.to_detach` function otherwise calls global `to_detach`\",\n",
" __call__=\"Call `event_name` for all `Callback`s in `self.cbs`\"\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "88698d6e",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L106){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner\n",
"\n",
"```python\n",
"\n",
"def Learner(\n",
" dls:DataLoaders, # `DataLoaders` containing fastai or PyTorch `DataLoader`s\n",
" model:Callable, # PyTorch model for training or inference\n",
" loss_func:Callable | None=None, # Loss function. Defaults to `dls` loss\n",
" opt_func:Optimizer | OptimWrapper=Adam, # Optimization function for training\n",
" lr:float | slice=0.001, # Default learning rate\n",
" splitter:Callable=trainable_params, # Split model into parameter groups. Defaults to one parameter group\n",
" cbs:Callback | MutableSequence | None=None, # `Callback`s to add to `Learner`\n",
" metrics:Callable | MutableSequence | None=None, # `Metric`s to calculate on validation set\n",
" path:str | Path | None=None, # Parent directory to save, load, and export models. Defaults to `dls` `path`\n",
" model_dir:str | Path='models', # Subdirectory to save and load models\n",
" wd:float | int | None=None, # Default weight decay\n",
" wd_bn_bias:bool=False, # Apply weight decay to normalization and bias parameters\n",
" train_bn:bool=True, # Train frozen normalization layers\n",
" moms:tuple=(0.95, 0.85, 0.95), # Default momentum for schedulers\n",
" default_cbs:bool=True, # Include default `Callback`s\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Group together a `model`, some `dls` and a `loss_func` to handle training*"
],
"text/plain": [
"```python\n",
"\n",
"def Learner(\n",
" dls:DataLoaders, # `DataLoaders` containing fastai or PyTorch `DataLoader`s\n",
" model:Callable, # PyTorch model for training or inference\n",
" loss_func:Callable | None=None, # Loss function. Defaults to `dls` loss\n",
" opt_func:Optimizer | OptimWrapper=Adam, # Optimization function for training\n",
" lr:float | slice=0.001, # Default learning rate\n",
" splitter:Callable=trainable_params, # Split model into parameter groups. Defaults to one parameter group\n",
" cbs:Callback | MutableSequence | None=None, # `Callback`s to add to `Learner`\n",
" metrics:Callable | MutableSequence | None=None, # `Metric`s to calculate on validation set\n",
" path:str | Path | None=None, # Parent directory to save, load, and export models. Defaults to `dls` `path`\n",
" model_dir:str | Path='models', # Subdirectory to save and load models\n",
" wd:float | int | None=None, # Default weight decay\n",
" wd_bn_bias:bool=False, # Apply weight decay to normalization and bias parameters\n",
" train_bn:bool=True, # Train frozen normalization layers\n",
" moms:tuple=(0.95, 0.85, 0.95), # Default momentum for schedulers\n",
" default_cbs:bool=True, # Include default `Callback`s\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Group together a `model`, some `dls` and a `loss_func` to handle training*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner)"
]
},
{
"cell_type": "markdown",
"id": "51d1eebe",
"metadata": {},
"source": [
"`opt_func` will be used to create an optimizer when `Learner.fit` is called, with `lr` as a default learning rate. `splitter` is a function that takes `self.model` and returns a list of parameter groups (or just one parameter group if there are no different parameter groups). The default is `trainable_params`, which returns all trainable parameters of the model.\n",
"\n",
"`cbs` is one or a list of `Callback`s to pass to the `Learner`. `Callback`s are used for every tweak of the training loop. Each `Callback` is registered as an attribute of `Learner` (with camel case). At creation, all the callbacks in `defaults.callbacks` (`TrainEvalCallback`, `Recorder` and `ProgressCallback`) are associated to the `Learner`.\n",
"\n",
"`metrics` is an optional list of metrics, that can be either functions or `Metric`s (see below). \n",
"\n",
"`path` and `model_dir` are used to save and/or load models. Often `path` will be inferred from `dls`, but you can override it or pass a `Path` object to `model_dir`. Make sure you can write in `path/model_dir`!\n",
"\n",
"`wd` is the default weight decay used when training the model; `moms`, the default momentums used in `Learner.fit_one_cycle`. `wd_bn_bias` controls if weight decay is applied to `BatchNorm` layers and bias. \n",
"\n",
"Lastly, `train_bn` controls if `BatchNorm` layers are trained even when they are supposed to be frozen according to the `splitter`. Our empirical experiments have shown that it's the best behavior for those layers in transfer learning."
]
},
{
"cell_type": "markdown",
"id": "4591c8a8",
"metadata": {},
"source": [
"### PyTorch interop"
]
},
{
"cell_type": "markdown",
"id": "1a383bdd",
"metadata": {},
"source": [
"You can use regular PyTorch functionality for most of the arguments of the `Learner`, although the experience will be smoother with pure fastai objects and you will be able to use the full functionality of the library. The expectation is that the training loop will work smoothly even if you did not use fastai end to end. What you might lose are interpretation objects or showing functionality. The list below explains how to use plain PyTorch objects for all the arguments and what you might lose.\n",
"\n",
"The most important is `opt_func`. If you are not using a fastai optimizer, you will need to write a function that wraps your PyTorch optimizer in an `OptimWrapper`. See the [optimizer module](http://docs.fast.ai/optimizer.html) for more details. This is to ensure the library's schedulers/freeze API work with your code.\n",
"\n",
"- `dls` is a `DataLoaders` object, that you can create from standard PyTorch dataloaders. By doing so, you will lose all showing functionality like `show_batch`/`show_results`. You can check the [data block API](http://docs.fast.ai/tutorial.datablock.html) or the [mid-level data API tutorial](http://docs.fast.ai/tutorial.pets.html) to learn how to use fastai to gather your data!\n",
"- `model` is a standard PyTorch model. You can use anyone you like, just make sure it accepts the number of inputs you have in your `DataLoaders` and returns as many outputs as you have targets.\n",
"- `loss_func` can be any loss function you like. It needs to be one of fastai's if you want to use `Learn.predict` or `Learn.get_preds`, or you will have to implement special methods (see more details after the `BaseLoss` documentation)."
]
},
{
"cell_type": "markdown",
"id": "8ec6ac44",
"metadata": {},
"source": [
"### Training loop"
]
},
{
"cell_type": "markdown",
"id": "9da608d2",
"metadata": {},
"source": [
"Now let's look at the main thing the `Learner` class implements: the training loop."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2685f7e2",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"if not hasattr(defaults, 'callbacks'): defaults.callbacks = [TrainEvalCallback]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df5ece3d",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L263){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.fit\n",
"\n",
"```python\n",
"\n",
"def fit(\n",
" n_epoch, lr:NoneType=None, wd:NoneType=None, cbs:NoneType=None, reset_opt:bool=False, start_epoch:int=0\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Fit `self.model` for `n_epoch` using `cbs`. Optionally `reset_opt`.*"
],
"text/plain": [
"```python\n",
"\n",
"def fit(\n",
" n_epoch, lr:NoneType=None, wd:NoneType=None, cbs:NoneType=None, reset_opt:bool=False, start_epoch:int=0\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Fit `self.model` for `n_epoch` using `cbs`. Optionally `reset_opt`.*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.fit)"
]
},
{
"cell_type": "markdown",
"id": "5805b888",
"metadata": {},
"source": [
"Uses `lr` and `wd` if they are provided, otherwise use the defaults values given by the `lr` and `wd` attributes of `Learner`."
]
},
{
"cell_type": "markdown",
"id": "9eacea58",
"metadata": {},
"source": [
"All the examples use `synth_learner` which is a simple `Learner` training a linear regression model."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "11838d01",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"def synth_learner(n_train=10, n_valid=2, cuda=False, tfmdDL=True, lr=defaults.lr, **kwargs):\n",
" data = synth_dbunch(n_train=n_train,n_valid=n_valid, cuda=cuda, tfmdDL=tfmdDL)\n",
" return Learner(data, RegModel(), loss_func=MSELossFlat(), lr=lr, **kwargs)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "460ab505",
"metadata": {},
"outputs": [],
"source": [
"#Training a few epochs should make the model better\n",
"learn = synth_learner(lr=0.1)\n",
"learn(_before_epoch)\n",
"learn.model = learn.model.cpu()\n",
"xb,yb = learn.dls.one_batch()\n",
"init_loss = learn.loss_func(learn.model(xb), yb)\n",
"learn.fit(10)\n",
"xb,yb = learn.dls.one_batch()\n",
"final_loss = learn.loss_func(learn.model(xb), yb)\n",
"assert final_loss < init_loss, (final_loss,init_loss)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2bc1199e",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Ensure we can train with raw torch\n",
"learn = synth_learner(lr=0.1, tfmdDL=False)\n",
"learn(_before_epoch)\n",
"learn.model = learn.model.cpu()\n",
"xb,yb = next(iter(learn.dls[0]))\n",
"init_loss = learn.loss_func(learn.model(xb), yb)\n",
"learn.fit(10)\n",
"xb,yb = next(iter(learn.dls[0]))\n",
"learn.model = learn.model.cpu() # Ensure we're still on CPU even in a CUDA environment\n",
"final_loss = learn.loss_func(learn.model(xb), yb)\n",
"assert final_loss < init_loss, (final_loss,init_loss)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7ee1374b",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"class TestTrainEvalCallback(Callback):\n",
" run_after,run_valid = TrainEvalCallback,False\n",
" def before_fit(self): \n",
" test_eq([self.pct_train,self.train_iter], [0., 0])\n",
" self.old_pct_train,self.old_train_iter = self.pct_train,self.train_iter\n",
" \n",
" def before_batch(self): test_eq(next(self.parameters()).device, find_device(self.xb))\n",
" \n",
" def after_batch(self):\n",
" assert self.training\n",
" test_eq(self.pct_train , self.old_pct_train+1/(self.n_iter*self.n_epoch))\n",
" test_eq(self.train_iter, self.old_train_iter+1)\n",
" self.old_pct_train,self.old_train_iter = self.pct_train,self.train_iter\n",
" \n",
" def before_train(self):\n",
" assert self.training and self.model.training\n",
" test_eq(self.pct_train, self.epoch/self.n_epoch)\n",
" self.old_pct_train = self.pct_train\n",
" \n",
" def before_validate(self):\n",
" assert not self.training and not self.model.training\n",
" \n",
"learn = synth_learner(cbs=TestTrainEvalCallback)\n",
"learn.fit(1)\n",
"#Check order is properly taken into account\n",
"learn.cbs = L(reversed(learn.cbs))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2c0096d4",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#| cuda\n",
"#Check model is put on the GPU if needed\n",
"learn = synth_learner(cbs=TestTrainEvalCallback, cuda=True)\n",
"learn.fit(1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7eb45be7",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#| cuda\n",
"#Check that raw DataLoaders are put on the GPU\n",
"learn = synth_learner(cbs=TestTrainEvalCallback, tfmdDL=False)\n",
"learn.fit(1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5aa0aa62",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Check wd is not applied on bn/bias when option wd_bn_bias=False\n",
"class _TstModel(nn.Module):\n",
" def __init__(self):\n",
" super().__init__()\n",
" self.a,self.b = nn.Parameter(torch.randn(1)),nn.Parameter(torch.randn(1))\n",
" self.tst = nn.Sequential(nn.Linear(4,5), nn.BatchNorm1d(3))\n",
" self.tst[0].bias.data,self.tst[1].bias.data = torch.randn(5),torch.randn(3) \n",
" def forward(self, x): return x * self.a + self.b\n",
" \n",
"class _PutGrad(Callback):\n",
" def before_step(self):\n",
" for p in self.learn.tst.parameters():\n",
" p.grad = torch.ones_like(p.data)\n",
" \n",
"learn = synth_learner(n_train=5, opt_func = partial(SGD, wd=1, decouple_wd=True), cbs=_PutGrad)\n",
"learn.model = _TstModel()\n",
"init = [p.clone() for p in learn.tst.parameters()]\n",
"learn.fit(1, lr=1e-2)\n",
"end = list(learn.tst.parameters())\n",
"assert not torch.allclose(end[0]-init[0], -0.05 * torch.ones_like(end[0]))\n",
"for i in [1,2,3]: test_close(end[i]-init[i], -0.05 * torch.ones_like(end[i]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7c5beb91",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L239){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.one_batch\n",
"\n",
"```python\n",
"\n",
"def one_batch(\n",
" i, b\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Train or evaluate `self.model` on batch `(xb,yb)`*"
],
"text/plain": [
"```python\n",
"\n",
"def one_batch(\n",
" i, b\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Train or evaluate `self.model` on batch `(xb,yb)`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.one_batch)"
]
},
{
"cell_type": "markdown",
"id": "88d54693",
"metadata": {},
"source": [
"This is an internal method called by `Learner.fit`. If passed, `i` is the index of this iteration in the epoch. In training mode, this does a full training step on the batch (compute predictions, loss, gradients, update the model parameters and zero the gradients). In validation mode, it stops at the loss computation. Training or validation is controlled internally by the `TrainEvalCallback` through the `training` attribute.\n",
"\n",
"Nothing is returned, but the attributes `x`, `y`, `pred`, `loss` of the `Learner` are set with the proper values:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "49fcfd4d",
"metadata": {},
"outputs": [],
"source": [
"b = learn.dls.one_batch()\n",
"learn.one_batch(0, b)\n",
"test_eq(learn.x, b[0])\n",
"test_eq(learn.y, b[1])\n",
"out = learn.model(learn.x)\n",
"test_eq(learn.pred, out)\n",
"test_eq(learn.loss, learn.loss_func(out, b[1]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "706a2214",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"class VerboseCallback(Callback):\n",
" \"Callback that prints the name of each event called\"\n",
" def __call__(self, event_name):\n",
" print(event_name)\n",
" super().__call__(event_name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "68a5c636",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"class TestOneBatch(VerboseCallback):\n",
" def __init__(self, xb, yb, i):\n",
" self.save_xb,self.save_yb,self.i = xb,yb,i\n",
" self.old_pred,self.old_loss = None,tensor(0.)\n",
" \n",
" def before_batch(self):\n",
" self.old_a,self.old_b = self.a.data.clone(),self.b.data.clone()\n",
" test_eq(self.iter, self.i)\n",
" test_eq(self.save_xb, *self.xb)\n",
" test_eq(self.save_yb, *self.yb)\n",
" if hasattr(self.learn, 'pred'): test_eq(self.pred, self.old_pred)\n",
" \n",
" def after_pred(self):\n",
" self.old_pred = self.pred\n",
" test_eq(self.pred, self.a.data * self.x + self.b.data)\n",
" test_eq(self.loss, self.old_loss)\n",
" \n",
" def after_loss(self):\n",
" self.old_loss = self.loss\n",
" test_eq(self.loss, self.loss_func(self.old_pred, self.save_yb))\n",
" for p in self.parameters(): \n",
" if not hasattr(p, 'grad') or p.grad is not None: test_eq(p.grad, tensor([0.]))\n",
" \n",
" def before_step(self):\n",
" self.grad_a = (2 * self.x * (self.pred.data - self.y)).mean()\n",
" self.grad_b = 2 * (self.pred.data - self.y).mean()\n",
" test_close(self.a.grad.data, self.grad_a)\n",
" test_close(self.b.grad.data, self.grad_b)\n",
" test_eq(self.a.data, self.old_a)\n",
" test_eq(self.b.data, self.old_b)\n",
" \n",
" def after_step(self):\n",
" test_close(self.a.data, self.old_a - self.lr * self.grad_a)\n",
" test_close(self.b.data, self.old_b - self.lr * self.grad_b)\n",
" self.old_a,self.old_b = self.a.data.clone(),self.b.data.clone()\n",
" test_close(self.a.grad.data, self.grad_a)\n",
" test_close(self.b.grad.data, self.grad_b)\n",
" \n",
" def after_batch(self):\n",
" for p in self.parameters(): test_eq(p.grad, tensor([0.]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8eac000c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_create\n"
]
}
],
"source": [
"#| hide\n",
"learn = synth_learner()\n",
"b = learn.dls.one_batch()\n",
"learn = synth_learner(cbs=TestOneBatch(*b, 42), lr=1e-2)\n",
"#Remove train/eval\n",
"learn.cbs = learn.cbs[1:]\n",
"#Setup\n",
"learn.loss,learn.training = tensor(0.),True\n",
"learn.opt = SGD(learn.parameters(), lr=learn.lr)\n",
"learn.model.train()\n",
"batch_events = ['before_batch', 'after_pred', 'after_loss', 'before_backward', 'after_backward', 'before_step', 'after_step', 'after_batch']\n",
"test_stdout(lambda: learn.one_batch(42, b), '\\n'.join(batch_events))\n",
"test_stdout(lambda: learn.one_batch(42, b), '\\n'.join(batch_events)) #Check it works for a second batch"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e7209ed1",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L211){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.all_batches\n",
"\n",
"```python\n",
"\n",
"def all_batches(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Train or evaluate `self.model` on all the batches of `self.dl`*"
],
"text/plain": [
"```python\n",
"\n",
"def all_batches(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Train or evaluate `self.model` on all the batches of `self.dl`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.all_batches)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "15408049",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_create\n"
]
}
],
"source": [
"#| hide\n",
"learn = synth_learner(n_train=5, cbs=VerboseCallback())\n",
"learn.opt = SGD(learn.parameters(), lr=learn.lr)\n",
"with redirect_stdout(io.StringIO()): \n",
" learn(_before_epoch)\n",
" learn.epoch,learn.dl = 0,learn.dls.train\n",
" learn('before_train')\n",
"test_stdout(learn.all_batches, '\\n'.join(batch_events * 5))\n",
"test_eq(learn.train_iter, 5)\n",
"\n",
"valid_events = ['before_batch', 'after_pred', 'after_loss', 'after_batch']\n",
"with redirect_stdout(io.StringIO()): \n",
" learn.dl = learn.dls.valid\n",
" learn('before_validate')\n",
"test_stdout(learn.all_batches, '\\n'.join(valid_events * 2))\n",
"test_eq(learn.train_iter, 5)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cb282dd1",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_create\n"
]
}
],
"source": [
"#| hide\n",
"learn = synth_learner(n_train=5, cbs=VerboseCallback())\n",
"test_stdout(lambda: learn(_before_epoch), 'before_fit\\nbefore_epoch')\n",
"test_eq(learn.loss, tensor(0.))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7b1dd615",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"learn.opt = SGD(learn.parameters(), lr=learn.lr)\n",
"learn.epoch = 0\n",
"test_stdout(lambda: learn._do_epoch_train(), '\\n'.join(['before_train'] + batch_events * 5 + ['after_train']))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6a9d5b65",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"test_stdout(learn._do_epoch_validate, '\\n'.join(['before_validate'] + valid_events * 2+ ['after_validate']))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a7924439",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L188){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.create_opt\n",
"\n",
"```python\n",
"\n",
"def create_opt(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Create an optimizer with default hyper-parameters*"
],
"text/plain": [
"```python\n",
"\n",
"def create_opt(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Create an optimizer with default hyper-parameters*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.create_opt)"
]
},
{
"cell_type": "markdown",
"id": "5307d8c4",
"metadata": {},
"source": [
"This method is called internally to create the optimizer, the hyper-parameters are then adjusted by what you pass to `Learner.fit` or your particular schedulers (see `callback.schedule`)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ff74246c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_create\n"
]
}
],
"source": [
"learn = synth_learner(n_train=5, cbs=VerboseCallback())\n",
"assert learn.opt is None\n",
"learn.create_opt()\n",
"assert learn.opt is not None\n",
"test_eq(learn.opt.hypers[0]['lr'], learn.lr)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b61dd606",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_create\n"
]
}
],
"source": [
"learn = synth_learner(n_train=5, cbs=VerboseCallback(), opt_func=partial(OptimWrapper, opt=torch.optim.Adam))\n",
"assert learn.opt is None\n",
"learn.create_opt()\n",
"assert learn.opt is not None\n",
"test_eq(learn.opt.hypers[0]['lr'], learn.lr)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a8c1a86f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_create\n"
]
}
],
"source": [
"wrapper_lr = 1\n",
"learn = synth_learner(n_train=5, cbs=VerboseCallback(), opt_func=partial(OptimWrapper, opt=torch.optim.Adam, lr=wrapper_lr))\n",
"assert learn.opt is None\n",
"learn.create_opt()\n",
"assert learn.opt is not None\n",
"test_eq(learn.opt.hypers[0]['lr'], wrapper_lr)"
]
},
{
"cell_type": "markdown",
"id": "78071fe6",
"metadata": {},
"source": [
"### Callback handling"
]
},
{
"cell_type": "markdown",
"id": "83c7f12b",
"metadata": {},
"source": [
"We only describe the basic functionality linked to `Callback`s here. To learn more about `Callback`s and how to write them, check the [callback.core](http://docs.fast.ai/callback.core.html) module documentation.\n",
"\n",
"Let's first see how the `Callback`s become attributes of `Learner`:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "07ea860e",
"metadata": {},
"outputs": [],
"source": [
"#Test init with callbacks\n",
"class TstCallback(Callback):\n",
" def batch_begin(self): self.learn.a = self.a + 1\n",
"\n",
"tst_learn = synth_learner()\n",
"test_eq(len(tst_learn.cbs), 1)\n",
"assert hasattr(tst_learn, ('train_eval'))\n",
"\n",
"tst_learn = synth_learner(cbs=TstCallback())\n",
"test_eq(len(tst_learn.cbs), 2)\n",
"assert hasattr(tst_learn, ('tst'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1e7eb4d7",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L180){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.__call__\n",
"\n",
"```python\n",
"\n",
"def __call__(\n",
" event_name\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Call `event_name` for all `Callback`s in `self.cbs`*"
],
"text/plain": [
"```python\n",
"\n",
"def __call__(\n",
" event_name\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Call `event_name` for all `Callback`s in `self.cbs`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.__call__)"
]
},
{
"cell_type": "markdown",
"id": "51093b83",
"metadata": {},
"source": [
"This how the `Callback`s are called internally. For instance a `VerboseCallback` just prints the event names (can be useful for debugging):"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "71463e0f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_create\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"after_fit\n"
]
}
],
"source": [
"learn = synth_learner(cbs=VerboseCallback())\n",
"learn('after_fit')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9a9c1a2e",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L152){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.add_cb\n",
"\n",
"```python\n",
"\n",
"def add_cb(\n",
" cb\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Add `cb` to the list of `Callback` and register `self` as their learner*"
],
"text/plain": [
"```python\n",
"\n",
"def add_cb(\n",
" cb\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Add `cb` to the list of `Callback` and register `self` as their learner*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.add_cb)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bbc3849d",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner()\n",
"learn.add_cb(TestTrainEvalCallback())\n",
"test_eq(len(learn.cbs), 2)\n",
"assert isinstance(learn.cbs[1], TestTrainEvalCallback)\n",
"test_eq(learn.train_eval.learn, learn)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8b3be539",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L144){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.add_cbs\n",
"\n",
"```python\n",
"\n",
"def add_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Add `cbs` to the list of `Callback` and register `self` as their learner*"
],
"text/plain": [
"```python\n",
"\n",
"def add_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Add `cbs` to the list of `Callback` and register `self` as their learner*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.add_cbs)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4a8705c0",
"metadata": {},
"outputs": [],
"source": [
"learn.add_cbs([TestTrainEvalCallback(), TestTrainEvalCallback()])\n",
"test_eq(len(learn.cbs), 4)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "21b1fe6d",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L168){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.added_cbs\n",
"\n",
"```python\n",
"\n",
"def added_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```"
],
"text/plain": [
"```python\n",
"\n",
"def added_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.added_cbs)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "33e4baa8",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner()\n",
"test_eq(len(learn.cbs), 1)\n",
"with learn.added_cbs(TestTrainEvalCallback()):\n",
" test_eq(len(learn.cbs), 2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eb3395de",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L179){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.ordered_cbs\n",
"\n",
"```python\n",
"\n",
"def ordered_cbs(\n",
" event\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*List of `Callback`s, in order, for an `event` in the training loop*"
],
"text/plain": [
"```python\n",
"\n",
"def ordered_cbs(\n",
" event\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*List of `Callback`s, in order, for an `event` in the training loop*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.ordered_cbs)"
]
},
{
"cell_type": "markdown",
"id": "8ce90612",
"metadata": {},
"source": [
"By order, we mean using the internal ordering of the `Callback`s (see `callback.core` for more information on how it works)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "22b9bf5c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[TrainEvalCallback, TestTrainEvalCallback]"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"learn = synth_learner()\n",
"learn.add_cb(TestTrainEvalCallback())\n",
"learn.ordered_cbs('before_fit')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "30827a37",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L159){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.remove_cb\n",
"\n",
"```python\n",
"\n",
"def remove_cb(\n",
" cb\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Add `cb` from the list of `Callback` and deregister `self` as their learner*"
],
"text/plain": [
"```python\n",
"\n",
"def remove_cb(\n",
" cb\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Add `cb` from the list of `Callback` and deregister `self` as their learner*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.remove_cb)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7a366e8b",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner()\n",
"learn.add_cb(TestTrainEvalCallback())\n",
"cb = learn.cbs[1]\n",
"learn.remove_cb(learn.cbs[1])\n",
"test_eq(len(learn.cbs), 1)\n",
"assert cb.learn is None\n",
"assert not getattr(learn,'test_train_eval',None)"
]
},
{
"cell_type": "markdown",
"id": "d5b326e1",
"metadata": {},
"source": [
"`cb` can simply be the class of the `Callback` we want to remove (in which case all instances of that callback are removed)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "33020e5b",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner()\n",
"learn.add_cbs([TestTrainEvalCallback(), TestTrainEvalCallback()])\n",
"learn.remove_cb(TestTrainEvalCallback)\n",
"test_eq(len(learn.cbs), 1)\n",
"assert not getattr(learn,'test_train_eval',None)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d8c75f1c",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L148){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.remove_cbs\n",
"\n",
"```python\n",
"\n",
"def remove_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Remove `cbs` from the list of `Callback` and deregister `self` as their learner*"
],
"text/plain": [
"```python\n",
"\n",
"def remove_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Remove `cbs` from the list of `Callback` and deregister `self` as their learner*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.remove_cbs)"
]
},
{
"cell_type": "markdown",
"id": "f1e38246",
"metadata": {},
"source": [
"Elements of `cbs` can either be types of callbacks or actual callbacks of the `Learner`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ac233a7b",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner()\n",
"learn.add_cbs([TestTrainEvalCallback() for _ in range(3)])\n",
"cb = learn.cbs[1]\n",
"learn.remove_cbs(learn.cbs[1:])\n",
"test_eq(len(learn.cbs), 1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d93ec49a",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L174){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.removed_cbs\n",
"\n",
"```python\n",
"\n",
"def removed_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```"
],
"text/plain": [
"```python\n",
"\n",
"def removed_cbs(\n",
" cbs\n",
"):\n",
"\n",
"\n",
"```"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.removed_cbs)"
]
},
{
"cell_type": "markdown",
"id": "a998ad92",
"metadata": {},
"source": [
"Elements of `cbs` can either be types of callbacks or actual callbacks of the `Learner`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ae6d8647",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner()\n",
"learn.add_cb(TestTrainEvalCallback())\n",
"with learn.removed_cbs(learn.cbs[1]):\n",
" test_eq(len(learn.cbs), 1)\n",
"test_eq(len(learn.cbs), 2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ad6ecf84",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L344){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.show_training_loop\n",
"\n",
"```python\n",
"\n",
"def show_training_loop(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Show each step in the training loop*"
],
"text/plain": [
"```python\n",
"\n",
"def show_training_loop(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Show each step in the training loop*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.show_training_loop)"
]
},
{
"cell_type": "markdown",
"id": "5bc5ed14",
"metadata": {},
"source": [
"At each step, callbacks are shown in order, which can help debugging."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "61869b81",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Start Fit\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - before_fit : [TrainEvalCallback]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Start Epoch Loop\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - before_epoch : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Start Train\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - before_train : [TrainEvalCallback]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Start Batch Loop\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - before_batch : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_pred : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_loss : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - before_backward: []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - before_step : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_step : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_cancel_batch: []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_batch : [TrainEvalCallback]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" End Batch Loop\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" End Train\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_cancel_train: []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_train : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Start Valid\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - before_validate: [TrainEvalCallback]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" Start Batch Loop\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - **CBs same as train batch**: []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" End Batch Loop\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" End Valid\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_cancel_validate: []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_validate : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" End Epoch Loop\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_cancel_epoch: []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_epoch : []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"End Fit\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_cancel_fit: []\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" - after_fit : []\n"
]
}
],
"source": [
"learn = synth_learner()\n",
"learn.show_training_loop()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "27f3a5b1",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def _before_batch_cb(f, self):\n",
" xb,yb = f(self, self.xb, self.yb)\n",
" self.learn.xb,self.learn.yb = xb,yb"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fe5cfd03",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def before_batch_cb(f):\n",
" \"Shortcut for creating a Callback on the `before_batch` event, which takes and returns `xb,yb`\"\n",
" return Callback(before_batch=partial(_before_batch_cb, f))"
]
},
{
"cell_type": "markdown",
"id": "a7e552ec",
"metadata": {},
"source": [
"In order to change the data passed to your model, you will generally want to hook into the `before_batch` event, like so:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "875daa41",
"metadata": {},
"outputs": [],
"source": [
"class TstCallback(Callback):\n",
" def before_batch(self):\n",
" self.learn.xb = self.xb + 1000\n",
" self.learn.yb = self.yb - 1000"
]
},
{
"cell_type": "markdown",
"id": "15f896f3",
"metadata": {},
"source": [
"Since that is so common, we provide the `before_batch_cb` decorator to make it easier."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1b67c731",
"metadata": {},
"outputs": [],
"source": [
"@before_batch_cb\n",
"def cb(self, xb, yb): return xb+1000,yb-1000"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cdbd150e",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"# test SkipToEpoch callback\n",
"class TestSkipToEpoch(Callback):\n",
" def after_train(self):\n",
" assert self.epoch >= 2\n",
"learn = synth_learner(cbs=TestSkipToEpoch())\n",
"learn.fit(4, start_epoch=2)\n",
"\n",
"learn = synth_learner()\n",
"p0_pre = first(learn.model.parameters()).data.clone()\n",
"learn.fit(3, start_epoch=3)\n",
"p0 = first(learn.model.parameters()).data\n",
"test_eq(p0_pre, p0)"
]
},
{
"cell_type": "markdown",
"id": "a64a6d1e",
"metadata": {},
"source": [
"### Serializing"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "94fb9ba9",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@patch\n",
"@delegates(save_model)\n",
"def save(self:Learner, file, **kwargs):\n",
" \"Save model and optimizer state (if `with_opt`) to `self.path/self.model_dir/file`\"\n",
" file = join_path_file(file, self.path/self.model_dir, ext='.pth')\n",
" save_model(file, self.model, getattr(self,'opt',None), **kwargs)\n",
" return file"
]
},
{
"cell_type": "markdown",
"id": "d44816d1",
"metadata": {},
"source": [
"`file` can be a `Path`, a `string` or a buffer. `pickle_protocol` is passed along to `torch.save`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4158bc6c",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@patch\n",
"@delegates(load_model)\n",
"def load(self:Learner, file, device=None, **kwargs):\n",
" \"Load model and optimizer state (if `with_opt`) from `self.path/self.model_dir/file` using `device`\"\n",
" if device is None and hasattr(self.dls, 'device'): device = self.dls.device\n",
" if self.opt is None: self.create_opt()\n",
" file = join_path_file(file, self.path/self.model_dir, ext='.pth')\n",
" distrib_barrier()\n",
" load_model(file, self.model, self.opt, device=device, **kwargs)\n",
" return self"
]
},
{
"cell_type": "markdown",
"id": "50860363",
"metadata": {},
"source": [
"`file` can be a `Path`, a `string` or a buffer. Use `device` to load the model/optimizer state on a device different from the one it was saved.\n",
"\n",
"As of PyTorch 2.5 and fastai 2.7.19, `Learner.load` uses PyTorch's safe serialization for model loading."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a8b3aff7",
"metadata": {},
"outputs": [],
"source": [
"with tempfile.TemporaryDirectory() as d:\n",
" learn = synth_learner(path=d)\n",
" learn.fit(1)\n",
" \n",
" #Test save created a file\n",
" learn.save('tmp')\n",
" assert (Path(d)/'models/tmp.pth').exists()\n",
" \n",
" #Test load did load the model\n",
" learn1 = synth_learner(path=d)\n",
" learn1 = learn1.load('tmp')\n",
" test_eq(learn.a, learn1.a)\n",
" test_eq(learn.b, learn1.b)\n",
" test_eq(learn.opt.state_dict(), learn1.opt.state_dict())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "96a95c1e",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test load works when the model is saved without opt\n",
"with tempfile.TemporaryDirectory() as d:\n",
" learn = synth_learner(path=d)\n",
" learn.fit(1)\n",
" learn.save('tmp', with_opt=False)\n",
" learn1 = synth_learner(path=d)\n",
" learn1 = learn1.load('tmp', with_opt=False)\n",
" test_eq(learn.a, learn1.a)\n",
" test_eq(learn.b, learn1.b)\n",
" test_ne(learn.opt.state_dict(), learn1.opt.state_dict())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c6085712",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@patch\n",
"def export(self:Learner, fname='export.pkl', pickle_module=cloudpickle, pickle_protocol=2):\n",
" \"Export the content of `self` without the items and the optimizer state for inference\"\n",
" if rank_distrib(): return # don't export if child proc\n",
" self._end_cleanup()\n",
" old_dbunch = self.dls\n",
" self.dls = self.dls.new_empty()\n",
" state = self.opt.state_dict() if self.opt is not None else None\n",
" self.opt = None\n",
" with warnings.catch_warnings():\n",
" #To avoid the warning that come from PyTorch about model not being checked\n",
" warnings.simplefilter(\"ignore\")\n",
" torch.save(self, self.path/fname, pickle_module=pickle_module, pickle_protocol=pickle_protocol)\n",
" self.create_opt()\n",
" if state is not None: self.opt.load_state_dict(state)\n",
" self.dls = old_dbunch"
]
},
{
"cell_type": "markdown",
"id": "3822bd53",
"metadata": {},
"source": [
"The `Learner` is saved in `self.path/fname`, using `pickle_protocol`. Note that serialization in Python saves the names of functions, not the code itself. Therefore, any custom code you have for models, data transformation, loss function etc... should be put in a module that you will import in your training environment before exporting, and in your deployment environment before loading it."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7fd3a9ae",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def load_learner(fname, cpu=True, pickle_module=pickle):\n",
" \"Load a `Learner` object in `fname`, by default putting it on the `cpu`\"\n",
" distrib_barrier()\n",
" map_loc = 'cpu' if cpu else default_device()\n",
" try:\n",
" warn(\"load_learner` uses Python's insecure pickle module, which can execute malicious arbitrary code when loading. Only load files you trust.\\nIf you only need to load model weights and optimizer state, use the safe `Learner.load` instead.\")\n",
" load_kwargs = {\"weights_only\": False} if ismin_torch(\"2.6\") else {}\n",
" res = torch.load(fname, map_location=map_loc, pickle_module=pickle_module, **load_kwargs)\n",
" except ImportError as e:\n",
" if any(o in str(e) for o in (\"fastcore.transform\",\"fastcore.dispatch\")): \n",
" raise RuntimeError(f\"Loading model {fname=}, attempted to import from `fastcore.dispatch` and/or `fastcore.transform` which are deprecated in `fastai>=2.8.0`.\\nDowngrade to `fastai<2.8.0` if you want to load this model.\")\n",
" except AttributeError as e:\n",
" e.args = [f\"Custom classes or functions exported with your `Learner` not available in namespace. Re-declare/import before loading:\\n\\t{e.args[0]}\"]\n",
" raise\n",
" if cpu:\n",
" res.dls.cpu()\n",
" if hasattr(res, 'channels_last'): res = res.to_contiguous(to_fp32=True)\n",
" elif hasattr(res, 'mixed_precision'): res = res.to_fp32()\n",
" elif hasattr(res, 'non_native_mixed_precision'): res = res.to_non_native_fp32()\n",
" return res"
]
},
{
"cell_type": "markdown",
"id": "62f53bac",
"metadata": {},
"source": [
":::{.callout-important}\n",
"\n",
"`load_learner` uses Python's insecure `pickle` module. Maliciously crafted pickle data can execute arbitrary code when loading. Never use `load_learner` on untrusted or potentially compromised sources. **Only load files you fully trust**.\"\n",
"\n",
"If you only need to load model weights and optimizer state, use the safe `Learner.load` instead.\n",
"\n",
":::\n",
"\n",
":::{.callout-warning}\n",
"\n",
"`load_learner` requires all your custom code be in the exact same place as when exporting your `Learner` (the main script, or the module you imported it from).\n",
"\n",
":::"
]
},
{
"cell_type": "markdown",
"id": "eb59a939",
"metadata": {},
"source": [
"### DataLoader aware `to_detach` -"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "10b041b3",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L361){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.to_detach\n",
"\n",
"```python\n",
"\n",
"def to_detach(\n",
" b, cpu:bool=True, gather:bool=True\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Calls `to_detach` if `self.dl` provides a `.to_detach` function otherwise calls global `to_detach`*"
],
"text/plain": [
"```python\n",
"\n",
"def to_detach(\n",
" b, cpu:bool=True, gather:bool=True\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Calls `to_detach` if `self.dl` provides a `.to_detach` function otherwise calls global `to_detach`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.to_detach)"
]
},
{
"cell_type": "markdown",
"id": "1cee91b0",
"metadata": {},
"source": [
"fastai provides `to_detach` which by default detachs tensor gradients, and gathers (calling `maybe_gather`) tensors from all ranks if running in distributed data parallel (DDP) mode.\n",
"\n",
"When running in DDP mode all ranks need to have the same batch size, and `DistributedDL` takes care of padding batches as needed; however when gathering all tensors (e.g. for calculating metrics, inference, etc.) we need to discard the padded items. `DistributedDL` provides a method `to_detach` that removes padding appropriately.\n",
"\n",
"Calling the learner's `to_detach` method will attempt to find a `to_detach` method in the learner's last used `DataLoader` `dl` and use that one if found, otherwise it will resort to the vanilla `to_detach`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c2b0794e",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"learn = synth_learner()\n",
"test_eq(learn.to_detach(Tensor([123])),Tensor([123]))\n",
"learn.dl = learn.dls[0]\n",
"test_eq(learn.to_detach(Tensor([123])),Tensor([123]))\n",
"learn.dl.to_detach = lambda b,cpu,gather: b-100\n",
"test_eq(learn.to_detach(Tensor([123.])),Tensor([23.]))"
]
},
{
"cell_type": "markdown",
"id": "40fa99b8",
"metadata": {},
"source": [
"## Metrics -"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d891f797",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@docs\n",
"class Metric():\n",
" \"Blueprint for defining a metric\"\n",
" def reset(self): pass\n",
" def accumulate(self, learn): pass\n",
" @property\n",
" def value(self): raise NotImplementedError\n",
"\n",
" @property\n",
" def name(self): return class2attr(self, 'Metric')\n",
"\n",
" _docs = dict(\n",
" reset=\"Reset inner state to prepare for new computation\",\n",
" name=\"Name of the `Metric`, camel-cased and with Metric removed\",\n",
" accumulate=\"Use `learn` to update the state with new results\",\n",
" value=\"The value of the metric\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d0e0cffe",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L473){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Metric\n",
"\n",
"```python\n",
"\n",
"def Metric(\n",
" args:VAR_POSITIONAL, kwargs:VAR_KEYWORD\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Blueprint for defining a metric*"
],
"text/plain": [
"```python\n",
"\n",
"def Metric(\n",
" args:VAR_POSITIONAL, kwargs:VAR_KEYWORD\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Blueprint for defining a metric*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Metric, title_level=3)"
]
},
{
"cell_type": "markdown",
"id": "502c9afc",
"metadata": {},
"source": [
"Metrics can be simple averages (like accuracy) but sometimes their computation is a little bit more complex and can't be averaged over batches (like precision or recall), which is why we need a special class for them. For simple functions that can be computed as averages over batches, we can use the class `AvgMetric`, otherwise you'll need to implement the following methods.\n",
"\n",
":::{.callout-note}\n",
"\n",
"If your <code>Metric</code> has state depending on tensors, don't forget to store it on the CPU to avoid any potential memory leaks.\n",
"\n",
":::"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d3f96442",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L475){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Metric.reset\n",
"\n",
"```python\n",
"\n",
"def reset(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Reset inner state to prepare for new computation*"
],
"text/plain": [
"```python\n",
"\n",
"def reset(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Reset inner state to prepare for new computation*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Metric.reset)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eb3a4ce2",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L476){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Metric.accumulate\n",
"\n",
"```python\n",
"\n",
"def accumulate(\n",
" learn\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Use `learn` to update the state with new results*"
],
"text/plain": [
"```python\n",
"\n",
"def accumulate(\n",
" learn\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Use `learn` to update the state with new results*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Metric.accumulate)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "60457e5b",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L478){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Metric.value\n",
"\n",
"```python\n",
"\n",
"def value(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
],
"text/plain": [
"```python\n",
"\n",
"def value(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Metric.value, name='Metric.value')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1acdc732",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L481){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Metric.name\n",
"\n",
"```python\n",
"\n",
"def name(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
],
"text/plain": [
"```python\n",
"\n",
"def name(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Metric.name, name='Metric.name')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "aae8a2d3",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class AvgMetric(Metric):\n",
" \"Average the values of `func` taking into account potential different batch sizes\"\n",
" def __init__(self, func): self.func = func\n",
" def reset(self): self.total,self.count = 0.,0\n",
" def accumulate(self, learn):\n",
" bs = find_bs(learn.yb)\n",
" self.total += learn.to_detach(self.func(learn.pred, *learn.yb))*bs\n",
" self.count += bs\n",
" @property\n",
" def value(self): return self.total/self.count if self.count != 0 else None\n",
" @property\n",
" def name(self): return self.func.func.__name__ if hasattr(self.func, 'func') else self.func.__name__"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e045973c",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L490){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### AvgMetric\n",
"\n",
"```python\n",
"\n",
"def AvgMetric(\n",
" func\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Average the values of `func` taking into account potential different batch sizes*"
],
"text/plain": [
"```python\n",
"\n",
"def AvgMetric(\n",
" func\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Average the values of `func` taking into account potential different batch sizes*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(AvgMetric, title_level=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "319a0fca",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner()\n",
"tst = AvgMetric(lambda x,y: (x-y).abs().mean())\n",
"t,u = torch.randn(100),torch.randn(100)\n",
"tst.reset()\n",
"for i in range(0,100,25): \n",
" learn.pred,learn.yb = t[i:i+25],(u[i:i+25],)\n",
" tst.accumulate(learn)\n",
"test_close(tst.value, (t-u).abs().mean())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "901930e9",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#With varying batch size\n",
"tst.reset()\n",
"splits = [0, 30, 50, 60, 100]\n",
"for i in range(len(splits )-1): \n",
" learn.pred,learn.yb = t[splits[i]:splits[i+1]],(u[splits[i]:splits[i+1]],)\n",
" tst.accumulate(learn)\n",
"test_close(tst.value, (t-u).abs().mean())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b55aebf5",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class AvgLoss(Metric):\n",
" \"Average the losses taking into account potential different batch sizes\"\n",
" def reset(self): self.total,self.count = 0.,0\n",
" def accumulate(self, learn):\n",
" bs = find_bs(learn.yb)\n",
" self.total += learn.to_detach(learn.loss.mean())*bs\n",
" self.count += bs\n",
" @property\n",
" def value(self): return self.total/self.count if self.count != 0 else None\n",
" @property\n",
" def name(self): return \"loss\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2f916fc4",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L504){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### AvgLoss\n",
"\n",
"```python\n",
"\n",
"def AvgLoss(\n",
" args:VAR_POSITIONAL, kwargs:VAR_KEYWORD\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Average the losses taking into account potential different batch sizes*"
],
"text/plain": [
"```python\n",
"\n",
"def AvgLoss(\n",
" args:VAR_POSITIONAL, kwargs:VAR_KEYWORD\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Average the losses taking into account potential different batch sizes*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(AvgLoss, title_level=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf3fb9f2",
"metadata": {},
"outputs": [],
"source": [
"tst = AvgLoss()\n",
"t = torch.randn(100)\n",
"tst.reset()\n",
"for i in range(0,100,25): \n",
" learn.yb,learn.loss = t[i:i+25],t[i:i+25].mean()\n",
" tst.accumulate(learn)\n",
"test_close(tst.value, t.mean())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "792fb1ad",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#With varying batch size\n",
"tst.reset()\n",
"splits = [0, 30, 50, 60, 100]\n",
"for i in range(len(splits )-1): \n",
" learn.yb,learn.loss = t[splits[i]:splits[i+1]],t[splits[i]:splits[i+1]].mean()\n",
" tst.accumulate(learn)\n",
"test_close(tst.value, t.mean())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f2ba91f7",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class AvgSmoothLoss(Metric):\n",
" \"Smooth average of the losses (exponentially weighted with `beta`)\"\n",
" def __init__(self, beta=0.98): self.beta,self.count,self.val = beta,0,tensor(0.)\n",
" def reset(self): self.count,self.val = 0,tensor(0.)\n",
" def accumulate(self, learn):\n",
" self.count += 1\n",
" self.val = torch.lerp(to_detach(learn.loss.mean()), self.val, self.beta)\n",
" @property\n",
" def value(self): return self.val/(1-self.beta**self.count)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3b1e0e25",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L517){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### AvgSmoothLoss\n",
"\n",
"```python\n",
"\n",
"def AvgSmoothLoss(\n",
" beta:float=0.98\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Smooth average of the losses (exponentially weighted with `beta`)*"
],
"text/plain": [
"```python\n",
"\n",
"def AvgSmoothLoss(\n",
" beta:float=0.98\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Smooth average of the losses (exponentially weighted with `beta`)*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(AvgSmoothLoss, title_level=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1935bd0c",
"metadata": {},
"outputs": [],
"source": [
"tst = AvgSmoothLoss()\n",
"t = torch.randn(100)\n",
"tst.reset()\n",
"val = tensor(0.)\n",
"for i in range(4): \n",
" learn.loss = t[i*25:(i+1)*25].mean()\n",
" tst.accumulate(learn)\n",
" val = val*0.98 + t[i*25:(i+1)*25].mean()*(1-0.98)\n",
" test_close(val/(1-0.98**(i+1)), tst.value)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a60ac36f",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class ValueMetric(Metric):\n",
" \"Use to include a pre-calculated metric value (for instance calculated in a `Callback`) and returned by `func`\"\n",
" def __init__(self, func, metric_name=None): store_attr('func, metric_name')\n",
"\n",
" @property\n",
" def value(self): return self.func()\n",
"\n",
" @property\n",
" def name(self): return self.metric_name if self.metric_name else self.func.__name__"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c66ec345",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L528){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### ValueMetric\n",
"\n",
"```python\n",
"\n",
"def ValueMetric(\n",
" func, metric_name:NoneType=None\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Use to include a pre-calculated metric value (for instance calculated in a `Callback`) and returned by `func`*"
],
"text/plain": [
"```python\n",
"\n",
"def ValueMetric(\n",
" func, metric_name:NoneType=None\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Use to include a pre-calculated metric value (for instance calculated in a `Callback`) and returned by `func`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(ValueMetric, title_level=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "735fbd56",
"metadata": {},
"outputs": [],
"source": [
"def metric_value_fn(): return 5e-3\n",
"\n",
"vm = ValueMetric(metric_value_fn, 'custom_value_metric')\n",
"test_eq(vm.value, 5e-3)\n",
"test_eq(vm.name, 'custom_value_metric')\n",
"\n",
"vm = ValueMetric(metric_value_fn)\n",
"test_eq(vm.name, 'metric_value_fn')"
]
},
{
"cell_type": "markdown",
"id": "bb29dd31",
"metadata": {},
"source": [
"## Recorder --"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e2d25e04",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"from fastprogress.fastprogress import format_time"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a091178b",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"def _maybe_item(t):\n",
" t = t.value\n",
" try: return t.item()\n",
" except: return t"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "244e6cf1",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class Recorder(Callback):\n",
" \"Callback that registers statistics (lr, loss and metrics) during training\"\n",
" _stateattrs=('lrs','iters','losses','values')\n",
" remove_on_fetch,order = True,50\n",
"\n",
" def __init__(self, add_time=True, train_metrics=False, valid_metrics=True, beta=0.98):\n",
" store_attr('add_time,train_metrics,valid_metrics')\n",
" self.loss,self.smooth_loss = AvgLoss(),AvgSmoothLoss(beta=beta)\n",
"\n",
" def before_fit(self):\n",
" \"Prepare state for training\"\n",
" self.lrs,self.iters,self.losses,self.values = [],[],[],[]\n",
" names = self.metrics.attrgot('name')\n",
" if self.train_metrics and self.valid_metrics:\n",
" names = L('loss') + names\n",
" names = names.map('train_{}') + names.map('valid_{}')\n",
" elif self.valid_metrics: names = L('train_loss', 'valid_loss') + names\n",
" else: names = L('train_loss') + names\n",
" if self.add_time: names.append('time')\n",
" self.metric_names = 'epoch'+names\n",
" self.smooth_loss.reset()\n",
"\n",
" def after_batch(self):\n",
" \"Update all metrics and records lr and smooth loss in training\"\n",
" if len(self.yb) == 0: return\n",
" mets = self._train_mets if self.training else self._valid_mets\n",
" for met in mets: met.accumulate(self.learn)\n",
" if not self.training: return\n",
" self.lrs.append(self.opt.hypers[-1]['lr'])\n",
" self.losses.append(self.smooth_loss.value)\n",
" self.learn.smooth_loss = self.smooth_loss.value\n",
"\n",
" def before_epoch(self):\n",
" \"Set timer if `self.add_time=True`\"\n",
" self.cancel_train,self.cancel_valid = False,False\n",
" if self.add_time: self.start_epoch = time.time()\n",
" self.log = L(getattr(self, 'epoch', 0))\n",
"\n",
" def before_train (self): self._train_mets[1:].map(~Self.reset())\n",
" def before_validate(self): self._valid_mets.map(~Self.reset())\n",
" def after_train (self): self.log += self._train_mets.map(_maybe_item)\n",
" def after_validate(self): self.log += self._valid_mets.map(_maybe_item)\n",
" def after_cancel_train(self): self.cancel_train = True\n",
" def after_cancel_validate(self): self.cancel_valid = True\n",
"\n",
" def after_epoch(self):\n",
" \"Store and log the loss/metric values\"\n",
" self.learn.final_record = self.log[1:].copy()\n",
" self.values.append(self.learn.final_record)\n",
" if self.add_time: self.log.append(format_time(time.time() - self.start_epoch))\n",
" self.logger(self.log)\n",
" self.iters.append(self.smooth_loss.count)\n",
"\n",
" @property\n",
" def _train_mets(self):\n",
" if getattr(self, 'cancel_train', False): return L()\n",
" return L(self.smooth_loss) + (self.metrics if self.train_metrics else L())\n",
"\n",
" @property\n",
" def _valid_mets(self):\n",
" if getattr(self, 'cancel_valid', False): return L()\n",
" return (L(self.loss) + self.metrics if self.valid_metrics else L())\n",
"\n",
" def plot_loss(self, skip_start=5, with_valid=True, log=False, show_epochs=False, ax=None):\n",
" if not ax:\n",
" ax=plt.gca()\n",
" if log:\n",
" ax.loglog(list(range(skip_start, len(self.losses))), self.losses[skip_start:], label='train')\n",
" else:\n",
" ax.plot(list(range(skip_start, len(self.losses))), self.losses[skip_start:], label='train')\n",
" if show_epochs:\n",
" for x in self.iters:\n",
" ax.axvline(x, color='grey', ls=':')\n",
" ax.set_ylabel('loss')\n",
" ax.set_xlabel('steps')\n",
" ax.set_title('learning curve')\n",
" if with_valid:\n",
" idx = (np.array(self.iters)<skip_start).sum()\n",
" valid_col = self.metric_names.index('valid_loss') - 1 \n",
" ax.plot(self.iters[idx:], L(self.values[idx:]).itemgot(valid_col), label='valid')\n",
" ax.legend()\n",
" return ax"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "653acadd",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"add_docs(Recorder,\n",
" before_train = \"Reset loss and metrics state\",\n",
" after_train = \"Log loss and metric values on the training set (if `self.training_metrics=True`)\",\n",
" before_validate = \"Reset loss and metrics state\",\n",
" after_validate = \"Log loss and metric values on the validation set\",\n",
" after_cancel_train = \"Ignore training metrics for this epoch\",\n",
" after_cancel_validate = \"Ignore validation metrics for this epoch\",\n",
" plot_loss = \"Plot the losses from `skip_start` and onward. Optionally `log=True` for logarithmic axis, `show_epochs=True` for indicate epochs and a matplotlib axis `ax` to plot on.\")\n",
"\n",
"if Recorder not in defaults.callbacks: defaults.callbacks.append(Recorder)"
]
},
{
"cell_type": "markdown",
"id": "4e9f77b9",
"metadata": {},
"source": [
"By default, metrics are computed on the validation set only, although that can be changed by adjusting `train_metrics` and `valid_metrics`. `beta` is the weight used to compute the exponentially weighted average of the losses (which gives the `smooth_loss` attribute to `Learner`).\n",
"\n",
"The `logger` attribute of a `Learner` determines what happens to those metrics. By default, it just print them:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "81b7d958",
"metadata": {},
"outputs": [],
"source": [
"#Test printed output\n",
"def tst_metric(out, targ): return F.mse_loss(out, targ)\n",
"learn = synth_learner(n_train=5, metrics=tst_metric, default_cbs=False, cbs=[TrainEvalCallback, Recorder])\n",
"# pat = r\"[tensor\\(\\d.\\d*\\), tensor\\(\\d.\\d*\\), tensor\\(\\d.\\d*\\), 'dd:dd']\"\n",
"pat = r\"\\[\\d, \\d+.\\d+, \\d+.\\d+, \\d+.\\d+, '\\d\\d:\\d\\d'\\]\"\n",
"test_stdout(lambda: learn.fit(1), pat, regex=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c410ea95",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"class TestRecorderCallback(Callback):\n",
" order=51\n",
" def before_fit(self): \n",
" self.train_metrics,self.add_time = self.recorder.train_metrics,self.recorder.add_time\n",
" self.beta = self.recorder.smooth_loss.beta\n",
" for m in self.metrics: assert isinstance(m, Metric)\n",
" test_eq(self.recorder.smooth_loss.val, 0.)\n",
" #To test what the recorder logs, we use a custom logger function.\n",
" self.learn.logger = self.test_log\n",
" self.old_smooth,self.count = tensor(0.),0\n",
" \n",
" def after_batch(self):\n",
" if self.training:\n",
" self.count += 1\n",
" test_eq(len(self.recorder.lrs), self.count)\n",
" test_eq(self.recorder.lrs[-1], self.opt.hypers[-1]['lr'])\n",
" test_eq(len(self.recorder.losses), self.count)\n",
" smooth = (1 - self.beta**(self.count-1)) * self.old_smooth * self.beta + self.loss * (1-self.beta)\n",
" smooth /= 1 - self.beta**self.count\n",
" test_close(self.recorder.losses[-1], smooth, eps=1e-4)\n",
" test_close(self.smooth_loss, smooth, eps=1e-4)\n",
" self.old_smooth = self.smooth_loss\n",
" self.bs += find_bs(self.yb)\n",
" if not self.training: test_eq(self.recorder.loss.count, self.bs)\n",
" if self.train_metrics or not self.training: \n",
" for m in self.metrics: test_eq(m.count, self.bs)\n",
" self.losses.append(self.loss.detach().cpu())\n",
" \n",
" def before_epoch(self): \n",
" if self.add_time: self.start_epoch = time.time()\n",
" self.log = [self.epoch]\n",
"\n",
" def before_train(self):\n",
" self.bs = 0\n",
" self.losses = []\n",
" for m in self.recorder._train_mets: test_eq(m.count, self.bs)\n",
" \n",
" def after_train(self):\n",
" mean = tensor(self.losses).mean()\n",
" self.log += [self.smooth_loss, mean] if self.train_metrics else [self.smooth_loss]\n",
" test_close(self.log, self.recorder.log)\n",
" self.losses = []\n",
" \n",
" def before_validate(self):\n",
" self.bs = 0\n",
" self.losses = []\n",
" for m in [self.recorder.loss] + self.metrics: test_eq(m.count, self.bs)\n",
" \n",
" def test_log(self, log):\n",
" res = tensor(self.losses).mean()\n",
" self.log += [res, res]\n",
" if self.add_time: self.log.append(format_time(time.time() - self.start_epoch))\n",
" test_close(log[:-1], self.log[:-1])\n",
" test_eq(log[-1], self.log[-1])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b98c27d5",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"def _get_learn():\n",
" return synth_learner(n_train=5, metrics = tst_metric, default_cbs=False, cbs=[TrainEvalCallback, Recorder, TestRecorderCallback])\n",
"\n",
"learn = _get_learn()\n",
"learn.fit(1)\n",
"test_eq(learn.recorder.metric_names, ['epoch', 'train_loss', 'valid_loss', 'tst_metric', 'time'])\n",
"\n",
"learn = _get_learn()\n",
"learn.recorder.train_metrics=True\n",
"learn.fit(1)\n",
"test_eq(learn.recorder.metric_names, \n",
" ['epoch', 'train_loss', 'train_tst_metric', 'valid_loss', 'valid_tst_metric', 'time'])\n",
"\n",
"learn = _get_learn()\n",
"learn.recorder.add_time=False\n",
"learn.fit(1)\n",
"test_eq(learn.recorder.metric_names, ['epoch', 'train_loss', 'valid_loss', 'tst_metric'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "09a34361",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0, 12.158802032470703, 10.726703643798828, 10.726703643798828, '00:00']\n"
]
}
],
"source": [
"#| hide\n",
"#Test numpy metric\n",
"def tst_metric_np(out, targ): return F.mse_loss(out, targ).numpy()\n",
"learn = synth_learner(n_train=5, metrics=tst_metric_np)\n",
"learn.fit(1)"
]
},
{
"cell_type": "markdown",
"id": "d34e0861",
"metadata": {},
"source": [
"### Internals"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "84b9286f",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L557){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Recorder.before_fit\n",
"\n",
"```python\n",
"\n",
"def before_fit(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Prepare state for training*"
],
"text/plain": [
"```python\n",
"\n",
"def before_fit(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Prepare state for training*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Recorder.before_fit)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8f1713f0",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L580){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Recorder.before_epoch\n",
"\n",
"```python\n",
"\n",
"def before_epoch(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Set timer if `self.add_time=True`*"
],
"text/plain": [
"```python\n",
"\n",
"def before_epoch(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Set timer if `self.add_time=True`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Recorder.before_epoch)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "926aa0de",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L587){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Recorder.before_validate\n",
"\n",
"```python\n",
"\n",
"def before_validate(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Reset loss and metrics state*"
],
"text/plain": [
"```python\n",
"\n",
"def before_validate(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Reset loss and metrics state*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Recorder.before_validate)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e5189b36",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L570){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Recorder.after_batch\n",
"\n",
"```python\n",
"\n",
"def after_batch(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Update all metrics and records lr and smooth loss in training*"
],
"text/plain": [
"```python\n",
"\n",
"def after_batch(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Update all metrics and records lr and smooth loss in training*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Recorder.after_batch)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f9c80fe4",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L593){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Recorder.after_epoch\n",
"\n",
"```python\n",
"\n",
"def after_epoch(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Store and log the loss/metric values*"
],
"text/plain": [
"```python\n",
"\n",
"def after_epoch(\n",
" \n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Store and log the loss/metric values*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Recorder.after_epoch)"
]
},
{
"cell_type": "markdown",
"id": "893be655",
"metadata": {},
"source": [
"### Plotting tools"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cca018fd",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L611){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Recorder.plot_loss\n",
"\n",
"```python\n",
"\n",
"def plot_loss(\n",
" skip_start:int=5, with_valid:bool=True, log:bool=False, show_epochs:bool=False, ax:NoneType=None\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Plot the losses from `skip_start` and onward. Optionally `log=True` for logarithmic axis, `show_epochs=True` for indicate epochs and a matplotlib axis `ax` to plot on.*"
],
"text/plain": [
"```python\n",
"\n",
"def plot_loss(\n",
" skip_start:int=5, with_valid:bool=True, log:bool=False, show_epochs:bool=False, ax:NoneType=None\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Plot the losses from `skip_start` and onward. Optionally `log=True` for logarithmic axis, `show_epochs=True` for indicate epochs and a matplotlib axis `ax` to plot on.*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Recorder.plot_loss)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8e0c219c",
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVKJJREFUeJzt3XdYFNf+P/D3LG3pxUKTpigoKmIHu6BYQiyxG1uMN81EQzRqvr9ETe6NsSTqjdwkGttNNEWjxsSIwYLYG6LYUJAmUlW6tN35/eF1ExQQEJhd5v16nnme7OzZ2c9hjLw9Z2aOIIqiCCIiIiIZUUhdABEREVFDYwAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiAMCWLVsgCAISEhKkLqVK4eHhEAQB4eHhUpdCRDqMAYiIiIhkR1/qAoiIaqJv3754+PAhDA0NpS6FiHQYR4CISFIFBQU1aq9QKKBUKqFQNL6/vkRRxMOHD6Uug0gWGt/fIERUp/bv348+ffrA1NQU5ubmGD58OK5evVquzeXLlzF9+nS0bNkSSqUSdnZ2eOWVV3Dv3r1y7ZYsWQJBEHDt2jVMmjQJ1tbW6N27NwDA1dUVL7zwAo4fP47u3btDqVSiZcuW+O9//1vuGBVdA9S/f3+0b98e165dw4ABA2BiYgJHR0esWLHiqf4kJibixRdfhKmpKZo3b453330XBw4cqPZ1RSkpKZg5cyYcHBxgZGQENzc3vPHGGygpKSnXxydVdI3V4z4fOHAAXbt2hbGxMb755hu0b98eAwYMeOoYarUajo6OGDNmTLl9a9asgZeXF5RKJWxtbfHaa6/hwYMHz+wLkZxxCoyIKvXdd99h2rRpCAwMxPLly1FYWIivvvoKvXv3xsWLF+Hq6goACAsLw+3btzFjxgzY2dnh6tWrWL9+Pa5evYrTp08/FQjGjh2L1q1b49NPP4Uoipr9sbGxGDNmDGbOnIlp06Zh06ZNmD59Orp06QIvL68qa33w4AGGDBmC0aNHY9y4cdi5cycWLFiADh06YOjQoQAejTYNHDgQqampmDNnDuzs7LB9+3YcOXKkWj+Pu3fvonv37sjOzsY//vEPeHp6IiUlBTt37kRhYWGtpuViYmIwceJEvPbaa5g1axY8PDwwfvx4LFmyBGlpabCzs9O0PX78OO7evYsJEyZo9r322mvYsmULZsyYgXfeeQfx8fFYt24dLl68iBMnTsDAwKDGNRHJgkhEJIri5s2bRQBifHy8KIqimJeXJ1pZWYmzZs0q1y4tLU20tLQst7+wsPCp4/3www8iADEiIkKzb/HixSIAceLEiU+1d3Fxeap9RkaGaGRkJL733nuafUeOHBEBiEeOHNHs69evnwhA/O9//6vZV1xcLNrZ2YkvvfSSZt/nn38uAhD37Nmj2ffw4UPR09PzqWNWZOrUqaJCoRDPnTv31HtqtbpcH5/05M/3730ODQ0t1zYmJkYEIH755Zfl9r/55puimZmZ5ud97NgxEYC4bdu2cu1CQ0Mr3E9Ef+EUGBFVKCwsDNnZ2Zg4cSKysrI0m56eHnr06FFu1MTY2Fjz30VFRcjKykLPnj0BAJGRkU8d+/XXX6/wO9u1a4c+ffpoXjdr1gweHh64ffv2M+s1MzPDyy+/rHltaGiI7t27l/tsaGgoHB0d8eKLL2r2KZVKzJo165nHV6vV2LNnD4KCgtC1a9en3q9o2qs63NzcEBgYWG5fmzZt0KlTJ/z000+afSqVCjt37kRQUJDm571jxw5YWlpi0KBB5c5Rly5dYGZmVu2RLSI54hQYEVXo1q1bAICBAwdW+L6FhYXmv+/fv4+lS5fixx9/REZGRrl2OTk5T33Wzc2twmM6Ozs/tc/a2rpa17O0aNHiqRBibW2Ny5cva14nJiaiVatWT7Vzd3d/5vEzMzORm5uL9u3bP7NtTVT2sxg/fjw++OADpKSkwNHREeHh4cjIyMD48eM1bW7duoWcnBw0b968wmM8eS6I6C8MQERUIbVaDeDRdUB/vw7lMX39v/76GDduHE6ePIn58+ejU6dOMDMzg1qtxpAhQzTH+bu/jxj9nZ6eXoX7xb9dJ1SZ5/lsXapsJEilUlW4v7Kfxfjx47Fo0SLs2LEDc+fOxc8//wxLS0sMGTJE00atVqN58+bYtm1bhcdo1qxZDasnkg8GICKqUKtWrQAAzZs3R0BAQKXtHjx4gEOHDmHp0qX46KOPNPsfjyBpExcXF1y7dg2iKJYLKrGxsc/8bLNmzWBhYYErV65U2c7a2hoAkJ2dDSsrK83+xMTEGtXq5uaG7t2746effsLs2bOxa9cujBw5EkZGRpo2rVq1wsGDB9GrV69KgxQRVYzXABFRhQIDA2FhYYFPP/0UpaWlT72fmZkJ4K+RlydHWtasWVPvNdZUYGAgUlJSsHfvXs2+oqIibNiw4ZmfVSgUGDlyJH777TecP3/+qfcf9/9xcIyIiNC8V1BQgK1bt9a43vHjx+P06dPYtGkTsrKyyk1/AY9G3lQqFT755JOnPltWVobs7OwafyeRXHAEiIgqZGFhga+++gpTpkxB586dMWHCBDRr1gxJSUnYt28fevXqhXXr1sHCwgJ9+/bFihUrUFpaCkdHR/z555+Ij4+XugtPee2117Bu3TpMnDgRc+bMgb29PbZt2walUgng2Rcyf/rpp/jzzz/Rr18//OMf/0Dbtm2RmpqKHTt24Pjx47CyssLgwYPh7OyMmTNnYv78+dDT08OmTZs0P7uaGDduHObNm4d58+bBxsbmqZG4fv364bXXXsOyZcsQFRWFwYMHw8DAALdu3cKOHTuwdu3acs8MIqK/MAARUaUmTZoEBwcHfPbZZ1i5ciWKi4vh6OiIPn36YMaMGZp227dvx9tvv42QkBCIoojBgwdj//79cHBwkLD6p5mZmeHw4cN4++23sXbtWpiZmWHq1Knw8/PDSy+9pAlClXF0dMSZM2fw4YcfYtu2bcjNzYWjoyOGDh0KExMTAICBgQF2796NN998Ex9++CHs7Owwd+5cWFtbl/uZVUeLFi3g5+eHEydO4NVXX63wmT5ff/01unTpgm+++QYffPAB9PX14erqipdffhm9evWq0fcRyYkgNvQVgkREWmbNmjV49913cefOHTg6OkpdDhE1AAYgIpKVhw8fPvXcIh8fH6hUKty8eVPCyoioIXEKjIhkZfTo0XB2dkanTp2Qk5OD77//Hjdu3Kj0VnIiapwYgIhIVgIDA/Htt99i27ZtUKlUaNeuHX788cen7rAiosaNU2BEREQkO3wOEBEREckOAxARERHJDq8BqoBarcbdu3dhbm5e6xWeiYiIqGGJooi8vDw4ODhAoah6jIcBqAJ3796Fk5OT1GUQERFRLSQnJ6NFixZVtmEAqoC5uTmARz9ACwsLiashIiKi6sjNzYWTk5Pm93hVJA1AERERWLlyJS5cuIDU1FTs3r0bI0eOrLDt66+/jm+++QarV6/G3LlzqzxuSEgIVq5cibS0NHh7e+PLL79E9+7dq13X42kvCwsLBiAiIiIdU53LVyS9CLqgoADe3t4ICQmpst3u3btx+vTpaq0r9NNPPyE4OBiLFy9GZGQkvL29ERgYiIyMjLoqm4iIiHScpAFo6NCh+Oc//4lRo0ZV2iYlJQVvv/02tm3bVuFCgE/64osvMGvWLMyYMQPt2rXD119/DRMTE2zatKkuSyciIiIdptW3wavVakyZMgXz58+Hl5fXM9uXlJTgwoULCAgI0OxTKBQICAjAqVOnKv1ccXExcnNzy21ERETUeGn1RdDLly+Hvr4+3nnnnWq1z8rKgkqlgq2tbbn9tra2uHHjRqWfW7ZsGZYuXfpctRIRET2LWq1GSUmJ1GXoLAMDA+jp6dXJsbQ2AF24cAFr165FZGRkvT+LZ9GiRQgODta8fnwVORERUV0pKSlBfHw81Gq11KXoNCsrK9jZ2T13NtDaAHTs2DFkZGTA2dlZs0+lUuG9997DmjVrkJCQ8NRnmjZtCj09PaSnp5fbn56eDjs7u0q/y8jICEZGRnVWOxER0d+JoojU1FTo6enBycnpmQ/po6eJoojCwkLNTU329vbPdTytDUBTpkwpdy0P8GgV5ylTpmDGjBkVfsbQ0BBdunTBoUOHNLfTq9VqHDp0CLNnz67vkomIiCpUVlaGwsJCODg4wMTEROpydJaxsTEAICMjA82bN3+u6TBJA1B+fj5iY2M1r+Pj4xEVFQUbGxs4OzujSZMm5dobGBjAzs4OHh4emn3+/v4YNWqUJuAEBwdj2rRp6Nq1K7p37441a9agoKCg0tBERERU31QqFYBH/1Cn5/M4QJaWlupuADp//jwGDBigef34Opxp06Zhy5Yt1TpGXFwcsrKyNK/Hjx+PzMxMfPTRR0hLS0OnTp0QGhr61IXRREREDY3rSz6/uvoZCqIoinVypEYkNzcXlpaWyMnJ4ZOgiYjouRUVFSE+Ph5ubm5QKpVSl6PTqvpZ1uT3N6/CIiIiogbh6uqKNWvWSF0GAC2+CJqIiIik179/f3Tq1KlOgsu5c+dgamr6/EXVAQagBnYzPQ8mhnpoYc27AIiISPeJogiVSgV9/WdHimbNmjVARdXDKbAGtPlEPIasicDy0BipSyEiInqm6dOn4+jRo1i7di0EQYAgCNiyZQsEQcD+/fvRpUsXGBkZ4fjx44iLi8OIESNga2sLMzMzdOvWDQcPHix3vCenwARBwLfffotRo0bBxMQErVu3xt69exukbwxADaiHWxOIAH67dBfRd3KkLoeIiCQiiiIKS8ok2Wpy79PatWvh6+uLWbNmITU1FampqZqVEhYuXIjPPvsM169fR8eOHZGfn49hw4bh0KFDuHjxIoYMGYKgoCAkJSVV+R1Lly7FuHHjcPnyZQwbNgyTJ0/G/fv3n+vnWx2cAmtA7RwsMMLbAXui7mLFgRv4bmYPqUsiIiIJPCxVod1HByT57msfB8LEsHq//i0tLWFoaAgTExPNigqP19b8+OOPMWjQIE1bGxsbeHt7a15/8skn2L17N/bu3Vvlw4inT5+OiRMnAgA+/fRT/Pvf/8bZs2cxZMiQGvetJjgC1MDeG+wBAz0Bx25l4URs1rM/QEREpIW6du1a7nV+fj7mzZuHtm3bwsrKCmZmZrh+/fozR4A6duyo+W9TU1NYWFholruoTxwBamBONiaY3MMFW04mYHnoDfz6Vi8+GIuISGaMDfRw7eNAyb67Ljx5N9e8efMQFhaGVatWwd3dHcbGxhgzZgxKSkqqPI6BgUG514IgNMiCsQxAEpg90B07zifj8p0c/BGdhuEdn29BNyIi0i2CIFR7GkpqhoaGmqU8qnLixAlMnz4do0aNAvBoRKiihcu1BafAJNDUzAiv9mkJAFj1ZwxKVfWfdImIiGrD1dUVZ86cQUJCArKysiodnWndujV27dqFqKgoXLp0CZMmTWqQkZzaYgCSyKy+LdHE1BDxWQX4+Xyy1OUQERFVaN68edDT00O7du3QrFmzSq/p+eKLL2BtbQ0/Pz8EBQUhMDAQnTt3buBqq49rgVWgodYC23wiHkt/u4bm5kY4On8AjA3rZl6WiIi0C9cCqztcC6wRmNTDGS2sjZGRV4xNJ+KlLoeIiEg2GIAkZKSvh3mDPQAAXx+Nw4OCqq+UJyIiorrBACSxF70d0NbeAnlFZfhPeKzU5RAREckCA5DEFAoB7w95NAq09VQiUrIfSlwRERFR48cApAX6t2mGHm42KClTY03YTanLISIiavQYgLSAIAhYMNQTAPBL5B3cSs+TuCIiIqLGjQFIS3R2tkagly3UIrDiQIzU5RARETVqDEBaZH6gBxQCEHYtHRcS70tdDhERUaPFAKRF3JubY2wXJwDA8v0x4DMqiYiI6gcDkJaZO6g1jPQVOJtwH0diMqQuh4iI6Lm4urpizZo1mteCIGDPnj2Vtk9ISIAgCIiKiqrXuhiAtIy9pTGm+7kCAFaExkCl5igQERE1HqmpqRg6dKjUZTAAaaM3+reChVIfN9Ly8GtUitTlEBER1Rk7OzsYGRlJXQYDkDayMjHEG/3dAQCf/3kTxWUqiSsiIiI5Wr9+PRwcHKBWq8vtHzFiBF555RXExcVhxIgRsLW1hZmZGbp164aDBw9Wecwnp8DOnj0LHx8fKJVKdO3aFRcvXqyPrjyFAUhLTfdzha2FEVKyH+L700lSl0NERHVJFIGSAmm2GtxgM3bsWNy7dw9HjhzR7Lt//z5CQ0MxefJk5OfnY9iwYTh06BAuXryIIUOGICgoCElJ1fu9lZ+fjxdeeAHt2rXDhQsXsGTJEsybN6/GP87a0G+Qb6EaMzbUw9yANli0KxohR2IxrmsLmCsNpC6LiIjqQmkh8KmDNN/9wV3A0LRaTa2trTF06FBs374d/v7+AICdO3eiadOmGDBgABQKBby9vTXtP/nkE+zevRt79+7F7Nmzn3n87du3Q61WY+PGjVAqlfDy8sKdO3fwxhtv1K5vNcARIC02tksLtGxmivsFJdgQcVvqcoiISIYmT56MX375BcXFxQCAbdu2YcKECVAoFMjPz8e8efPQtm1bWFlZwczMDNevX6/2CND169fRsWNHKJVKzT5fX9966ceTOAKkxfT1FJg/2ANvbIvEt8fjMcXXFc3Mpb9wjIiInpOByaORGKm+uwaCgoIgiiL27duHbt264dixY1i9ejUAYN68eQgLC8OqVavg7u4OY2NjjBkzBiUlJfVReZ1iANJyQ9rbwdvJCpeSs/Hl4Vv4eER7qUsiIqLnJQjVnoaSmlKpxOjRo7Ft2zbExsbCw8MDnTt3BgCcOHEC06dPx6hRowA8uqYnISGh2sdu27YtvvvuOxQVFWlGgU6fPl3nfagIp8C0nCAIWDDEAwCw/UwSEu8VSFwRERHJzeTJk7Fv3z5s2rQJkydP1uxv3bo1du3ahaioKFy6dAmTJk166o6xqkyaNAmCIGDWrFm4du0a/vjjD6xatao+uvAUBiAd4NeqKfq2aYYytYjP/7wpdTlERCQzAwcOhI2NDWJiYjBp0iTN/i+++ALW1tbw8/NDUFAQAgMDNaND1WFmZobffvsN0dHR8PHxwf/93/9h+fLl9dGFpwgiF5x6Sm5uLiwtLZGTkwMLCwupywEAXEnJwQtfHgcA/P52b7R3tJS4IiIiqq6ioiLEx8fDzc2t3AW/VHNV/Sxr8vubI0A6or2jJV70fnTL5IoDMRJXQ0REpNsYgHTIvMEeMNATEHEzEydjs6Quh4iISGcxAOkQ5yYmmNTdGQCwPPQGOHtJRERUOwxAOmb2wNYwMdTDpTs52H8lTepyiIiIdBIDkI5pZm6EV/u0BACsOhCDMlX1bzckIiJpceT++dXVz5ABSAfN6uMGG1ND3M4qwM/n70hdDhERPYOenh4A6MQTkrVdYWEhAMDA4PnWx+SToHWQudIAswe44+Pfr2HNwZsY5eMIY0M9qcsiIqJK6Ovrw8TEBJmZmTAwMIBCwfGHmhJFEYWFhcjIyICVlZUmVNYWA5COmtzTGZtOxOPOg4fYfDIeb/Z3l7okIiKqhCAIsLe3R3x8PBITE6UuR6dZWVnBzs7uuY/DAKSjjPT1EDyoDYJ/voSvwuMwqbszrEwMpS6LiIgqYWhoiNatW3Ma7DkYGBg898jPY5KOwUVERCAoKAgODg4QBAF79uwp9/6SJUvg6ekJU1NTWFtbIyAgAGfOnKnymEuWLIEgCOU2T0/PeuyFdEZ0coSnnTnyisrwVXic1OUQEdEzKBQKKJVKbrXc6ir8ABIHoIKCAnh7eyMkJKTC99u0aYN169YhOjoax48fh6urKwYPHozMzMwqj+vl5YXU1FTNdvz48fooX3J6CgHv/2+h1C0nE5Ca81DiioiIiHSDpFNgQ4cOxdChQyt9/+8LrgGPFl3buHEjLl++DH9//0o/p6+vXyfzg7pggEdzdHezwdn4+1gTdgvLx3SUuiQiIiKtpzOXoZeUlGD9+vWwtLSEt7d3lW1v3boFBwcHtGzZEpMnT0ZSUlKV7YuLi5Gbm1tu0xWCIGDh0EdTfDsuJONWep7EFREREWk/rQ9Av//+O8zMzKBUKrF69WqEhYWhadOmlbbv0aMHtmzZgtDQUHz11VeIj49Hnz59kJdXeTBYtmwZLC0tNZuTk1N9dKXedHa2xuB2tlCLwEoulEpERPRMgqglj6UUBAG7d+/GyJEjy+0vKChAamoqsrKysGHDBhw+fBhnzpxB8+bNq3Xc7OxsuLi44IsvvsDMmTMrbFNcXIzi4mLN69zcXDg5OSEnJwcWFha17lNDis3Iw+DVEVCLwC9v+KGLi7XUJRERETWo3NxcWFpaVuv3t9aPAJmamsLd3R09e/bExo0boa+vj40bN1b781ZWVmjTpg1iY2MrbWNkZAQLC4tym65xb26OMV1aAOBCqURERM+i9QHoSWq1utxozbPk5+cjLi4O9vb29ViVdpgb0AaG+gqcjb+P8Jiq75QjIiKSM0kDUH5+PqKiohAVFQUAiI+PR1RUFJKSklBQUIAPPvgAp0+fRmJiIi5cuIBXXnkFKSkpGDt2rOYY/v7+WLduneb1vHnzcPToUSQkJODkyZMYNWoU9PT0MHHixIbuXoNzsDLGdD9XAI9GgdRqjgIRERFVRNIAdP78efj4+MDHxwcAEBwcDB8fH3z00UfQ09PDjRs38NJLL6FNmzYICgrCvXv3cOzYMXh5eWmOERcXh6ysLM3rO3fuYOLEifDw8MC4cePQpEkTnD59Gs2aNWvw/knhzf6tYK7Ux420PPx6KUXqcoiIiLSS1lwErU1qchGVNgo5EouVB2LQwtoYh97rByN9LpRKRESNX6O6CJpq7pVebmhuboQ7Dx5i+5mqn4FEREQkRwxAjZCxoR7mBLQGAKw7HIv84jKJKyIiItIuDECN1LiuTmjZ1BT3CkqwIeK21OUQERFpFQagRspAT4F5gY8WSt1w7DYy86r/6AAiIqLGjgGoERva3g7eLSxRWKLCusO3pC6HiIhIazAANWKCIGDBkEcLpW4/m4Ske4USV0RERKQdGIAaOT/3pujTuilKVSI+D+NCqURERAADkCw8HgX6Neourt7NkbgaIiIi6TEAyUB7R0sEeTsAAFaEchSIiIiIAUgm3hvUBvoKAUdvZuJU3D2pyyEiIpIUA5BMuDY1xcTuzgCAz0JvgCugEBGRnDEAycjb/u4wNtDDpeRsHLiaJnU5REREkmEAkpHm5krM6uMGAFhxIAZlKrXEFREREUmDAUhmZvVtCRtTQ9zOLMCOC3ekLoeIiEgSDEAyY640wFsD3AEAaw7exMMSlcQVERERNTwGIBl6uaczHK2MkZ5bjC0nE6Quh4iIqMExAMmQkb4egge1AQB8FR6LnMJSiSsiIiJqWAxAMjXSxxEetubILSrDf47GSl0OERFRg2IAkik9hYD3h3gAALacSEBqzkOJKyIiImo4DEAyNtCzObq5WqO4TI21B29JXQ4REVGDYQCSMUEQsHDoo4VSfz6fjNiMfIkrIiIiahgMQDLXxcUGAW1toRaBVQe4UCoREckDAxDh/SEeUAhA6NU0XEx6IHU5RERE9Y4BiNDG1hwvdW4BAPhsPxdKJSKixo8BiAAA7w5qA0N9Bc7E30f4zUypyyEiIqpXDEAEAHCwMsY0XxcAwIrQGKjVHAUiIqLGiwGINN7s7w5zI31cT83F3kt3pS6HiIio3jAAkYa1qSFe798KAPB5WAxKytQSV0RERFQ/GIConBm9XNHM3AjJ9x9i+5lEqcshIiKqFwxAVI6JoT7m+LcGAHx5OBb5xWUSV0RERFT3GIDoKeO7OcGtqSnuFZTg22O3pS6HiIiozjEA0VMM9BR4b3AbAMCGiNvIyi+WuCIiIqK6xQBEFRrW3h4dHC1RUKLCusOxUpdDRERUpxiAqEIKxV8LpW47k4jk+4USV0RERFR3GICoUr3cm6JP66YoVYn4/E8ulEpERI0HAxBVacGQR6NAv166i2t3cyWuhoiIqG4wAFGV2jta4oWO9hBFYMWBG1KXQ0REVCcYgOiZ5g32gL5CQHhMJk7fvid1OURERM+NAYieybWpKSZ0dwIAfLb/BkSRC6USEZFuYwCiannHvzWMDfQQlZyNA1fTpS6HiIjouTAAUbU0N1diZm83AMDKAzdQpuJCqUREpLsYgKja/tGvJaxNDBCXWYBfIu9IXQ4REVGtSRqAIiIiEBQUBAcHBwiCgD179pR7f8mSJfD09ISpqSmsra0REBCAM2fOPPO4ISEhcHV1hVKpRI8ePXD27Nl66oG8WCgN8NYAdwDA6rBbKCpVSVwRERFR7UgagAoKCuDt7Y2QkJAK32/Tpg3WrVuH6OhoHD9+HK6urhg8eDAyMzMrPeZPP/2E4OBgLF68GJGRkfD29kZgYCAyMjLqqxuy8nJPFzhaGSMttwhbTyZIXQ4REVGtCKKW3NIjCAJ2796NkSNHVtomNzcXlpaWOHjwIPz9/Sts06NHD3Tr1g3r1q0DAKjVajg5OeHtt9/GwoULq1XL4+/JycmBhYVFjfvS2O28cAfzdlyChVIfx94fCEsTA6lLIiIiqtHvb525BqikpATr16+HpaUlvL29K21z4cIFBAQEaPYpFAoEBATg1KlTlR67uLgYubm55Taq3CgfR3jYmiO3qAxfHY2TuhwiIqIa0/oA9Pvvv8PMzAxKpRKrV69GWFgYmjZtWmHbrKwsqFQq2Nralttva2uLtLS0Sr9j2bJlsLS01GxOTk512ofGRk8hYH6gBwBg84l4pOUUSVwRERFRzWh9ABowYACioqJw8uRJDBkyBOPGjavz63kWLVqEnJwczZacnFynx2+M/Ns2R1cXaxSXqbH20E2pyyEiIqoRrQ9ApqamcHd3R8+ePbFx40bo6+tj48aNFbZt2rQp9PT0kJ5e/kF96enpsLOzq/Q7jIyMYGFhUW6jqgmCgIVDHy2U+vP5O4jLzJe4IiIiourT+gD0JLVajeLi4grfMzQ0RJcuXXDo0KFy7Q8dOgRfX9+GKlE2urraIKBtc6jUIlYdiJG6HCIiomqTNADl5+cjKioKUVFRAID4+HhERUUhKSkJBQUF+OCDD3D69GkkJibiwoULeOWVV5CSkoKxY8dqjuHv76+54wsAgoODsWHDBmzduhXXr1/HG2+8gYKCAsyYMaOhuycL8wM9IQjA/itpiErOlrocIiKiatGX8svPnz+PAQMGaF4HBwcDAKZNm4avv/4aN27cwNatW5GVlYUmTZqgW7duOHbsGLy8vDSfiYuLQ1ZWlub1+PHjkZmZiY8++ghpaWno1KkTQkNDn7owmuqGh505Rvu0wC+Rd7B8/w1sn9UDgiBIXRYREVGVtOY5QNqEzwGqmTsPCjFw1VGUqNTY+kp39GvTTOqSiIhIhhrlc4BIe7WwNsEUXxcAwPL9N6BWM1MTEZF2YwCiOvHWAHeYG+njWmoufrt8V+pyiIiIqsQARHXCxtQQr/VrCQD4/M+bKClTS1wRERFR5RiAqM680tsNzcyNkHS/ED+cTZK6HCIiokoxAFGdMTHUxzv+rQEAXx6+hYLiMokrIiIiqhgDENWpCd2c4NrEBFn5Jfj2WLzU5RAREVWIAYjqlIGeAu8NfrRQ6vqIONzLr/ip3URERFJiAKI6N7yDPdo7WqCgRIV1R2KlLoeIiOgpDEBU5xQKAQuGPFooddvpJCTfL5S4IiIiovIYgKhe9GndDL3cm6BEpcbqsJtSl0NERFQOAxDVm8ejQLujUnA9NVfiaoiIiP7CAET1pmMLKwzvaA9RBFYeiJG6HCIiIg0GIKpX8wZ7QF8h4PCNDJy5fU/qcoiIiAAwAFE9c2tqivHdnAAAn4XegChyoVQiIpIeAxDVuzn+rWFsoIeLSdn481q61OUQERExAFH9a26hxCu9XQE8uhaoTMWFUomISFoMQNQgXuvXClYmBojNyMeuyBSpyyEiIpljAKIGYaE0wFv93QEAqw/eRFGpSuKKiIhIzhiAqMFM8XWBg6USqTlF+O+pBKnLISIiGWMAogajNNDD3EFtAAAhR+KQ87BU4oqIiEiuGICoQb3UuQVaNzdDzsNSfHM0TupyiIhIphiAqEHpKQTMD/QAAGw6EY/03CKJKyIiIjliAKIGN6idLbq4WKOoVI01B29JXQ4REckQAxA1OEEQsHDoo4VSfz6fjLjMfIkrIiIiuWEAIkl0c7WBv2dzqNQiPv+TC6USEVHDYgAiycwf4gFBAP6ITsOl5GypyyEiIhlhACLJeNpZYJSPIwBgORdKJSKiBsQARJIKHtQGhnoKnIy7h2O3sqQuh4iIZIIBiCTVwtoEL/d0AfBoFEit5igQERHVPwYgktzsge4wM9LH1bu5+D06VepyiIhIBhiASHI2pob4R9+WAIDP/4xBSZla4oqIiKixYwAirTCztxuamhkh8V4hfjqXJHU5RETUyDEAkVYwNdLHHH93AMDaQ7EoKC6TuCJqrPKKSnHnQaHUZRCRxBiASGtM6O4MlyYmyMovxsbj8VKXQ43Qvsup6L8yHL2XH8Hkb08j7Fo6VLzwnkiWGIBIaxjoKfDe4EcLpa6PuI17+cUSV0SNRWZeMd74/gLe2h6JewUlAIATsfcw67/n0X/VEWyIuI2cwlKJqySihsQARFrlhQ728HKwQH5xGUKOxEldDuk4URTxa1QKBq8+iv1X0qCvEPCOf2scmdcfr/VrCUtjAyTff4h//XEdPZcdwge7o3EzPU/qsomoAQgiH7/7lNzcXFhaWiInJwcWFhZSlyM7ETczMXXTWRjqKXB4Xj+0sDaRuiTSQRm5Rfi/PVcQdi0dANDO3gIrx3aEl4Olps3DEhV+jUrBlpMJuJH2V/Dp5d4E03xd4d/WFnoKocFrJ6LaqcnvbwagCjAASUsURUz+9gxOxt3D6M6O+GJcJ6lLIh0iiiJ2X0zB0t+uIedhKQz0BLw9sDXe6N8KBnoVD3qLoogz8fex5UQC/ryWhseXBbWwNsY0X1eM6+oESxODBuwFEdUGA9BzYgCS3qXkbIwIOQFBAPbP6QNPO54Hera0nCJ8sDsah29kAAA6OFpi5diONfrzc+dBIb4/nYQfzyUh+3/XBRkb6GFUZ0dM93NFG1vzeqmdiJ4fA9BzYgDSDm9uu4A/otPg79kcG6d3k7oc0mKiKGLHhTv45PdryCsqg6GeAnMCWuO1vi2hX8moz7M8LFFh76UUbD5RfnrMr1UTTPfj9BiRNmIAek4MQNrhdmY+Bq2OgEotYsfrvujmaiN1SaSF7mY/xKJd0Th6MxMA4O1khZVjOtbZSM3j6bGtJxNw4Gr56bGpvi4Y39WZ02NEWoIB6DkxAGmPRbui8cPZJHRxscbO130hCPwXNz0iiiJ+PJeMf+27jvziMhjqKxA8qA1e7e1W61GfZ6loekxpoMAonxaY7ucKDztOjxFJqSa/vyW9DT4iIgJBQUFwcHCAIAjYs2eP5r3S0lIsWLAAHTp0gKmpKRwcHDB16lTcvXu3ymMuWbIEgiCU2zw9Peu5J1Rf5ga0htJAgQuJD3DweobU5ZCWuPOgEFM3ncWiXdHILy5DZ2cr/PFOH7zer1W9hR8AaGFtgoVDPXF6kT+Wv9QBnnbmKCpV44ezSQhcE4GJ60/jwNU0PlyRSAdIGoAKCgrg7e2NkJCQp94rLCxEZGQkPvzwQ0RGRmLXrl2IiYnBiy+++Mzjenl5ITU1VbMdP368PsqnBmBrocQrvdwAACtCb/AXi8yp1SK+O52IwNUROHYrC0b6Cvy/4W2x43U/uDc3a7A6lAZ6GN/NGfvn9MFP/+iJoe3toBCAU7fv4bXvLqDviiP45mgcsgtLGqwmIqoZrZkCEwQBu3fvxsiRIyttc+7cOXTv3h2JiYlwdnausM2SJUuwZ88eREVF1boWToFpl5yHpei74ghyHpZixZiOGNfVSeqSSAJJ9wqx4JfLOHX7HgCgm6s1VozxhltTU4kreyQl+yG+P52IH84+OT3miGl+rryTkagB6MwUWE3l5ORAEARYWVlV2e7WrVtwcHBAy5YtMXnyZCQlVb26eHFxMXJzc8ttpD0sjQ3w1oBWAIA1YTdRVKqSuCJqSGq1iC0n4hG4JgKnbt+DsYEeFge1w0//8NWa8AMAjlbGWDDk0fTYipc6oq29xf+mx5IxZM0xTFh/CqFX0lCmUktdKhFBh0aAioqK0KtXL3h6emLbtm2VHmf//v3Iz8+Hh4cHUlNTsXTpUqSkpODKlSswN6/4AsUlS5Zg6dKlT+3nCJD2KCpVYcCqcKTmFOH/hrXFrL4tpS6JGkBCVgHe/+UyzsbfBwD0bGmD5S91hEsT7Qk+lRFFEecSHmDLyXgcuPrXoquOVsaY4uuCCd2cYGViKHGVRI2LTt4FVlUAKi0txUsvvYQ7d+4gPDy8RqEkOzsbLi4u+OKLLzBz5swK2xQXF6O4+K+FN3Nzc+Hk5MQApGV+PpeM93+5DCsTA0S8PwAWSt563Fip1CK2nEzAygM3UFSqhomhHhYN9cTkHi5Q6OCzd+7+bXrsAafHiOpNTQKQfgPVVGulpaUYN24cEhMTcfjw4RoHEisrK7Rp0waxsbGVtjEyMoKRkdHzlkr1bHRnR6w/dhuxGfn45mgc5gfy7r7GKC4zH+/vvIwLiQ8APHrw4PKXOsLJRnfXhHOwMsb7Qzzxjn9r7I26i80nE3A9NRc/nE3GD2eT0bOlDab7uSKgrW293sVGRH/R6v/THoefW7du4eDBg2jSpEmNj5Gfn4+4uDjY29vXQ4XUkPT1FJgf6AEA2Hg8Hhm5RRJXRHVJpRaxPiIOw9Yew4XEBzAz0senozpg26s9dDr8/J3SQA/jujnhj3d64+fXfDGsgx30FAJO376P17+PRL+V4fj6aBweFPDuMaL6JukIUH5+frmRmfj4eERFRcHGxgb29vYYM2YMIiMj8fvvv0OlUiEtLQ0AYGNjA0PDR3Pn/v7+GDVqFGbPng0AmDdvHoKCguDi4oK7d+9i8eLF0NPTw8SJExu+g1TnBrezRWdnK0QmZWPtoVv416gOUpdEdSA2Iw/zdlxGVHI2AKBP66b47KWOcLQylraweiIIArq72aC7m0256bGU7If4bP8NrA67qZkea2vP6TGi+iDpNUDh4eEYMGDAU/unTZuGJUuWwM3NrcLPHTlyBP379wcAuLq6Yvr06ViyZAkAYMKECYiIiMC9e/fQrFkz9O7dG//617/QqlWratfF2+C125nb9zB+/WnoKQQcDO6nVXcCUc2UqdRYf+w21hy8hZIyNcyN9PHhC+0wtmsL2T31u6hUhb2X7mLLiQRcS/3rTtQebjaY0YvTY0TVoZMXQWsTBiDtN2PzWRyJycTwjvYImdRZ6nKoFmLS8jB/5yVcvpMDABjg0Qyfju4Ae8vGOepTXaIo4nziA2w5kYDQvz1V2sFSiSm+rpjQzQnWprx7jKgiDEDPiQFI+11PzcWwfx+DKAJ7Z/dCxxZWUpdE1VSqUuPr8Dj8+/AtlKpEWCj1sTjIC6M7O8pu1OdZUnMeT48l4/7/rgsy0ldgZKdH02PtHPj3E9HfMQA9JwYg3RD8UxR2XUxBL/cm2PZqT6nLoWq4npqLeTsu4erdR1M8AW1t8a9R7WFroZS4Mu1WVKrCb5fuYsvJBM3PDgC6u9lghp8rBrXj9BgRwAD03BiAdEPy/UL4f34UJSo1vpvZHX1aN5O6JKpESZka/wmPxbrDsShTi7AyMcDSF73worcDR31qQBRFXEh8gM0nExB6pfz02Mu+LpjQzRk2nB4jGav3pTC2bt2Kffv2aV6///77sLKygp+fHxITE2tzSKIac7IxweSej9aEWx56A2oulKqVrqTk4MV1x7Hm4C2UqUUEetniz3f7YkQnTnnVlCAI6Opqg5BJnXF8wQDMHuAOG1ND3M0pworQGPguO4T3d17C1bs5UpdKpPVqNQLk4eGBr776CgMHDsSpU6cQEBCA1atX4/fff4e+vj527dpVH7U2GI4A6Y57+cXotzIc+cVl+HKiD4K8HaQuif6nuEyFdYdj8Z/wOKjUImxMDbH0RS+80NGewacOVTU9Nt3PFYM5PUYyUu9TYCYmJrhx4wacnZ2xYMECpKam4r///S+uXr2K/v37IzMzs9bFawMGIN2y9uAtrD54E65NTBAW3A8G/MtecpfvZGPejku4mZ4PABjewR5LR3ihqRmfuF5fHk+PbTmZgP1PTI9N7umCid05PUaNX71PgZmZmeHevXsAgD///BODBg0CACiVSjx8+LA2hySqtVf7uKGpmSES7hXix3PJUpcja0WlKiwPvYFR/zmJm+n5aGJqiP9M7oyQyZ0ZfurZ4+mxdX+bHmvyv+mxlQdi0JPTY0Tl1GoEaPLkybhx4wZ8fHzwww8/ICkpCU2aNMHevXvxwQcf4MqVK/VRa4PhCJDu2XoyAYv3XkUzcyMcnd8fJoZav8xdoxOZ9ADv77yM2IxHoz4vejtgyYteHHWQUFGpCr9fTsWWk/G4kvK36TFXG0zzc0WgF6fHqHGp9xGgkJAQ+Pr6IjMzE7/88otmja4LFy5wyQmSxMTuznC2MUFmXjE2HY+XuhxZKSpV4dM/rmPMVycRm5GPpmZG+GZKF/x7og/Dj8SUBnoY06UFfpvdG7+84YsXOtpDXyHgbMJ9vLU9En1WHEHIkVjNM4aI5IS3wVeAI0C66deoFMz5MQrmRvqIeH8An5bbAM4n3Mf7Oy/jdlYBAGC0jyM+CmoHKxP+7LVVWk4Rtp1JxPYzSbj3v+BjqK/ACG8HTPNzRXtHS4krJKq9er8IOjQ0FGZmZujduzeARyNCGzZsQLt27RASEgJra+vaVa4lGIB0k1ot4oUvj+Naai5e7e2G//dCO6lLarQelqiw8kAMNp+MhygCthZG+HRUB/i3tZW6NKqmolIV9l1OxZaTCYhO+eu6oG6u1pju54bBXra8oYB0Tr0HoA4dOmD58uUYNmwYoqOj0a1bNwQHB+PIkSPw9PTE5s2ba128NmAA0l1Hb2Zi2qazMNRT4PC8fmhhbSJ1SY3Omdv3sOCXy0i4VwgAGNulBf7fC+1gaWwgcWVUG6IoIjIp+9HdY9GpKPvf3WN2FkpM8XXBhG5OaMIL2ElH1HsAMjMzw5UrV+Dq6oolS5bgypUr2LlzJyIjIzFs2DCkpaXVunhtwACku0RRxKQNZ3Dq9j281LkFPh/nLXVJjUZhSRlWhMZgy8kEAIC9pRKfju6AAR7NpS2M6kx6bhG2nU7Etiemx170dsB0To+RDqj3i6ANDQ1RWPjoX38HDx7E4MGDAQA2NjbIzc2t6qNE9UoQBCwY6gkA2HXxDmLS8iSuqHE4GZeFwDURmvAzoZsTDrzbl+GnkbG1UCJ4sAdOLhqIL8Z5o4OjJUrK1Nh54Q5e+PI4xn59Er9fvotSlVrqUomeW63uFe7duzeCg4PRq1cvnD17Fj/99BMA4ObNm2jRokWdFkhUU52crDC0vR32X0nDygM38O20blKXpLPyi8vw2f7r+P50EgDA0coYn73UgeuuNXJG+noY3bkFRvk4IjIpG1tPJuCP6FScS3iAcwkPYGehxMs9nTGxuzOnx0hn1WoKLCkpCW+++SaSk5PxzjvvYObMmQCAd999FyqVCv/+97/rvNCGxCkw3ReXmY/BqyOgUovY8bovurnaSF2Szjl+KwsLfrmMlOxHDzed3MMZi4a1hZkRn7EkR+m5Rdh2JgnbzyQiK/+v6bGgjg6Y0YvTY6QduBr8c2IAahwW7bqMH84mo6uLNXa87sv1p6opt6gUy/64jh/OPnqqtpONMZaP7gg/96YSV0baoLjsr7vHLt/56+6xri7WmObniiHt7Xj3GEmmQQKQSqXCnj17cP36dQCAl5cXXnzxRejp6dXmcFqFAahxSMspQr+VR1Bcpsa3U7sioB1v0X6W8JgMLNoVjdScIgDANF8XvD/EE6Yc9aEniKKIi8nZ2HLi0fTY47vHbC2M8HIPF0zs4czlT6jB1XsAio2NxbBhw5CSkgIPDw8AQExMDJycnLBv3z60atWqdpVrCQagxuOz/Tfw9dE4tLE1w/45faGn4ChQRXIeluKfv1/Djgt3AAAuTUyw4qWO6NGyicSVkS6ocHpMT4Gg/9091qEFp8eoYdR7ABo2bBhEUcS2bdtgY/Po2op79+7h5ZdfhkKhwL59+2pXuZZgAGo8cgpL0WfFYeQWlWHVWG+M6cKL9J90+EY6Fu2KRnpuMQQBmOHnhvmBHjA21P3RXGpYxWUq/BGdii0nEnDpb9NjXf43PTaU02NUz+o9AJmamuL06dPo0KFDuf2XLl1Cr169kJ+fX9NDahUGoMbl66Nx+Gz/DThaGePQe/2gNOAvdgDILizBx79dw66LKQAAt6amWDmmI7rygnGqAxeTHmDLyQTsu8zpMWo49f4cICMjI+TlPf18lfz8fBgacg0g0i7T/VxhZ6FESvZDfH86UepytMKfV9MwaHUEdl1MgUIA/tG3JfbP6cPwQ3XGx9kaayf44OTCgZjj3xpNzYyQnluMz8Nuwm/ZYQT/HIXov40SETW0Wo0ATZ06FZGRkdi4cSO6d+8OADhz5gxmzZqFLl26YMuWLXVdZ4PiCFDj89O5JCz4JRpWJgaIeH8ALJTyXLbhfkEJluy9ir2X7gIAWjUzxcqx3ujsrNvr95H2Ky5TYX90GjafTMCl5GzN/s7OVpjey43TY1Qn6n0KLDs7G9OmTcNvv/0GA4NHv0hKS0sxYsQIbN68GVZWVrUqXFswADU+ZSo1AtdEIC6zALMHuGNeoIfUJTW4/dGp+PDXK8jKL4FCAF7r1wpz/FtzSpAa3MWkB9h6MgH7olNRqvpremxyDxdM7O6MZuacHqPaabDnAMXGxmpug2/bti3c3d1reyitwgDUOIVeScPr31+AsYEejs7vj+YWSqlLahBZ+cVY/OtV7ItOBQC0sTXDyjHe8HaykrYwkr2MvCJsP5OE708nISu/GMCju8de6GiP6b1c0bGFlbQFks6plwAUHBxc7QK++OKLarfVRgxAjZMoihj91UlcTMrGyz2d8c+RHZ79IR0miiJ+v5yKxXuv4n5BCfQUAt7s3wqzB7rDSJ+jPqQ9SsrUj+4eO5mAqCemxx7dPWYPQ31Oj9Gz1UsAGjBgQLW+XBAEHD58uFpttRUDUON1+vY9TFh/GvoKAQeD+8G1qanUJdWLzLxifLjnCkKvpgEAPO3MsWqsN5crIK1X0fRYc/NH02OTenB6jKrGpTCeEwNQ4zZ981mEx2TihY72WDeps9Tl1ClRFLH30l0s3nsV2YWl0FcImD3QHW/2d+e/oEmnPJ4e23YmCZl5f02PDe9oj+l+rpzCpQoxAD0nBqDG7drdXAz/8hhEEfhtdu9G85TajNwifLD7Cg5eTwcAtLO3wKqx3mjnwD/DpLtKytTYfyUVm0+Unx7zcbbCt1O7cjV6KqfenwNEpMvaOVhghLcDAGDFgRsSV/P8RFHELxfuIOCLozh4PR0GegLeG9QGv87uxfBDOs9QX4ERnRyx561e2PNWL4zycYSBnoD8ojLYmPK5c1R7HAGqAEeAGr/k+4UY+Hk4SlUivp/ZA71b6+ZK52k5RVi06zKOxGQCADo4WmLl2I7wtOOfW2q8MvKKkJZTxLvE6CkcASJ6BicbE0zu4QIAWB56A2q1bv07QBRF/HwuGYNWH8WRmEwY6inw/hAP7H7Tj+GHGr3m5kqGH3puDEAkW7MHusPUUA/RKTn440qq1OVUW0r2Q0zbfA7v/3IZeUVl6ORkhX3v9Mab/d2hzyfpEhFVC/+2JNlqamaEWX1bAgBWHYhBqUotcUVVE0UR288kIXB1BCJuZsJQX4EPhnnilzf80NrWXOryiIh0CgMQydqrfVqiiakhEu4V4qdzyVKXU6nk+4V4eeMZfLA7GvnFZejiYo39c/rgH31bQU8hSF0eEZHOYQAiWTMz0sfbAx8t4bL20C0UlpRJXFF5arWI704nInBNBE7E3oPSQIEPX2iHn1/zRatmZlKXR0SksxiASPYm9XCBk40xMvOKsflEgtTlaCTdK8Skb0/jwz1XUFiiQndXG4TO6YuZvd046kNE9JwYgEj2DPUVeG/Qo9Xhvw6Pw4OCEknrUatFbDkRj8A1ETh9+z6MDfSwJKgdfvxHz0a7dAcRUUNjACIC8KK3A9raWyCvuAz/CY+VrI74rAJMWH8aS367hoelKvRsaYMDc/tiei83KDjqQ0RUZxiAiAAoFALeH/JoFGjrqUSkZD9s0O9XqUV8e+w2hq6NwNmE+zA11MMnI9tj+6s94dzEpEFrISKSAwYgov/p36YZerjZoKRMjTVhNxvse+My8zH265P4577rKCpVo7d7U4TO7YspPV046kNEVE8YgIj+RxAELBjqCQD4JfIObqbn1ev3qdQivjkah6FrjyEyKRtmRvpYNroDvpvZHU42HPUhIqpPkgagiIgIBAUFwcHBAYIgYM+ePZr3SktLsWDBAnTo0AGmpqZwcHDA1KlTcffu3WceNyQkBK6urlAqlejRowfOnj1bj72gxqSzszWGeNlBLQIrQmPq7Xtupedh9FcnsWz/DZSUqdG3TTMceLcvJnZ3hiBw1IeIqL5JGoAKCgrg7e2NkJCQp94rLCxEZGQkPvzwQ0RGRmLXrl2IiYnBiy++WOUxf/rpJwQHB2Px4sWIjIyEt7c3AgMDkZGRUV/doEZmXqAHFAJw8Ho6zifcr9Njl6nUCDkSi+H/Po5LydkwV+pjxZiO2DqjGxytjOv0u4iIqHJasxq8IAjYvXs3Ro4cWWmbc+fOoXv37khMTISzs3OFbXr06IFu3bph3bp1AAC1Wg0nJye8/fbbWLhwYbVq4WrwtPCXy/jxXDK6uVrj59d862RUJiYtD/N3XsLlOzkAgIGezfHpqA6ws1Q+97GJiKgRrwafk5MDQRBgZWVV4fslJSW4cOECAgICNPsUCgUCAgJw6tSpSo9bXFyM3NzcchvJ29yANjDSV+BcwgMcvvF8o4elKjX+fegWXvjyGC7fyYGFUh+fj/XGxmldGX6IiCSiMwGoqKgICxYswMSJEytNdVlZWVCpVLC1tS2339bWFmlpaZUee9myZbC0tNRsTk5OdVo76R47SyWm93IF8OhaIJW6dgOl1+7mYmTICXwRdhOlKhEBbW1xMLgfXurSgtf6EBFJSCcCUGlpKcaNGwdRFPHVV1/V+fEXLVqEnJwczZacrL2LYlLDebOfOyyU+ohJz8Oeiyk1+mxJmRqrw27ixXXHcfVuLqxMDLB2QidsmNoFzS046kNEJDV9qQt4lsfhJzExEYcPH65yTq9p06bQ09NDenp6uf3p6emws7Or9HNGRkYwMjKqs5qpcbA0McAb/d2xPPQGvgi7iRe87WGkr/fMz11JycG8HZdwI+3RbfRDvOzwycj2aGbOP2NERNpCq0eAHoefW7du4eDBg2jSpEmV7Q0NDdGlSxccOnRIs0+tVuPQoUPw9fWt73KpEZru5wpbCyOkZD/E96eTqmxbXKbCqgMxGBFyAjfS8mBjaoh1k3zw1cudGX6IiLSMpAEoPz8fUVFRiIqKAgDEx8cjKioKSUlJKC0txZgxY3D+/Hls27YNKpUKaWlpSEtLQ0nJX4tV+vv7a+74AoDg4GBs2LABW7duxfXr1/HGG2+goKAAM2bMaOjuUSNgbKiHuQFtAAAhR2KRV1RaYbtLydkI+vI41h2JhUotYnhHe4S92xcvdHTgtT5ERFpI0imw8+fPY8CAAZrXwcHBAIBp06ZhyZIl2Lt3LwCgU6dO5T535MgR9O/fHwAQFxeHrKwszXvjx49HZmYmPvroI6SlpaFTp04IDQ196sJoouoa26UFNhy7jduZBdgQcRvBgz007xWVqrDm4C2sj4iDWgSamhninyPbY0h7ewkrJiKiZ9Ga5wBpEz4HiJ4UeiUVr38fCWMDPRx9vz+amysRmfQA83dcQlxmAQBgRCcHLAnygrWpocTVEhHJU01+f2v9RdBE2iDQyw6dnKwQlZyNzw/chIWxPjYej4daBJqZG+FfI9tjsFflF9oTEZF2YQAiqgZBELBgiCcmbjiNn87/9ZiE0Z0d8dEL7WBlwlEfIiJdwgBEVE2+rZqgv0czhMdkwtbCCMtGd8BAT15bRkSkixiAiGrg3xN9cOh6OgZ62sLS2EDqcoiIqJYYgIhqwEJpgFE+LaQug4iInpNWPwiRiIiIqD4wABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkewwABEREZHsMAARERGR7DAAERERkexIGoAiIiIQFBQEBwcHCIKAPXv2lHt/165dGDx4MJo0aQJBEBAVFfXMY27ZsgWCIJTblEpl/XSAiIiIdJKkAaigoADe3t4ICQmp9P3evXtj+fLlNTquhYUFUlNTNVtiYmJdlEtERESNhL6UXz506FAMHTq00venTJkCAEhISKjRcQVBgJ2d3fOURkRERI1Yo7wGKD8/Hy4uLnBycsKIESNw9erVKtsXFxcjNze33EZERESNV6MLQB4eHti0aRN+/fVXfP/991Cr1fDz88OdO3cq/cyyZctgaWmp2ZycnBqwYiIiImpojS4A+fr6YurUqejUqRP69euHXbt2oVmzZvjmm28q/cyiRYuQk5Oj2ZKTkxuwYiIiImpokl4D1BAMDAzg4+OD2NjYStsYGRnByMioAasiIiIiKTW6EaAnqVQqREdHw97eXupSiIiISEtIOgKUn59fbmQmPj4eUVFRsLGxgbOzM+7fv4+kpCTcvXsXABATEwMAsLOz09zlNXXqVDg6OmLZsmUAgI8//hg9e/aEu7s7srOzsXLlSiQmJuLVV19t4N4RERGRtpI0AJ0/fx4DBgzQvA4ODgYATJs2DVu2bMHevXsxY8YMzfsTJkwAACxevBhLliwBACQlJUGh+Gsg68GDB5g1axbS0tJgbW2NLl264OTJk2jXrl0D9IiIiIh0gSCKoih1EdomNzcXlpaWyMnJgYWFhdTlEBERUTXU5Pd3o78GiIiIiOhJDEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDuSBqCIiAgEBQXBwcEBgiBgz5495d7ftWsXBg8ejCZNmkAQBERFRVXruDt27ICnpyeUSiU6dOiAP/74o+6LJyIiIp0laQAqKCiAt7c3QkJCKn2/d+/eWL58ebWPefLkSUycOBEzZ87ExYsXMXLkSIwcORJXrlypq7KJiIhIxwmiKIpSFwEAgiBg9+7dGDly5FPvJSQkwM3NDRcvXkSnTp2qPM748eNRUFCA33//XbOvZ8+e6NSpE77++utq1ZKbmwtLS0vk5OTAwsKiJt0gIiIiidTk93ejuwbo1KlTCAgIKLcvMDAQp06dqvQzxcXFyM3NLbcRERFR49XoAlBaWhpsbW3L7bO1tUVaWlqln1m2bBksLS01m5OTU32XSURERBJqdAGoNhYtWoScnBzNlpycLHVJREREVI/0pS6grtnZ2SE9Pb3cvvT0dNjZ2VX6GSMjIxgZGdV3aURERKQlGt0IkK+vLw4dOlRuX1hYGHx9fSWqiIiIiLSNpCNA+fn5iI2N1byOj49HVFQUbGxs4OzsjPv37yMpKQl3794FAMTExAB4NMrzeERn6tSpcHR0xLJlywAAc+bMQb9+/fD5559j+PDh+PHHH3H+/HmsX7++gXtHRERE2krSEaDz58/Dx8cHPj4+AIDg4GD4+Pjgo48+AgDs3bsXPj4+GD58OABgwoQJ8PHxKXc7e1JSElJTUzWv/fz8sH37dqxfvx7e3t7YuXMn9uzZg/bt2zdgz4iIiEibac1zgLQJnwNERESke2T9HCAiIiKiZ2EAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItlhACIiIiLZYQAiIiIi2WEAIiIiItnRl7oAbSSKIgAgNzdX4kqIiIiouh7/3n78e7wqDEAVyMvLAwA4OTlJXAkRERHVVF5eHiwtLatsI4jViUkyo1arcffuXZibm0MQhDo9dm5uLpycnJCcnAwLC4s6PbY2YP90X2PvY2PvH9D4+8j+6b766qMoisjLy4ODgwMUiqqv8uEIUAUUCgVatGhRr99hYWHRaP9gA+xfY9DY+9jY+wc0/j6yf7qvPvr4rJGfx3gRNBEREckOAxARERHJDgNQAzMyMsLixYthZGQkdSn1gv3TfY29j429f0Dj7yP7p/u0oY+8CJqIiIhkhyNAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQHUoIiICQUFBcHBwgCAI2LNnzzM/Ex4ejs6dO8PIyAju7u7YsmVLvdf5PGrax/DwcAiC8NSWlpbWMAXXwLJly9CtWzeYm5ujefPmGDlyJGJiYp75uR07dsDT0xNKpRIdOnTAH3/80QDV1k5t+rhly5anzp9SqWygimvmq6++QseOHTUPV/P19cX+/fur/IwunT+g5n3UpfNXkc8++wyCIGDu3LlVttO18/hYdfqna+dwyZIlT9Xr6elZ5WekOH8MQHWooKAA3t7eCAkJqVb7+Ph4DB8+HAMGDEBUVBTmzp2LV199FQcOHKjnSmuvpn18LCYmBqmpqZqtefPm9VRh7R09ehRvvfUWTp8+jbCwMJSWlmLw4MEoKCio9DMnT57ExIkTMXPmTFy8eBEjR47EyJEjceXKlQasvPpq00fg0dNa/37+EhMTG6jimmnRogU+++wzXLhwAefPn8fAgQMxYsQIXL16tcL2unb+gJr3EdCd8/ekc+fO4ZtvvkHHjh2rbKeL5xGofv8A3TuHXl5e5eo9fvx4pW0lO38i1QsA4u7du6ts8/7774teXl7l9o0fP14MDAysx8rqTnX6eOTIERGA+ODBgwapqS5lZGSIAMSjR49W2mbcuHHi8OHDy+3r0aOH+Nprr9V3eXWiOn3cvHmzaGlp2XBF1TFra2vx22+/rfA9XT9/j1XVR109f3l5eWLr1q3FsLAwsV+/fuKcOXMqbauL57Em/dO1c7h48WLR29u72u2lOn8cAZLQqVOnEBAQUG5fYGAgTp06JVFF9adTp06wt7fHoEGDcOLECanLqZacnBwAgI2NTaVtdP0cVqePAJCfnw8XFxc4OTk9c7RBW6hUKvz4448oKCiAr69vhW10/fxVp4+Abp6/t956C8OHD3/q/FREF89jTfoH6N45vHXrFhwcHNCyZUtMnjwZSUlJlbaV6vxxMVQJpaWlwdbWttw+W1tb5Obm4uHDhzA2Npaosrpjb2+Pr7/+Gl27dkVxcTG+/fZb9O/fH2fOnEHnzp2lLq9SarUac+fORa9evdC+fftK21V2DrXxGqcnVbePHh4e2LRpEzp27IicnBysWrUKfn5+uHr1ar0vGlwb0dHR8PX1RVFREczMzLB79260a9euwra6ev5q0kddO38A8OOPPyIyMhLnzp2rVntdO4817Z+uncMePXpgy5Yt8PDwQGpqKpYuXYo+ffrgypUrMDc3f6q9VOePAYjqlYeHBzw8PDSv/fz8EBcXh9WrV+O7776TsLKqvfXWW7hy5UqV89a6rrp99PX1LTe64Ofnh7Zt2+Kbb77BJ598Ut9l1piHhweioqKQk5ODnTt3Ytq0aTh69GilAUEX1aSPunb+kpOTMWfOHISFhWn1hb61VZv+6do5HDp0qOa/O3bsiB49esDFxQU///wzZs6cKWFl5TEAScjOzg7p6enl9qWnp8PCwqJRjP5Upnv37lodLGbPno3ff/8dERERz/zXVWXn0M7Orj5LfG416eOTDAwM4OPjg9jY2Hqq7vkYGhrC3d0dANClSxecO3cOa9euxTfffPNUW109fzXp45O0/fxduHABGRkZ5UaIVSoVIiIisG7dOhQXF0NPT6/cZ3TpPNamf0/S9nP4JCsrK7Rp06bSeqU6f7wGSEK+vr44dOhQuX1hYWFVzuU3BlFRUbC3t5e6jKeIoojZs2dj9+7dOHz4MNzc3J75GV07h7Xp45NUKhWio6O18hxWRK1Wo7i4uML3dO38VaaqPj5J28+fv78/oqOjERUVpdm6du2KyZMnIyoqqsJwoEvnsTb9e5K2n8Mn5efnIy4urtJ6JTt/9XqJtczk5eWJFy9eFC9evCgCEL/44gvx4sWLYmJioiiKorhw4UJxypQpmva3b98WTUxMxPnz54vXr18XQ0JCRD09PTE0NFSqLjxTTfu4evVqcc+ePeKtW7fE6Ohocc6cOaJCoRAPHjwoVRcq9cYbb4iWlpZieHi4mJqaqtkKCws1baZMmSIuXLhQ8/rEiROivr6+uGrVKvH69evi4sWLRQMDAzE6OlqKLjxTbfq4dOlS8cCBA2JcXJx44cIFccKECaJSqRSvXr0qRReqtHDhQvHo0aNifHy8ePnyZXHhwoWiIAjin3/+KYqi7p8/Uax5H3Xp/FXmybukGsN5/Ltn9U/XzuF7770nhoeHi/Hx8eKJEyfEgIAAsWnTpmJGRoYoitpz/hiA6tDjW76f3KZNmyaKoihOmzZN7Nev31Of6dSpk2hoaCi2bNlS3Lx5c4PXXRM17ePy5cvFVq1aiUqlUrSxsRH79+8vHj58WJrin6GifgEod0769eun6etjP//8s9imTRvR0NBQ9PLyEvft29ewhddAbfo4d+5c0dnZWTQ0NBRtbW3FYcOGiZGRkQ1ffDW88soroouLi2hoaCg2a9ZM9Pf31wQDUdT98yeKNe+jLp2/yjwZEBrDefy7Z/VP187h+PHjRXt7e9HQ0FB0dHQUx48fL8bGxmre15bzJ4iiKNbvGBMRERGRduE1QERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxAREREJDsMQERERCQ7DEBEREQkOwxARNQoTJ8+HSNHjpS6DCLSEQxAREREJDsMQESkU3bu3IkOHTrA2NgYTZo0QUBAAObPn4+tW7fi119/hSAIEAQB4eHhAIDk5GSMGzcOVlZWsLGxwYgRI5CQkKA53uORo6VLl6JZs2awsLDA66+/jpKSkiq/s6CgoIF7TkR1SV/qAoiIqis1NRUTJ07EihUrMGrUKOTl5eHYsWOYOnUqkpKSkJubi82bNwMAbGxsUFpaisDAQPj6+uLYsWPQ19fHP//5TwwZMgSXL1+GoaEhAODQoUNQKpUIDw9HQkICZsyYgSZNmuBf//pXpd/JZRSJdBsDEBHpjNTUVJSVlWH06NFwcXEBAHTo0AEAYGxsjOLiYtjZ2Wnaf//991Cr1fj2228hCAIAYPPmzbCyskJ4eDgGDx4MADA0NMSmTZtgYmICLy8vfPzxx5g/fz4++eSTKr+TiHQXp8CISGd4e3vD398fHTp0wNixY7FhwwY8ePCg0vaXLl1CbGwszM3NYWZmBjMzM9jY2KCoqAhxcXHljmtiYqJ57evri/z8fCQnJ9f4O4lINzAAEZHO0NPTQ1hYGPbv34927drhyy+/hIeHB+Lj4ytsn5+fjy5duiAqKqrcdvPmTUyaNKlevpOIdAMDEBHpFEEQ0KtXLyxduhQXL16EoaEhdu/eDUNDQ6hUqnJtO3fujFu3bqF58+Zwd3cvt1laWmraXbp0CQ8fPtS8Pn36NMzMzODk5FTldxKR7mIAIiKdcebMGXz66ac4f/48kpKSsGvXLmRmZqJt27ZwdXXF5cuXERMTg6ysLJSWlmLy5Mlo2rQpRowYgWPHjiE+Ph7h4eF45513cOfOHc1xS0pKMHPmTFy7dg1//PEHFi9ejNmzZ0OhUFT5nUSku3gRNBHpDAsLC0RERGDNmjXIzc2Fi4sLPv/8cwwdOhRdu3ZFeHg4unbtivz8fBw5cgT9+/dHREQEFixYgNGjRyMvLw+Ojo7w9/eHhYWF5rj+/v5o3bo1+vbti+LiYkycOBFLlix55ncSke4SRN7LSUQyNn36dGRnZ2PPnj1Sl0JEDYhTYERERCQ7DEBEREQkO5wCIyIiItnhCBARERHJDgMQERERyQ4DEBEREckOAxARERHJDgMQERERyQ4DEBEREckOAxARERHJDgMQERERyQ4DEBEREcnO/wekrgTA6719aQAAAABJRU5ErkJggg==",
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"<Axes: title={'center': 'learning curve'}, xlabel='steps', ylabel='loss'>"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#| hide\n",
"learn.recorder.plot_loss(skip_start=1)"
]
},
{
"cell_type": "markdown",
"id": "e9e4b37e",
"metadata": {},
"source": [
"## CastToTensor -"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ecb8fb93",
"metadata": {},
"outputs": [],
"source": [
"#| exporti\n",
"def _cast_tensor(x): \n",
" if isinstance(x, tuple): return tuple(_cast_tensor(x_) for x_ in x)\n",
" else: return cast(x, Tensor) if isinstance(x,torch.Tensor) else x"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "85905f91",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"class CastToTensor(Callback):\n",
" \"Cast Subclassed Tensors to `Tensor`\"\n",
" order=9 # Right before MixedPrecision\n",
"\n",
" def before_batch(self):\n",
" self.learn.xb,self.learn.yb = _cast_tensor(self.learn.xb),_cast_tensor(self.learn.yb)"
]
},
{
"cell_type": "markdown",
"id": "7923284c",
"metadata": {},
"source": [
"Workaround for bug in PyTorch where subclassed tensors, such as `TensorBase`, train up to ~20% slower than `Tensor` when passed to a model. Added to `Learner` by default.\n",
"\n",
"CastToTensor's order is right before `MixedPrecision` so callbacks which make use of fastai's tensor subclasses still can use them.\n",
"\n",
"If inputs are not a subclassed tensor or tuple of tensors, you may need to cast inputs in `Learner.xb` and `Learner.yb` to `Tensor` via your own callback or in the dataloader before `Learner` performs the forward pass.\n",
"\n",
"If the CastToTensor workaround interferes with custom code, it can be removed:\n",
"\n",
"```python\n",
"learn = Learner(...)\n",
"learn.remove_cb(CastToTensor)\n",
"```\n",
"\n",
"You should verify your inputs are of type `Tensor` or implement a cast to `Tensor` via a custom callback or dataloader if CastToTensor is removed."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3f5c5790",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"if CastToTensor not in defaults.callbacks: defaults.callbacks.append(CastToTensor)"
]
},
{
"cell_type": "markdown",
"id": "1c483c76",
"metadata": {},
"source": [
"## Inference functions"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "88c03677",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L284){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.validate\n",
"\n",
"```python\n",
"\n",
"def validate(\n",
" ds_idx:int=1, dl:NoneType=None, cbs:NoneType=None\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Validate on `dl` with potential new `cbs`.*"
],
"text/plain": [
"```python\n",
"\n",
"def validate(\n",
" ds_idx:int=1, dl:NoneType=None, cbs:NoneType=None\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Validate on `dl` with potential new `cbs`.*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.validate)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0d48c6c9",
"metadata": {},
"outputs": [],
"source": [
"#Test result\n",
"learn = synth_learner(n_train=5, metrics=tst_metric)\n",
"res = learn.validate()\n",
"test_eq(res[0], res[1])\n",
"x,y = learn.dls.valid_ds.tensors\n",
"test_close(res[0], F.mse_loss(learn.model(x), y), 1e-3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "45d130ba",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test other dl\n",
"res = learn.validate(dl=learn.dls.train)\n",
"test_eq(res[0], res[1])\n",
"x,y = learn.dls.train_ds.tensors\n",
"test_close(res[0], F.mse_loss(learn.model(x), y), 1e-3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "14c0698a",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L290){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.get_preds\n",
"\n",
"```python\n",
"\n",
"def get_preds(\n",
" ds_idx:int=1, # `DataLoader` to use for predictions if `dl` is None. 0: train. 1: valid\n",
" dl:NoneType=None, # `DataLoader` to use for predictions, defaults to `ds_idx=1` if None\n",
" with_input:bool=False, # Return inputs with predictions\n",
" with_decoded:bool=False, # Return decoded predictions\n",
" with_loss:bool=False, # Return per item loss with predictions\n",
" act:NoneType=None, # Apply activation to predictions, defaults to `self.loss_func`'s activation\n",
" inner:bool=False, # If False, create progress bar, show logger, use temporary `cbs`\n",
" reorder:bool=True, # Reorder predictions on dataset indicies, if applicable\n",
" cbs:Callback | MutableSequence | None=None, # Temporary `Callback`s to apply during prediction\n",
" save_preds:Path=None, # Path to save predictions\n",
" save_targs:Path=None, # Path to save targets\n",
" with_preds:bool=True, # Whether to return predictions\n",
" with_targs:bool=True, # Whether to return targets\n",
" concat_dim:int=0, # Dimension to concatenate returned tensors\n",
" pickle_protocol:int=2, # Pickle protocol used to save predictions and targets\n",
")->tuple:\n",
"\n",
"\n",
"```\n",
"\n",
"*Get the predictions and targets on the `ds_idx`-th dbunchset or `dl`, optionally `with_input` and `with_loss`*"
],
"text/plain": [
"```python\n",
"\n",
"def get_preds(\n",
" ds_idx:int=1, # `DataLoader` to use for predictions if `dl` is None. 0: train. 1: valid\n",
" dl:NoneType=None, # `DataLoader` to use for predictions, defaults to `ds_idx=1` if None\n",
" with_input:bool=False, # Return inputs with predictions\n",
" with_decoded:bool=False, # Return decoded predictions\n",
" with_loss:bool=False, # Return per item loss with predictions\n",
" act:NoneType=None, # Apply activation to predictions, defaults to `self.loss_func`'s activation\n",
" inner:bool=False, # If False, create progress bar, show logger, use temporary `cbs`\n",
" reorder:bool=True, # Reorder predictions on dataset indicies, if applicable\n",
" cbs:Callback | MutableSequence | None=None, # Temporary `Callback`s to apply during prediction\n",
" save_preds:Path=None, # Path to save predictions\n",
" save_targs:Path=None, # Path to save targets\n",
" with_preds:bool=True, # Whether to return predictions\n",
" with_targs:bool=True, # Whether to return targets\n",
" concat_dim:int=0, # Dimension to concatenate returned tensors\n",
" pickle_protocol:int=2, # Pickle protocol used to save predictions and targets\n",
")->tuple:\n",
"\n",
"\n",
"```\n",
"\n",
"*Get the predictions and targets on the `ds_idx`-th dbunchset or `dl`, optionally `with_input` and `with_loss`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.get_preds)"
]
},
{
"cell_type": "markdown",
"id": "5365cefd",
"metadata": {},
"source": [
"`with_decoded` will also return the decoded predictions using the <code>decodes</code> function of the loss function (if it exists). For instance, fastai's `CrossEntropyFlat` takes the argmax or predictions in its decodes. \n",
"\n",
"Depending on the `loss_func` attribute of `Learner`, an activation function will be picked automatically so that the predictions make sense. For instance if the loss is a case of cross-entropy, a softmax will be applied, or if the loss is binary cross entropy with logits, a sigmoid will be applied. If you want to make sure a certain activation function is applied, you can pass it with `act`.\n",
"\n",
"`save_preds` and `save_targs` should be used when your predictions are too big to fit all in memory. Give a `Path` object that points to a folder where the predictions and targets will be saved.\n",
"\n",
"`concat_dim` is the batch dimension, where all the tensors will be concatenated.\n",
"\n",
"`inner` is an internal attribute that tells `get_preds` it's called internally, inside another training loop, to avoid recursion errors."
]
},
{
"cell_type": "markdown",
"id": "0a1552ed",
"metadata": {},
"source": [
":::{.callout-note}\n",
"\n",
"If you want to use the option `with_loss=True` on a custom loss function, make sure you have implemented a `reduction` attribute that supports 'none'\n",
"\n",
":::"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cacb8c82",
"metadata": {},
"outputs": [],
"source": [
"#Test result\n",
"learn = synth_learner(n_train=5, metrics=tst_metric)\n",
"preds,targs = learn.get_preds()\n",
"x,y = learn.dls.valid_ds.tensors\n",
"test_eq(targs, y)\n",
"test_close(preds, learn.model(x))\n",
"\n",
"preds,targs = learn.get_preds(act = torch.sigmoid)\n",
"test_eq(targs, y)\n",
"test_close(preds, torch.sigmoid(learn.model(x)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "66bf9a46",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test get_preds work with ds not evenly divisible by bs\n",
"learn = synth_learner(n_train=2.5, metrics=tst_metric)\n",
"preds,targs = learn.get_preds(ds_idx=0)\n",
"#Also make sure this version works when drop_last of the dl would be True\n",
"preds,targs = learn.get_preds(dl=learn.dls.train)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f3be4199",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test other dataset\n",
"x = torch.randn(16*5)\n",
"y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
"dl = TfmdDL(TensorDataset(x, y), bs=16)\n",
"preds,targs = learn.get_preds(dl=dl)\n",
"test_eq(targs, y)\n",
"test_close(preds, learn.model(x))\n",
"\n",
"#Test with loss\n",
"preds,targs,losses = learn.get_preds(dl=dl, with_loss=True)\n",
"test_eq(targs, y)\n",
"test_close(preds, learn.model(x))\n",
"test_close(losses, F.mse_loss(preds, targs, reduction='none'))\n",
"\n",
"#Test with inputs\n",
"inps,preds,targs = learn.get_preds(dl=dl, with_input=True)\n",
"test_eq(inps,x)\n",
"test_eq(targs, y)\n",
"test_close(preds, learn.model(x))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4f247b79",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test with no target\n",
"learn = synth_learner(n_train=5)\n",
"x = torch.randn(16*5)\n",
"dl = TfmdDL(TensorDataset(x), bs=16)\n",
"preds,targs = learn.get_preds(dl=dl)\n",
"assert targs is None"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0eddf099",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test with targets that are tuples\n",
"def _fake_loss(x,y,z,reduction=None): return F.mse_loss(x,y)\n",
"\n",
"learn = synth_learner(n_train=5)\n",
"x = torch.randn(16*5)\n",
"y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
"learn.dls.n_inp=1\n",
"learn.loss_func = _fake_loss\n",
"dl = TfmdDL(TensorDataset(x, y, y), bs=16)\n",
"preds,targs = learn.get_preds(dl=dl)\n",
"test_eq(targs, [y,y])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "315e7ab3",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test with inputs that are tuples\n",
"class _TupleModel(Module):\n",
" def __init__(self, model): self.model=model\n",
" def forward(self, x1, x2): return self.model(x1)\n",
"\n",
"learn = synth_learner(n_train=5)\n",
"#learn.dls.n_inp=2\n",
"x = torch.randn(16*5)\n",
"y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
"learn.model = _TupleModel(learn.model)\n",
"learn.dls = DataLoaders(TfmdDL(TensorDataset(x, x, y), bs=16),TfmdDL(TensorDataset(x, x, y), bs=16))\n",
"inps,preds,targs = learn.get_preds(ds_idx=0, with_input=True)\n",
"test_eq(inps, [x,x])\n",
"t = learn.get_preds(ds_idx=0, with_input=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "990c62e2",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test auto activation function is picked\n",
"learn = synth_learner(n_train=5)\n",
"learn.loss_func = BCEWithLogitsLossFlat()\n",
"x = torch.randn(16*5)\n",
"y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
"dl = TfmdDL(TensorDataset(x, y), bs=16)\n",
"preds,targs = learn.get_preds(dl=dl)\n",
"test_close(preds, torch.sigmoid(learn.model(x)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e92a98a0",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"#Test reorder is done\n",
"learn = synth_learner(n_train=5)\n",
"x = torch.randn(16*5)\n",
"y = 2*x + 3 + 0.1*torch.randn(16*5)\n",
"dl = TfmdDL(TensorDataset(x, y), bs=16, shuffle=True)\n",
"preds,targs = learn.get_preds(dl=dl)\n",
"test_eq(targs, y)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "de61b5b9",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"inps,preds,targs = learn.get_preds(ds_idx=0, with_input=True)\n",
"tst = learn.get_preds(ds_idx=0, with_input=True, with_decoded=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "97020c62",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L327){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.predict\n",
"\n",
"```python\n",
"\n",
"def predict(\n",
" item, rm_type_tfms:NoneType=None, with_input:bool=False\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Prediction on `item`, fully decoded, loss function decoded and probabilities*"
],
"text/plain": [
"```python\n",
"\n",
"def predict(\n",
" item, rm_type_tfms:NoneType=None, with_input:bool=False\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Prediction on `item`, fully decoded, loss function decoded and probabilities*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.predict)"
]
},
{
"cell_type": "markdown",
"id": "638ff7b4",
"metadata": {},
"source": [
"It returns a tuple of three elements with, in reverse order,\n",
"- the prediction from the model, potentially passed through the activation of the loss function (if it has one)\n",
"- the decoded prediction, using the potential <code>decodes</code> method from it\n",
"- the fully decoded prediction, using the transforms used to build the `Datasets`/`DataLoaders`"
]
},
{
"cell_type": "markdown",
"id": "48e3218f",
"metadata": {},
"source": [
"`rm_type_tfms` is a deprecated argument that should not be used and will be removed in a future version. `with_input` will add the decoded inputs to the result."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bb9f13cb",
"metadata": {},
"outputs": [],
"source": [
"class _FakeLossFunc(Module):\n",
" reduction = 'none'\n",
" def forward(self, x, y): return F.mse_loss(x,y)\n",
" def activation(self, x): return x+1\n",
" def decodes(self, x): return 2*x\n",
"\n",
"class _Add1(Transform):\n",
" def encodes(self, x): return x+1\n",
" def decodes(self, x): return x-1\n",
" \n",
"learn = synth_learner(n_train=5)\n",
"dl = TfmdDL(Datasets(torch.arange(50), tfms = [L(), [_Add1()]]))\n",
"learn.dls = DataLoaders(dl, dl)\n",
"learn.loss_func = _FakeLossFunc()\n",
"\n",
"inp = tensor([2.])\n",
"out = learn.model(inp).detach()+1 #applying model + activation\n",
"dec = 2*out #decodes from loss function\n",
"full_dec = dec-1 #decodes from _Add1\n",
"test_eq(learn.predict(inp), [full_dec,dec,out])\n",
"test_eq(learn.predict(inp, with_input=True), [inp,full_dec,dec,out])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8efe8fdc",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L338){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.show_results\n",
"\n",
"```python\n",
"\n",
"def show_results(\n",
" ds_idx:int=1, dl:NoneType=None, max_n:int=9, shuffle:bool=True, kwargs:VAR_KEYWORD\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Show some predictions on `ds_idx`-th dataset or `dl`*"
],
"text/plain": [
"```python\n",
"\n",
"def show_results(\n",
" ds_idx:int=1, dl:NoneType=None, max_n:int=9, shuffle:bool=True, kwargs:VAR_KEYWORD\n",
"):\n",
"\n",
"\n",
"```\n",
"\n",
"*Show some predictions on `ds_idx`-th dataset or `dl`*"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.show_results)"
]
},
{
"cell_type": "markdown",
"id": "b88947d1",
"metadata": {},
"source": [
"Will show `max_n` samples (unless the batch size of `ds_idx` or `dl` is less than `max_n`, in which case it will show as many samples) and `shuffle` the data unless you pass `false` to that flag. `kwargs` are application-dependent.\n",
"\n",
"We can't show an example on our synthetic `Learner`, but check all the beginners tutorials which will show you how that method works across applications."
]
},
{
"cell_type": "markdown",
"id": "85035180",
"metadata": {},
"source": [
"The last functions in this section are used internally for inference, but should be less useful to you."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "129cb2d7",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L352){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.no_logging\n",
"\n",
"```python\n",
"\n",
"def no_logging(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
],
"text/plain": [
"```python\n",
"\n",
"def no_logging(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.no_logging)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7bd77f84",
"metadata": {},
"outputs": [],
"source": [
"learn = synth_learner(n_train=5, metrics=tst_metric)\n",
"with learn.no_logging():\n",
" test_stdout(lambda: learn.fit(1), '')\n",
"test_eq(learn.logger, print)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5fc99a23",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"---\n",
"\n",
"[source](https://github.com/fastai/fastai/blob/main/fastai/learner.py#L357){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### Learner.loss_not_reduced\n",
"\n",
"```python\n",
"\n",
"def loss_not_reduced(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
],
"text/plain": [
"```python\n",
"\n",
"def loss_not_reduced(\n",
" \n",
"):\n",
"\n",
"\n",
"```"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"show_doc(Learner.loss_not_reduced)"
]
},
{
"cell_type": "markdown",
"id": "00bfaefc",
"metadata": {},
"source": [
"This requires your loss function to either have a `reduction` attribute or a `reduction` argument (like all fastai and PyTorch loss functions)."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "60295cb8",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"test_eq(learn.loss_func.reduction, 'mean')\n",
"with learn.loss_not_reduced():\n",
" test_eq(learn.loss_func.reduction, 'none')\n",
" x,y = learn.dls.one_batch()\n",
" p = learn.model(x)\n",
" losses = learn.loss_func(p, y)\n",
" test_eq(losses.shape, y.shape)\n",
" test_eq(losses, F.mse_loss(p,y, reduction='none'))\n",
"test_eq(learn.loss_func.reduction, 'mean')"
]
},
{
"cell_type": "markdown",
"id": "131368f3",
"metadata": {},
"source": [
"## Transfer learning"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "88a475f2",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@patch\n",
"def freeze_to(self:Learner, n):\n",
" if self.opt is None: self.create_opt()\n",
" self.opt.freeze_to(n)\n",
" self.opt.clear_state()\n",
"\n",
"@patch\n",
"def freeze(self:Learner): self.freeze_to(-1)\n",
"\n",
"@patch\n",
"def unfreeze(self:Learner): self.freeze_to(0)\n",
"\n",
"add_docs(Learner,\n",
" freeze_to=\"Freeze parameter groups up to `n`\",\n",
" freeze=\"Freeze up to last parameter group\",\n",
" unfreeze=\"Unfreeze the entire model\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9eac243d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0, 10.180200576782227, 10.910623550415039, '00:00']\n"
]
}
],
"source": [
"#| hide\n",
"class _TstModel(nn.Module):\n",
" def __init__(self):\n",
" super().__init__()\n",
" self.a,self.b = nn.Parameter(torch.randn(1)),nn.Parameter(torch.randn(1))\n",
" self.tst = nn.Sequential(nn.Linear(4,5), nn.BatchNorm1d(3))\n",
" self.tst[0].bias.data,self.tst[1].bias.data = torch.randn(5),torch.randn(3) \n",
" def forward(self, x): return x * self.a + self.b\n",
" \n",
"class _PutGrad(Callback):\n",
" def before_step(self):\n",
" for p in self.learn.tst.parameters():\n",
" if p.requires_grad: p.grad = torch.ones_like(p.data)\n",
"\n",
"def _splitter(m): return [list(m.tst[0].parameters()), list(m.tst[1].parameters()), [m.a,m.b]]\n",
" \n",
"learn = synth_learner(n_train=5, opt_func = partial(SGD), cbs=_PutGrad, splitter=_splitter, lr=1e-2)\n",
"learn.model = _TstModel()\n",
"learn.freeze()\n",
"init = [p.clone() for p in learn.tst.parameters()]\n",
"learn.fit(1, wd=0.)\n",
"end = list(learn.tst.parameters())\n",
"#linear was not trained\n",
"for i in [0,1]: test_close(end[i],init[i])\n",
"#bn was trained even frozen since `train_bn=True` by default\n",
"for i in [2,3]: test_close(end[i]-init[i], -0.05 * torch.ones_like(end[i]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5fac8ce9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0, 8.895973205566406, 9.646461486816406, '00:00']\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0, 7.298573017120361, 7.921627521514893, '00:00']\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0, 6.048855304718018, 6.502256393432617, '00:00']\n"
]
}
],
"source": [
"#| hide\n",
"learn = synth_learner(n_train=5, opt_func = partial(SGD), cbs=_PutGrad, splitter=_splitter, train_bn=False, lr=1e-2)\n",
"learn.model = _TstModel()\n",
"learn.freeze()\n",
"init = [p.clone() for p in learn.tst.parameters()]\n",
"learn.fit(1, wd=0.)\n",
"end = list(learn.tst.parameters())\n",
"#linear and bn were not trained\n",
"for i in range(4): test_close(end[i],init[i])\n",
"\n",
"learn.freeze_to(-2)\n",
"init = [p.clone() for p in learn.tst.parameters()]\n",
"learn.fit(1, wd=0.)\n",
"end = list(learn.tst.parameters())\n",
"#linear was not trained\n",
"for i in [0,1]: test_close(end[i],init[i])\n",
"#bn was trained \n",
"for i in [2,3]: test_close(end[i]-init[i], -0.05 * torch.ones_like(end[i]))\n",
" \n",
"learn.unfreeze()\n",
"init = [p.clone() for p in learn.tst.parameters()]\n",
"learn.fit(1, wd=0.)\n",
"end = list(learn.tst.parameters())\n",
"#linear and bn were trained\n",
"for i in range(4): test_close(end[i]-init[i], -0.05 * torch.ones_like(end[i]), 1e-3)"
]
},
{
"cell_type": "markdown",
"id": "73102736",
"metadata": {},
"source": [
"## TTA"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "65eb6e1e",
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"@patch\n",
"def tta(self:Learner, ds_idx=1, dl=None, n=4, item_tfms=None, batch_tfms=None, beta=0.25, use_max=False):\n",
" \"Return predictions on the `ds_idx` dataset or `dl` using Test Time Augmentation\"\n",
" if dl is None: dl = self.dls[ds_idx].new(shuffled=False, drop_last=False)\n",
" if item_tfms is not None or batch_tfms is not None: dl = dl.new(after_item=item_tfms, after_batch=batch_tfms)\n",
" try:\n",
" self(_before_epoch)\n",
" with dl.dataset.set_split_idx(0), self.no_mbar():\n",
" if hasattr(self,'progress'): self.progress.mbar = master_bar(list(range(n)))\n",
" aug_preds = []\n",
" for i in self.progress.mbar if hasattr(self,'progress') else range(n):\n",
" self.epoch = i #To keep track of progress on mbar since the progress callback will use self.epoch\n",
" aug_preds.append(self.get_preds(dl=dl, inner=True)[0][None])\n",
" aug_preds = torch.cat(aug_preds)\n",
" aug_preds = aug_preds.max(0)[0] if use_max else aug_preds.mean(0)\n",
" self.epoch = n\n",
" with dl.dataset.set_split_idx(1): preds,targs = self.get_preds(dl=dl, inner=True)\n",
" finally: self(event.after_fit)\n",
"\n",
" if use_max: return torch.stack([preds, aug_preds], 0).max(0)[0],targs\n",
" preds = (aug_preds,preds) if beta is None else torch.lerp(aug_preds, preds, beta)\n",
" return preds,targs"
]
},
{
"cell_type": "markdown",
"id": "37b21532",
"metadata": {},
"source": [
"In practice, we get the predictions `n` times with the transforms of the training set and average those. The final predictions are `(1-beta)` multiplied by this average + `beta` multiplied by the predictions obtained with the transforms of the dataset. Set `beta` to `None` to get a tuple of the predictions and tta results. You can also use the maximum of all predictions instead of an average by setting `use_max=True`.\n",
"\n",
"If you want to use new transforms, you can pass them with `item_tfms` and `batch_tfms`."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a620ff6a",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"learn = synth_learner()\n",
"dl = TfmdDL(Datasets(torch.arange(50), [noop,noop]))\n",
"learn.dls = DataLoaders(dl, dl)\n",
"preds,targs = learn.tta()\n",
"assert len(preds),len(targs)"
]
},
{
"cell_type": "markdown",
"id": "7efa412e",
"metadata": {},
"source": [
"## Export -"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8fe8c461",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"from nbdev import nbdev_export\n",
"nbdev_export()"
]
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink