wehub-resource-sync
1300 lines
40 KiB
Plaintext
1300 lines
40 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c7a9b021",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#| eval: false\n",
|
|
"! [ -e /content ] && pip install -Uqq fastai # upgrade fastai on colab"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4fb8ecd9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| default_exp callback.tracker"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d767cb7d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"from fastai.basics import *\n",
|
|
"from fastai.callback.progress import *\n",
|
|
"from fastai.callback.fp16 import MixedPrecision"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a2e17841",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"from nbdev.showdoc import *\n",
|
|
"from fastai.test_utils import *"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "de041f32",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Tracking callbacks\n",
|
|
"\n",
|
|
"> Callbacks that make decisions depending how a monitored metric/loss behaves"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "91ed9958",
|
|
"metadata": {},
|
|
"source": [
|
|
"## TerminateOnNaNCallback -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8495bac6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class TerminateOnNaNCallback(Callback):\n",
|
|
" \"A `Callback` that terminates training if loss is NaN.\"\n",
|
|
" order=-9\n",
|
|
" def after_batch(self):\n",
|
|
" \"Test if `last_loss` is NaN and interrupts training.\"\n",
|
|
" if torch.isinf(self.loss) or torch.isnan(self.loss): raise CancelFitException"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "82c6cfdf",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"learn = synth_learner()\n",
|
|
"learn.fit(10, lr=100, cbs=TerminateOnNaNCallback())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7fa89872",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"assert len(learn.recorder.losses) < 10 * len(learn.dls.train)\n",
|
|
"for l in learn.recorder.losses:\n",
|
|
" assert not torch.isinf(l) and not torch.isnan(l) "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0c8c9515",
|
|
"metadata": {},
|
|
"source": [
|
|
"## TrackerCallback -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2c785426",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class TrackerCallback(Callback):\n",
|
|
" \"A `Callback` that keeps track of the best value in `monitor`.\"\n",
|
|
" order,remove_on_fetch,_only_train_loop = 60,True,True\n",
|
|
" def __init__(self, \n",
|
|
" monitor='valid_loss', # value (usually loss or metric) being monitored.\n",
|
|
" comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.\n",
|
|
" min_delta=0., # minimum delta between the last monitor value and the best monitor value.\n",
|
|
" reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).\n",
|
|
" ):\n",
|
|
" if comp is None: comp = np.less if 'loss' in monitor or 'error' in monitor else np.greater\n",
|
|
" if comp == np.less: min_delta *= -1\n",
|
|
" self.monitor,self.comp,self.min_delta,self.reset_on_fit,self.best= monitor,comp,min_delta,reset_on_fit,None\n",
|
|
"\n",
|
|
" def before_fit(self):\n",
|
|
" \"Prepare the monitored value\"\n",
|
|
" self.run = not hasattr(self, \"lr_finder\") and not hasattr(self, \"gather_preds\")\n",
|
|
" if self.reset_on_fit or self.best is None: self.best = float('inf') if self.comp == np.less else -float('inf')\n",
|
|
" assert self.monitor in self.recorder.metric_names[1:]\n",
|
|
" self.idx = list(self.recorder.metric_names[1:]).index(self.monitor)\n",
|
|
"\n",
|
|
" def after_epoch(self):\n",
|
|
" \"Compare the last value to the best up to now\"\n",
|
|
" val = self.recorder.values[-1][self.idx]\n",
|
|
" if self.comp(val - self.min_delta, self.best): self.best,self.new_best = val,True\n",
|
|
" else: self.new_best = False\n",
|
|
"\n",
|
|
" def after_fit(self): self.run=True"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "48fd3679",
|
|
"metadata": {},
|
|
"source": [
|
|
"When implementing a `Callback` that has behavior that depends on the best value of a metric or loss, subclass this `Callback` and use its `best` (for best value so far) and `new_best` (there was a new best value this epoch) attributes. If you want to maintain `best` over subsequent calls to `fit` (e.g., `Learner.fit_one_cycle`), set `reset_on_fit` = True.\n",
|
|
"\n",
|
|
"`comp` is the comparison operator used to determine if a value is best than another (defaults to `np.less` if 'loss' is in the name passed in `monitor`, `np.greater` otherwise) and `min_delta` is an optional float that requires a new value to go over the current best (depending on `comp`) by at least that amount."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "be690bf6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"class FakeRecords(Callback):\n",
|
|
" order=51\n",
|
|
" def __init__(self, monitor, values): self.monitor,self.values = monitor,values\n",
|
|
" \n",
|
|
" def before_fit(self): self.idx = list(self.recorder.metric_names[1:]).index(self.monitor)\n",
|
|
" def after_epoch(self): self.recorder.values[-1][self.idx] = self.values[self.epoch]\n",
|
|
" \n",
|
|
"class TestTracker(Callback):\n",
|
|
" order=61\n",
|
|
" def before_fit(self): self.bests,self.news = [],[]\n",
|
|
" def after_epoch(self): \n",
|
|
" self.bests.append(self.tracker.best)\n",
|
|
" self.news.append(self.tracker.new_best)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "00f8d320",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" /* Turns off some styling */\n",
|
|
" progress {\n",
|
|
" /* gets rid of default border in Firefox and Opera. */\n",
|
|
" border: none;\n",
|
|
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
|
|
" background-size: auto;\n",
|
|
" }\n",
|
|
" progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
|
|
" background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
|
|
" }\n",
|
|
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
|
|
" background: #F44336;\n",
|
|
" }\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" /* Turns off some styling */\n",
|
|
" progress {\n",
|
|
" /* gets rid of default border in Firefox and Opera. */\n",
|
|
" border: none;\n",
|
|
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
|
|
" background-size: auto;\n",
|
|
" }\n",
|
|
" progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
|
|
" background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
|
|
" }\n",
|
|
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
|
|
" background: #F44336;\n",
|
|
" }\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"learn = synth_learner(n_trn=2, cbs=TestTracker())\n",
|
|
"cbs=[TrackerCallback(monitor='valid_loss'), FakeRecords('valid_loss', [0.2,0.1])]\n",
|
|
"with learn.no_logging(): learn.fit(2, cbs=cbs)\n",
|
|
"test_eq(learn.test_tracker.bests, [0.2, 0.1])\n",
|
|
"test_eq(learn.test_tracker.news, [True,True])\n",
|
|
"\n",
|
|
"#With a min_delta\n",
|
|
"cbs=[TrackerCallback(monitor='valid_loss', min_delta=0.15), FakeRecords('valid_loss', [0.2,0.1])]\n",
|
|
"with learn.no_logging(): learn.fit(2, cbs=cbs)\n",
|
|
"test_eq(learn.test_tracker.bests, [0.2, 0.2])\n",
|
|
"test_eq(learn.test_tracker.news, [True,False])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e73c9547",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" /* Turns off some styling */\n",
|
|
" progress {\n",
|
|
" /* gets rid of default border in Firefox and Opera. */\n",
|
|
" border: none;\n",
|
|
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
|
|
" background-size: auto;\n",
|
|
" }\n",
|
|
" progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
|
|
" background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
|
|
" }\n",
|
|
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
|
|
" background: #F44336;\n",
|
|
" }\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" /* Turns off some styling */\n",
|
|
" progress {\n",
|
|
" /* gets rid of default border in Firefox and Opera. */\n",
|
|
" border: none;\n",
|
|
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
|
|
" background-size: auto;\n",
|
|
" }\n",
|
|
" progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
|
|
" background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
|
|
" }\n",
|
|
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
|
|
" background: #F44336;\n",
|
|
" }\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#By default metrics have to be bigger at each epoch.\n",
|
|
"def tst_metric(out,targ): return F.mse_loss(out,targ)\n",
|
|
"learn = synth_learner(n_trn=2, cbs=TestTracker(), metrics=tst_metric)\n",
|
|
"cbs=[TrackerCallback(monitor='tst_metric'), FakeRecords('tst_metric', [0.2,0.1])]\n",
|
|
"with learn.no_logging(): learn.fit(2, cbs=cbs)\n",
|
|
"test_eq(learn.test_tracker.bests, [0.2, 0.2])\n",
|
|
"test_eq(learn.test_tracker.news, [True,False])\n",
|
|
"\n",
|
|
"#This can be overwritten by passing `comp=np.less`.\n",
|
|
"learn = synth_learner(n_trn=2, cbs=TestTracker(), metrics=tst_metric)\n",
|
|
"cbs=[TrackerCallback(monitor='tst_metric', comp=np.less), FakeRecords('tst_metric', [0.2,0.1])]\n",
|
|
"with learn.no_logging(): learn.fit(2, cbs=cbs)\n",
|
|
"test_eq(learn.test_tracker.bests, [0.2, 0.1])\n",
|
|
"test_eq(learn.test_tracker.news, [True,True])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0085af30",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" /* Turns off some styling */\n",
|
|
" progress {\n",
|
|
" /* gets rid of default border in Firefox and Opera. */\n",
|
|
" border: none;\n",
|
|
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
|
|
" background-size: auto;\n",
|
|
" }\n",
|
|
" progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
|
|
" background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
|
|
" }\n",
|
|
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
|
|
" background: #F44336;\n",
|
|
" }\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" /* Turns off some styling */\n",
|
|
" progress {\n",
|
|
" /* gets rid of default border in Firefox and Opera. */\n",
|
|
" border: none;\n",
|
|
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
|
|
" background-size: auto;\n",
|
|
" }\n",
|
|
" progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
|
|
" background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
|
|
" }\n",
|
|
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
|
|
" background: #F44336;\n",
|
|
" }\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#Setting reset_on_fit=True will maintain the \"best\" value over subsequent calls to fit\n",
|
|
"learn = synth_learner(n_val=2, cbs=TrackerCallback(monitor='tst_metric', reset_on_fit=False), metrics=tst_metric)\n",
|
|
"tracker_cb = learn.cbs.filter(lambda cb: isinstance(cb, TrackerCallback))[0]\n",
|
|
"with learn.no_logging(): learn.fit(1)\n",
|
|
"first_best = tracker_cb.best\n",
|
|
"with learn.no_logging(): learn.fit(1)\n",
|
|
"test_eq(tracker_cb.best, first_best)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "207f07e9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"#A tracker callback is not run during an lr_find\n",
|
|
"from fastai.callback.schedule import *"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8825d94d",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" /* Turns off some styling */\n",
|
|
" progress {\n",
|
|
" /* gets rid of default border in Firefox and Opera. */\n",
|
|
" border: none;\n",
|
|
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
|
|
" background-size: auto;\n",
|
|
" }\n",
|
|
" progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
|
|
" background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
|
|
" }\n",
|
|
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
|
|
" background: #F44336;\n",
|
|
" }\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"#| hide\n",
|
|
"learn = synth_learner(n_trn=2, cbs=TrackerCallback(monitor='tst_metric'), metrics=tst_metric)\n",
|
|
"learn.lr_find(num_it=15, show_plot=False)\n",
|
|
"assert not hasattr(learn, 'new_best')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4884c6bc",
|
|
"metadata": {},
|
|
"source": [
|
|
"## EarlyStoppingCallback -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bbf62555",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class EarlyStoppingCallback(TrackerCallback):\n",
|
|
" \"A `TrackerCallback` that terminates training when monitored quantity stops improving.\"\n",
|
|
" order=TrackerCallback.order+3\n",
|
|
" def __init__(self, \n",
|
|
" monitor='valid_loss', # value (usually loss or metric) being monitored.\n",
|
|
" comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.\n",
|
|
" min_delta=0., # minimum delta between the last monitor value and the best monitor value.\n",
|
|
" patience=1, # number of epochs to wait when training has not improved model.\n",
|
|
" reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).\n",
|
|
" ):\n",
|
|
" super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)\n",
|
|
" self.patience = patience\n",
|
|
"\n",
|
|
" def before_fit(self): self.wait = 0; super().before_fit()\n",
|
|
" def after_epoch(self):\n",
|
|
" \"Compare the value monitored to its best score and maybe stop training.\"\n",
|
|
" super().after_epoch()\n",
|
|
" if self.new_best: self.wait = 0\n",
|
|
" else:\n",
|
|
" self.wait += 1\n",
|
|
" if self.wait >= self.patience:\n",
|
|
" print(f'No improvement since epoch {self.epoch-self.wait}: early stopping')\n",
|
|
" raise CancelFitException()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0939e39b",
|
|
"metadata": {},
|
|
"source": [
|
|
"`comp` is the comparison operator used to determine if a value is best than another (defaults to `np.less` if 'loss' is in the name passed in `monitor`, `np.greater` otherwise) and `min_delta` is an optional float that requires a new value to go over the current best (depending on `comp`) by at least that amount. `patience` is the number of epochs you're willing to wait without improvement."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fe490e27",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>mse_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>20.437918</td>\n",
|
|
" <td>26.406773</td>\n",
|
|
" <td>26.406773</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>20.418514</td>\n",
|
|
" <td>26.406715</td>\n",
|
|
" <td>26.406715</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>2</td>\n",
|
|
" <td>20.410892</td>\n",
|
|
" <td>26.406639</td>\n",
|
|
" <td>26.406639</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"No improvement since epoch 0: early stopping\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"learn = synth_learner(n_trn=2, metrics=F.mse_loss)\n",
|
|
"learn.fit(n_epoch=200, lr=1e-7, cbs=EarlyStoppingCallback(monitor='mse_loss', min_delta=0.1, patience=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "922b0dd0",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"(#2) [26.406639099121094,26.406639099121094]"
|
|
]
|
|
},
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"learn.validate()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d74710ac",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>13.408870</td>\n",
|
|
" <td>19.617222</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>13.403553</td>\n",
|
|
" <td>19.617184</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>2</td>\n",
|
|
" <td>13.403143</td>\n",
|
|
" <td>19.617126</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"No improvement since epoch 0: early stopping\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"learn = synth_learner(n_trn=2)\n",
|
|
"learn.fit(n_epoch=200, lr=1e-7, cbs=EarlyStoppingCallback(monitor='valid_loss', min_delta=0.1, patience=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e495e91d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"test_eq(len(learn.recorder.values), 3)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7e9013a5",
|
|
"metadata": {},
|
|
"source": [
|
|
"## SaveModelCallback -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "682fb519",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class SaveModelCallback(TrackerCallback):\n",
|
|
" \"A `TrackerCallback` that saves the model's best during training and loads it at the end.\"\n",
|
|
" order = TrackerCallback.order+1\n",
|
|
" def __init__(self, \n",
|
|
" monitor='valid_loss', # value (usually loss or metric) being monitored.\n",
|
|
" comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.\n",
|
|
" min_delta=0., # minimum delta between the last monitor value and the best monitor value.\n",
|
|
" fname='model', # model name to be used when saving model.\n",
|
|
" every_epoch=False, # if true, save model after every epoch; else save only when model is better than existing best.\n",
|
|
" at_end=False, # if true, save model when training ends; else load best model if there is only one saved model.\n",
|
|
" with_opt=False, # if true, save optimizer state (if any available) when saving model. \n",
|
|
" reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).\n",
|
|
" ):\n",
|
|
" super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)\n",
|
|
" assert not (every_epoch and at_end), \"every_epoch and at_end cannot both be set to True\"\n",
|
|
" # keep track of file path for loggers\n",
|
|
" self.last_saved_path = None\n",
|
|
" store_attr('fname,every_epoch,at_end,with_opt')\n",
|
|
"\n",
|
|
" def _save(self, name): self.last_saved_path = self.learn.save(name, with_opt=self.with_opt)\n",
|
|
"\n",
|
|
" def after_epoch(self):\n",
|
|
" \"Compare the value monitored to its best score and save if best.\"\n",
|
|
" if self.every_epoch:\n",
|
|
" if (self.epoch%self.every_epoch) == 0: self._save(f'{self.fname}_{self.epoch}')\n",
|
|
" else: #every improvement\n",
|
|
" super().after_epoch()\n",
|
|
" if self.new_best:\n",
|
|
" print(f'Better model found at epoch {self.epoch} with {self.monitor} value: {self.best}.')\n",
|
|
" self._save(f'{self.fname}')\n",
|
|
"\n",
|
|
" def after_fit(self, **kwargs):\n",
|
|
" \"Load the best model.\"\n",
|
|
" if self.at_end: self._save(f'{self.fname}')\n",
|
|
" elif not self.every_epoch: self.learn.load(f'{self.fname}', with_opt=self.with_opt, weights_only=False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "18822840",
|
|
"metadata": {},
|
|
"source": [
|
|
"`comp` is the comparison operator used to determine if a value is best than another (defaults to `np.less` if 'loss' is in the name passed in `monitor`, `np.greater` otherwise) and `min_delta` is an optional float that requires a new value to go over the current best (depending on `comp`) by at least that amount. Model will be saved in `learn.path/learn.model_dir/name.pth`, maybe `every_epoch` if `True`, every nth epoch if an integer is passed to `every_epoch` or at each improvement of the monitored quantity. "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0438c1a4",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>19.453270</td>\n",
|
|
" <td>12.539286</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>19.248507</td>\n",
|
|
" <td>12.123456</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Better model found at epoch 0 with valid_loss value: 12.539285659790039.\n",
|
|
"Better model found at epoch 1 with valid_loss value: 12.123456001281738.\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>5.197007</td>\n",
|
|
" <td>5.579152</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>5.154862</td>\n",
|
|
" <td>5.445522</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Better model found at epoch 0 with valid_loss value: 5.5791521072387695.\n",
|
|
"Better model found at epoch 1 with valid_loss value: 5.445522308349609.\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>4.982775</td>\n",
|
|
" <td>5.264440</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>4.887252</td>\n",
|
|
" <td>5.038480</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>4.578584</td>\n",
|
|
" <td>4.781651</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>4.454868</td>\n",
|
|
" <td>4.507101</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>2</td>\n",
|
|
" <td>4.322047</td>\n",
|
|
" <td>4.232390</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>3</td>\n",
|
|
" <td>4.186467</td>\n",
|
|
" <td>3.957614</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"learn = synth_learner(n_trn=2, path=Path.cwd()/'tmp')\n",
|
|
"learn.fit(n_epoch=2, cbs=SaveModelCallback())\n",
|
|
"assert (Path.cwd()/'tmp/models/model.pth').exists()\n",
|
|
"learn = synth_learner(n_trn=2, path=Path.cwd()/'tmp')\n",
|
|
"learn.fit(n_epoch=2, cbs=SaveModelCallback(fname='end',at_end=True))\n",
|
|
"assert (Path.cwd()/'tmp/models/end.pth').exists()\n",
|
|
"learn.fit(n_epoch=2, cbs=SaveModelCallback(every_epoch=True))\n",
|
|
"for i in range(2): assert (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
|
|
"shutil.rmtree(Path.cwd()/'tmp')\n",
|
|
"learn.fit(n_epoch=4, cbs=SaveModelCallback(every_epoch=2))\n",
|
|
"for i in range(4): \n",
|
|
" if not i%2: assert (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
|
|
" else: assert not (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
|
|
"shutil.rmtree(Path.cwd()/'tmp')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e075e30b",
|
|
"metadata": {},
|
|
"source": [
|
|
"## ReduceLROnPlateau"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "71413ed0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| export\n",
|
|
"class ReduceLROnPlateau(TrackerCallback):\n",
|
|
" \"A `TrackerCallback` that reduces learning rate when a metric has stopped improving.\"\n",
|
|
" order=TrackerCallback.order+2\n",
|
|
" def __init__(self, \n",
|
|
" monitor='valid_loss', # value (usually loss or metric) being monitored.\n",
|
|
" comp=None, # numpy comparison operator; np.less if monitor is loss, np.greater if monitor is metric.\n",
|
|
" min_delta=0., # minimum delta between the last monitor value and the best monitor value.\n",
|
|
" patience=1, # number of epochs to wait when training has not improved model.\n",
|
|
" factor=10., # the denominator to divide the learning rate by, when reducing the learning rate.\n",
|
|
" min_lr=0, # the minimum learning rate allowed; learning rate cannot be reduced below this minimum.\n",
|
|
" reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).\n",
|
|
" ):\n",
|
|
" super().__init__(monitor=monitor, comp=comp, min_delta=min_delta, reset_on_fit=reset_on_fit)\n",
|
|
" self.patience,self.factor,self.min_lr = patience,factor,min_lr\n",
|
|
"\n",
|
|
" def before_fit(self): self.wait = 0; super().before_fit()\n",
|
|
" def after_epoch(self):\n",
|
|
" \"Compare the value monitored to its best score and reduce LR by `factor` if no improvement.\"\n",
|
|
" super().after_epoch()\n",
|
|
" if self.new_best: self.wait = 0\n",
|
|
" else:\n",
|
|
" self.wait += 1\n",
|
|
" if self.wait >= self.patience:\n",
|
|
" old_lr = self.opt.hypers[-1]['lr']\n",
|
|
" for h in self.opt.hypers: h['lr'] = max(h['lr'] / self.factor, self.min_lr)\n",
|
|
" self.wait = 0\n",
|
|
" if self.opt.hypers[-1][\"lr\"] < old_lr:\n",
|
|
" print(f'Epoch {self.epoch}: reducing lr to {self.opt.hypers[-1][\"lr\"]}')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "acb7ae63",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>6.122743</td>\n",
|
|
" <td>7.348515</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>6.119377</td>\n",
|
|
" <td>7.348499</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>2</td>\n",
|
|
" <td>6.125790</td>\n",
|
|
" <td>7.348477</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>3</td>\n",
|
|
" <td>6.131386</td>\n",
|
|
" <td>7.348475</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 2: reducing lr to 1e-08\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"learn = synth_learner(n_trn=2)\n",
|
|
"learn.fit(n_epoch=4, lr=1e-7, cbs=ReduceLROnPlateau(monitor='valid_loss', min_delta=0.1, patience=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "64a611e4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"test_eq(learn.opt.hypers[-1]['lr'], 1e-8)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "475a9c2c",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: left;\">\n",
|
|
" <th>epoch</th>\n",
|
|
" <th>train_loss</th>\n",
|
|
" <th>valid_loss</th>\n",
|
|
" <th>time</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <td>0</td>\n",
|
|
" <td>16.747515</td>\n",
|
|
" <td>15.265999</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>1</td>\n",
|
|
" <td>16.725756</td>\n",
|
|
" <td>15.265974</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>2</td>\n",
|
|
" <td>16.735016</td>\n",
|
|
" <td>15.265943</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>3</td>\n",
|
|
" <td>16.733360</td>\n",
|
|
" <td>15.265934</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>4</td>\n",
|
|
" <td>16.733513</td>\n",
|
|
" <td>15.265925</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <td>5</td>\n",
|
|
" <td>16.730352</td>\n",
|
|
" <td>15.265915</td>\n",
|
|
" <td>00:00</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch 2: reducing lr to 1e-08\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"learn = synth_learner(n_trn=2)\n",
|
|
"learn.fit(n_epoch=6, lr=5e-8, cbs=ReduceLROnPlateau(monitor='valid_loss', min_delta=0.1, patience=2, min_lr=1e-8))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "29f6b1fe",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"test_eq(learn.opt.hypers[-1]['lr'], 1e-8)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "d673c19a",
|
|
"metadata": {},
|
|
"source": [
|
|
"Each of these three derived `TrackerCallback`s (`SaveModelCallback`, `ReduceLROnPlateu`, and `EarlyStoppingCallback`) all have an adjusted order so they can each run with each other without interference. That order is as follows:\n",
|
|
"\n",
|
|
":::{.callout-note}\n",
|
|
"\n",
|
|
"in parenthesis is the actual `Callback` order number\n",
|
|
"\n",
|
|
":::\n",
|
|
"\n",
|
|
"1. `TrackerCallback` (60)\n",
|
|
"2. `SaveModelCallback` (61)\n",
|
|
"3. `ReduceLrOnPlateu` (62)\n",
|
|
"4. `EarlyStoppingCallback` (63)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "37d8a32c",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Export -"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "60315804",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#| hide\n",
|
|
"from nbdev import nbdev_export\n",
|
|
"nbdev_export()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "aa7df950",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"jupytext": {
|
|
"split_at_heading": true
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "python3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|