wehub/ageron--handson-ml3
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:36:24 +08:00
commit 472091efb3
55 changed files with 103022 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/.env
+1
View File
@@ -0,0 +1 @@
COMPOSE_PROJECT_NAME=handson-ml3
docker/Dockerfile
+78
View File
@@ -0,0 +1,78 @@
FROM continuumio/miniconda3:latest
RUN apt-get update && apt-get install -y \
protobuf-compiler \
sudo \
&& apt clean \
&& rm -rf /var/lib/apt/lists/*
COPY environment.yml /tmp/
RUN conda env create -f /tmp/environment.yml \
&& conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& rm /tmp/environment.yml
ARG username
ARG userid
ARG home=/home/${username}
ARG workdir=${home}/handson-ml3
RUN adduser ${username} --uid ${userid} --gecos '' --disabled-password \
&& echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username} \
&& chmod 0440 /etc/sudoers.d/${username}
WORKDIR ${workdir}
RUN chown ${username}:${username} ${workdir}
USER ${username}
WORKDIR ${workdir}
ENV PATH /opt/conda/envs/homl3/bin:$PATH
# The config below enables diffing notebooks with nbdiff (and nbdiff support
# in git diff command) after connecting to the container by "make exec" (or
# "docker-compose exec handson-ml3 bash")
# You may also try running:
# nbdiff NOTEBOOK_NAME.ipynb
# to get nbdiff between checkpointed version and current version of the
# given notebook.
RUN git-nbdiffdriver config --enable --global
# INFO: Optionally uncomment any (one) of the following RUN commands below to ignore either
# metadata or details in nbdiff within git diff
#RUN git config --global diff.jupyternotebook.command 'git-nbdiffdriver diff --ignore-metadata'
RUN git config --global diff.jupyternotebook.command 'git-nbdiffdriver diff --ignore-details'
COPY docker/bashrc.bash /tmp/
RUN cat /tmp/bashrc.bash >> ${home}/.bashrc \
&& echo "export PATH=\"${workdir}/docker/bin:$PATH\"" >> ${home}/.bashrc \
&& sudo rm /tmp/bashrc.bash
# INFO: Uncomment lines below to enable automatic save of python-only and html-only
# exports alongside the notebook
#COPY docker/jupyter_notebook_config.py /tmp/
#RUN cat /tmp/jupyter_notebook_config.py >> ${home}/.jupyter/jupyter_notebook_config.py
#RUN sudo rm /tmp/jupyter_notebook_config.py
# INFO: Uncomment the RUN command below to disable git diff paging
#RUN git config --global core.pager ''
# INFO: Uncomment the RUN command below for easy and constant notebook URL (just localhost:8888)
# That will switch Jupyter to using empty password instead of a token.
# To avoid making a security hole you SHOULD in fact not only uncomment but
# regenerate the hash for your own non-empty password and replace the hash below.
# You can compute a password hash in any notebook, just run the code:
# from notebook.auth import passwd
# passwd()
# and take the hash from the output
#RUN mkdir -p ${home}/.jupyter && \
# echo 'c.NotebookApp.password = u"sha1:c6bbcba2d04b:f969e403db876dcfbe26f47affe41909bd53392e"' \
# >> ${home}/.jupyter/jupyter_notebook_config.py
docker/Dockerfile.gpu
+204
View File
@@ -0,0 +1,204 @@
# This Dockerfile includes sections from tensorflow/tensorflow:latest-gpu's Dockerfile:
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile
# and sections from continuumio/miniconda3:latest's Dockerfile:
# https://github.com/ContinuumIO/docker-images/blob/master/miniconda3/debian/Dockerfile
# First we need CUDA and everything else needed to support GPUs
###############################################
#### FROM tensorflow/tensorflow:latest-gpu ####
###############################################
ARG UBUNTU_VERSION=20.04
ARG ARCH=
ARG CUDA_BASE=11.2.2
FROM nvidia/cuda:${CUDA_BASE}-base-ubuntu${UBUNTU_VERSION} as base
# ARCH and CUDA are specified again because the FROM directive resets ARGs
# (but their default value is retained if set previously)
ARG ARCH
ARG CUDA=11.2
ARG CUDNN=8.1.0.77-1
ARG CUDNN_MAJOR_VERSION=8
ARG LIB_DIR_PREFIX=x86_64
ARG LIBNVINFER=7.2.2-1
ARG LIBNVINFER_MAJOR_VERSION=7
# Let us install tzdata painlessly
ENV DEBIAN_FRONTEND=noninteractive
# Needed for string substitution
SHELL ["/bin/bash", "-c"]
# Pick up some TF dependencies
# [HOML3] Tweaked for handson-ml3: added all the libs before build-essentials
# and call apt clean + delete apt cache files.
RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub && \
apt-get update && apt-get install -y --no-install-recommends \
git \
protobuf-compiler \
sudo \
wget \
build-essential \
cuda-command-line-tools-${CUDA/./-} \
libcublas-${CUDA/./-} \
cuda-nvrtc-${CUDA/./-} \
libcufft-${CUDA/./-} \
libcurand-${CUDA/./-} \
libcusolver-${CUDA/./-} \
libcusparse-${CUDA/./-} \
curl \
libcudnn8=${CUDNN}+cuda${CUDA} \
libfreetype6-dev \
libhdf5-serial-dev \
libzmq3-dev \
pkg-config \
software-properties-common \
unzip \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Install TensorRT if not building for PowerPC
# NOTE: libnvinfer uses cuda11.1 versions
RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub && \
echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/tensorRT.list && \
apt-get update && \
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.0 \
libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.0 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*; }
# For CUDA profiling, TensorFlow requires CUPTI.
ENV LD_LIBRARY_PATH /usr/local/cuda-11.0/targets/x86_64-linux/lib:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure
# dynamic linker run-time bindings
RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
&& echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
&& ldconfig
# [HOML3] Tweaked for handson-ml3: removed Python3 & TensorFlow installation using pip
#################################################
#### End of tensorflow/tensorflow:latest-gpu ####
#################################################
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ENV PATH /opt/conda/bin:/opt/conda/envs/homl3/bin:$PATH
ENV CONDA_DEFAULT_ENV=homl3
# Next we need to install miniconda
############################################
#### FROM continuumio/miniconda3:latest ####
############################################
# [HOML2] Tweaked for handson-ml3: removed the beginning of the Dockerfile
CMD [ "/bin/bash" ]
# Leave these args here to better use the Docker build cache
ARG CONDA_VERSION=py39_4.12.0
RUN set -x && \
UNAME_M="$(uname -m)" && \
if [ "${UNAME_M}" = "x86_64" ]; then \
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh"; \
SHA256SUM="78f39f9bae971ec1ae7969f0516017f2413f17796670f7040725dd83fcff5689"; \
elif [ "${UNAME_M}" = "s390x" ]; then \
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-s390x.sh"; \
SHA256SUM="ff6fdad3068ab5b15939c6f422ac329fa005d56ee0876c985e22e622d930e424"; \
elif [ "${UNAME_M}" = "aarch64" ]; then \
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-aarch64.sh"; \
SHA256SUM="5f4f865812101fdc747cea5b820806f678bb50fe0a61f19dc8aa369c52c4e513"; \
elif [ "${UNAME_M}" = "ppc64le" ]; then \
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-ppc64le.sh"; \
SHA256SUM="1fe3305d0ccc9e55b336b051ae12d82f33af408af4b560625674fa7ad915102b"; \
fi && \
wget "${MINICONDA_URL}" -O miniconda.sh -q && \
echo "${SHA256SUM} miniconda.sh" > shasum && \
if [ "${CONDA_VERSION}" != "latest" ]; then sha256sum --check --status shasum; fi && \
mkdir -p /opt && \
sh miniconda.sh -b -p /opt/conda && \
rm miniconda.sh shasum && \
ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
echo "conda activate base" >> ~/.bashrc && \
find /opt/conda/ -follow -type f -name '*.a' -delete && \
find /opt/conda/ -follow -type f -name '*.js.map' -delete && \
/opt/conda/bin/conda clean -afy
##############################################
#### End of continuumio/miniconda3:latest ####
##############################################
# Now we're ready to create our conda environment
COPY environment.yml /tmp/
RUN conda env create -f /tmp/environment.yml \
&& conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& rm /tmp/environment.yml
ARG username
ARG userid
ARG home=/home/${username}
ARG workdir=${home}/handson-ml3
RUN adduser ${username} --uid ${userid} --gecos '' --disabled-password \
&& echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username} \
&& chmod 0440 /etc/sudoers.d/${username}
WORKDIR ${workdir}
RUN chown ${username}:${username} ${workdir}
USER ${username}
WORKDIR ${workdir}
# The config below enables diffing notebooks with nbdiff (and nbdiff support
# in git diff command) after connecting to the container by "make exec" (or
# "docker-compose exec handson-ml3 bash")
# You may also try running:
# nbdiff NOTEBOOK_NAME.ipynb
# to get nbdiff between checkpointed version and current version of the
# given notebook.
RUN git-nbdiffdriver config --enable --global
# INFO: Optionally uncomment any (one) of the following RUN commands below to ignore either
# metadata or details in nbdiff within git diff
#RUN git config --global diff.jupyternotebook.command 'git-nbdiffdriver diff --ignore-metadata'
RUN git config --global diff.jupyternotebook.command 'git-nbdiffdriver diff --ignore-details'
COPY docker/bashrc.bash /tmp/
RUN cat /tmp/bashrc.bash >> ${home}/.bashrc \
&& echo "export PATH=\"${workdir}/docker/bin:$PATH\"" >> ${home}/.bashrc \
&& sudo rm /tmp/bashrc.bash
# INFO: Uncomment lines below to enable automatic save of python-only and html-only
# exports alongside the notebook
#COPY docker/jupyter_notebook_config.py /tmp/
#RUN cat /tmp/jupyter_notebook_config.py >> ${home}/.jupyter/jupyter_notebook_config.py
#RUN sudo rm /tmp/jupyter_notebook_config.py
# INFO: Uncomment the RUN command below to disable git diff paging
#RUN git config --global core.pager ''
# INFO: Uncomment the RUN command below for easy and constant notebook URL (just localhost:8888)
# That will switch Jupyter to using empty password instead of a token.
# To avoid making a security hole you SHOULD in fact not only uncomment but
# regenerate the hash for your own non-empty password and replace the hash below.
# You can compute a password hash in any notebook, just run the code:
# from notebook.auth import passwd
# passwd()
# and take the hash from the output
#RUN mkdir -p ${home}/.jupyter && \
# echo 'c.NotebookApp.password = u"sha1:c6bbcba2d04b:f969e403db876dcfbe26f47affe41909bd53392e"' \
# >> ${home}/.jupyter/jupyter_notebook_config.py
docker/Makefile
+14
View File
@@ -0,0 +1,14 @@
help:
cat Makefile
run:
docker-compose up
exec:
docker-compose exec handson-ml3 bash
build: stop .FORCE
docker-compose build
rebuild: stop .FORCE
docker-compose build --no-cache
stop:
docker stop handson-ml3 || true; docker rm handson-ml3 || true;
.FORCE:
docker/README.md
+151
View File
@@ -0,0 +1,151 @@
# Hands-on Machine Learning in Docker
This is the Docker configuration which allows you to run and tweak the book's notebooks without installing any dependencies on your machine!<br/>OK, any except `docker` and `docker-compose`.<br />And optionally `make`.<br />And a few more things if you want GPU support (see below for details).
## Prerequisites
Follow the instructions on [Install Docker](https://docs.docker.com/engine/installation/) and [Install Docker Compose](https://docs.docker.com/compose/install/) for your environment if you haven't got `docker` and `docker-compose` already.
Some general knowledge about `docker` infrastructure might be useful (that's an interesting topic on its own) but is not strictly *required* to just run the notebooks.
## Usage
### Prepare the image (once)
The first option is to pull the image from Docker Hub (this will download about 1.9 GB of compressed data):
```bash
$ docker pull ageron/handson-ml3
```
**Note**: this is the CPU-only image. For GPU support, read the GPU section below.
Alternatively, you can build the image yourself. This will be slower, but it will ensure the image is up to date, with the latest libraries. For this, assuming you already downloaded this project into the directory `/path/to/project/handson-ml3`:
```bash
$ cd /path/to/project/handson-ml3/docker
$ docker-compose build
```
This will take quite a while, but is only required once.
After the process is finished you have an `ageron/handson-ml3:latest` image, that will be the base for your experiments. You can confirm that by running the following command:
```bash
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
ageron/handson-ml3 latest 3ebafebc604a 2 minutes ago 4.87GB
```
### Run the notebooks
Still assuming you already downloaded this project into the directory `/path/to/project/handson-ml3`, run the following commands to start the Jupyter server inside the container, which is named `handson-ml3`:
```bash
$ cd /path/to/project/handson-ml3/docker
$ docker-compose up
```
Next, just point your browser to the URL printed on the screen (or go to <http://localhost:8888> if you enabled password authentication inside the `jupyter_notebook_config.py` file, before building the image) and you're ready to play with the book's code!
The server runs in the directory containing the notebooks, and the changes you make from the browser will be persisted there.
You can close the server just by pressing `Ctrl-C` in the terminal window.
### Using `make` (optional)
If you have `make` installed on your computer, you can use it as a thin layer to run `docker-compose` commands. For example, executing `make rebuild` will actually run `docker-compose build --no-cache`, which will rebuild the image without using the cache. This ensures that your image is based on the latest version of the `continuumio/miniconda3` image which the `ageron/handson-ml3` image is based on.
If you don't have `make` (and you don't want to install it), just examine the contents of `Makefile` to see which `docker-compose` commands you can run instead.
### Run additional commands in the container
Run `make exec` (or `docker-compose exec handson-ml3 bash`) while the server is running to run an additional `bash` shell inside the `handson-ml3` container. Now you're inside the environment prepared within the image.
One of the useful things that can be done there would be starting TensorBoard (for example with simple `tb` command, see bashrc file).
Another one may be comparing versions of the notebooks using the `nbdiff` command if you haven't got `nbdime` installed locally (it is **way** better than plain `diff` for notebooks). See [Tools for diffing and merging of Jupyter notebooks](https://github.com/jupyter/nbdime) for more details.
You can see changes you made relative to the version in git using `git diff` which is integrated with `nbdiff`.
You may also try `nbd NOTEBOOK_NAME.ipynb` command (custom, see bashrc file) to compare one of your notebooks with its `checkpointed` version.<br/>
To be precise, the output will tell you *what modifications should be re-played on the **manually saved** version of the notebook (located in `.ipynb_checkpoints` subdirectory) to update it to the **current** i.e. **auto-saved** version (given as command's argument - located in working directory)*.
## GPU Support on Linux (experimental)
### Prerequisites
If you're running on Linux, and you have a TensorFlow-compatible GPU card (NVidia card with Compute Capability ≥ 3.5) that you would like TensorFlow to use inside the Docker container, then you should download and install the latest driver for your card from [nvidia.com](https://www.nvidia.com/Download/index.aspx?lang=en-us). You will also need to install [NVidia Docker support](https://github.com/NVIDIA/nvidia-docker): if you are using Docker 19.03 or above, you must install the `nvidia-container-toolkit` package, and for earlier versions, you must install `nvidia-docker2`.
Next, edit the `docker-compose.yml` file:
```bash
$ cd /path/to/project/handson-ml3/docker
$ edit docker-compose.yml # use your favorite editor
```
* Replace `dockerfile: ./docker/Dockerfile` with `dockerfile: ./docker/Dockerfile.gpu`
* Replace `image: ageron/handson-ml3:latest` with `image: ageron/handson-ml3:latest-gpu`
* If you want to use `docker-compose`, you will need version 1.28 or above for GPU support, and you must uncomment the whole `deploy` section in `docker-compose.yml`.
### Prepare the image (once)
If you want to pull the prebuilt image from Docker Hub (this will download over 3.5 GB of compressed data):
```bash
$ docker pull ageron/handson-ml3:latest-gpu
```
If you prefer to build the image yourself:
```bash
$ cd /path/to/project/handson-ml3/docker
$ docker-compose build
```
### Run the notebooks with `docker-compose` (version 1.28 or above)
If you have `docker-compose` version 1.28 or above, that's great! You can simply run:
```bash
$ cd /path/to/project/handson-ml3/docker
$ docker-compose up
[...]
or http://127.0.0.1:8888/?token=[...]
```
Then point your browser to the URL and Jupyter should appear. If you then open or create a notebook and execute the following code, a list containing your GPU device(s) should be displayed (success!):
```python
import tensorflow as tf
tf.config.list_physical_devices("GPU")
```
To stop the server, just press Ctrl-C.
### Run the notebooks without `docker-compose`
If you have a version of `docker-compose` earlier than 1.28, you will have to use `docker run` directly.
If you are using Docker 19.03 or above, you can run:
```bash
$ cd /path/to/project/handson-ml3
$ docker run --name handson-ml3 --gpus all -p 8888:8888 -p 6006:6006 --log-opt mode=non-blocking --log-opt max-buffer-size=50m -v `pwd`:/home/devel/handson-ml3 ageron/handson-ml3:latest-gpu /opt/conda/envs/homl3/bin/jupyter notebook --ip='0.0.0.0' --port=8888 --no-browser
```
If you are using an older version of Docker, then replace `--gpus all` with `--runtime=nvidia`.
Now point your browser to the displayed URL: Jupyter should appear, and you can open a notebook and run `import tensorflow as tf` and `tf.config.list_physical_devices("GPU")` as above to confirm that TensorFlow does indeed see your GPU device(s).
Lastly, to interrupt the server, press Ctrl-C, then run:
```bash
$ docker rm handson-ml3
```
This will remove the container so you can start a new one later (but it will not remove the image or the notebooks, don't worry!).
Have fun!
docker/bashrc.bash
+3
View File
@@ -0,0 +1,3 @@
alias ll="ls -alF"
alias nbd="nbdiff_checkpoint"
alias tb="tensorboard --logdir=tf_logs"
docker/bin/nbclean_checkpoints
Executable
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python
import collections
import glob
import hashlib
import os
import subprocess
class NotebookAnalyser:
def __init__(self, dry_run=False, verbose=False, colorful=False):
self._dry_run = dry_run
self._verbose = verbose
self._colors = collections.defaultdict(lambda: "")
if colorful:
for color in [
NotebookAnalyser.COLOR_WHITE,
NotebookAnalyser.COLOR_RED,
NotebookAnalyser.COLOR_GREEN,
NotebookAnalyser.COLOR_YELLOW,
]:
self._colors[color] = "\033[{}m".format(color)
NOTEBOOK_SUFFIX = ".ipynb"
CHECKPOINT_DIR = NOTEBOOK_SUFFIX + "_checkpoints"
CHECKPOINT_MASK = "*-checkpoint" + NOTEBOOK_SUFFIX
CHECKPOINT_MASK_LEN = len(CHECKPOINT_MASK) - 1
@staticmethod
def get_hash(file_path):
with open(file_path, "rb") as input:
hash = hashlib.md5()
for chunk in iter(lambda: input.read(4096), b""):
hash.update(chunk)
return hash.hexdigest()
MESSAGE_ORPHANED = "missing "
MESSAGE_MODIFIED = "modified"
MESSAGE_DELETED = "DELETING"
COLOR_WHITE = "0"
COLOR_RED = "31"
COLOR_GREEN = "32"
COLOR_YELLOW = "33"
def log(self, message, file, color=COLOR_WHITE):
color_on = self._colors[color]
color_off = self._colors[NotebookAnalyser.COLOR_WHITE]
print("{}{}{}: {}".format(color_on, message, color_off, file))
def clean_checkpoints(self, directory):
for checkpoint_path in sorted(glob.glob(os.path.join(directory, NotebookAnalyser.CHECKPOINT_MASK))):
workfile_dir = os.path.dirname(os.path.dirname(checkpoint_path))
workfile_name = os.path.basename(checkpoint_path)[:-NotebookAnalyser.CHECKPOINT_MASK_LEN] + NotebookAnalyser.NOTEBOOK_SUFFIX
workfile_path = os.path.join(workfile_dir, workfile_name)
status = ""
if not os.path.isfile(workfile_path):
if self._verbose:
self.log(NotebookAnalyser.MESSAGE_ORPHANED, workfile_path, NotebookAnalyser.COLOR_RED)
else:
checkpoint_stat = os.stat(checkpoint_path)
workfile_stat = os.stat(workfile_path)
modified = workfile_stat.st_size != checkpoint_stat.st_size
if not modified:
checkpoint_hash = NotebookAnalyser.get_hash(checkpoint_path)
workfile_hash = NotebookAnalyser.get_hash(workfile_path)
modified = checkpoint_hash != workfile_hash
if modified:
if self._verbose:
self.log(NotebookAnalyser.MESSAGE_MODIFIED, workfile_path, NotebookAnalyser.COLOR_YELLOW)
else:
self.log(NotebookAnalyser.MESSAGE_DELETED, checkpoint_path, NotebookAnalyser.COLOR_GREEN)
if not self._dry_run:
os.remove(checkpoint_path)
if not self._dry_run and not os.listdir(directory):
self.log(NotebookAnalyser.MESSAGE_DELETED, directory, NotebookAnalyser.COLOR_GREEN)
os.rmdir(directory)
def clean_checkpoints_recursively(self, directory):
for (root, subdirs, files) in os.walk(directory):
subdirs.sort() # INFO: traverse alphabetically
if NotebookAnalyser.CHECKPOINT_DIR in subdirs:
subdirs.remove(NotebookAnalyser.CHECKPOINT_DIR) # INFO: don't recurse there
self.clean_checkpoints(os.path.join(root, NotebookAnalyser.CHECKPOINT_DIR))
def main():
import argparse
parser = argparse.ArgumentParser(description="Remove checkpointed versions of those jupyter notebooks that are identical to their working copies.",
epilog="""Notebooks will be reported as either
"DELETED" if the working copy and checkpointed version are identical
(checkpoint will be deleted),
"missing" if there is a checkpoint but no corresponding working file can be found
or "modified" if notebook and the checkpoint are not byte-to-byte identical.
If removal of checkpoints results in empty ".ipynb_checkpoints" directory
that directory is also deleted.
""") #, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("dirs", metavar="DIR", type=str, nargs="*", default=".", help="directories to search")
parser.add_argument("-d", "--dry-run", action="store_true", help="only print messages, don't perform any removals")
parser.add_argument("-v", "--verbose", action="store_true", help="verbose mode")
parser.add_argument("-c", "--color", action="store_true", help="colorful mode")
args = parser.parse_args()
analyser = NotebookAnalyser(args.dry_run, args.verbose, args.color)
for directory in args.dirs:
analyser.clean_checkpoints_recursively(directory)
if __name__ == "__main__":
main()
docker/bin/nbdiff_checkpoint
Executable
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
if [[ "$#" -lt 1 || "$1" =~ ^((-h)|(--help))$ ]] ; then
echo "usage: nbdiff_checkpoint NOTEBOOK.ipynb"
echo
echo "Show differences between given jupyter notebook and its checkpointed version (in .ipynb_checkpoints subdirectory)"
exit
fi
DIRNAME=$(dirname "$1")
BASENAME=$(basename "$1" .ipynb)
shift
WORKING_COPY=$DIRNAME/$BASENAME.ipynb
CHECKPOINT_COPY=$DIRNAME/.ipynb_checkpoints/$BASENAME-checkpoint.ipynb
echo "----- Analysing how to change $CHECKPOINT_COPY into $WORKING_COPY -----"
nbdiff "$CHECKPOINT_COPY" "$WORKING_COPY" --ignore-details "$@"
docker/bin/rm_empty_subdirs
Executable
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env python
import os
def remove_empty_directories(initial_dir,
allow_initial_delete=False, ignore_nonexistant_initial=False,
dry_run=False, quiet=False):
FORBIDDEN_SUBDIRS = set([".git"])
if not os.path.isdir(initial_dir) and not ignore_nonexistant_initial:
raise RuntimeError("Initial directory '{}' not found!".format(initial_dir))
message = "removed"
if dry_run:
message = "to be " + message
deleted = set()
for (directory, subdirs, files) in os.walk(initial_dir, topdown=False):
forbidden = False
parent = directory
while parent:
parent, dirname = os.path.split(parent)
if dirname in FORBIDDEN_SUBDIRS:
forbidden = True
break
if forbidden:
continue
is_empty = len(files) < 1 and len(set([os.path.join(directory, s) for s in subdirs]) - deleted) < 1
if is_empty and (initial_dir != directory or allow_initial_delete):
if not quiet:
print("{}: {}".format(message, directory))
deleted.add(directory)
if not dry_run:
os.rmdir(directory)
def main():
import argparse
parser = argparse.ArgumentParser(description="Remove empty directories recursively in subtree.")
parser.add_argument("dir", metavar="DIR", type=str, nargs="+", help="directory to be searched")
parser.add_argument("-r", "--allow-dir-removal", action="store_true", help="allow deletion of DIR itself")
parser.add_argument("-i", "--ignore-nonexistent-dir", action="store_true", help="don't throw an error if DIR doesn't exist")
parser.add_argument("-d", "--dry-run", action="store_true", help="only print messages, don't perform any removals")
parser.add_argument("-q", "--quiet", action="store_true", help="don't print names of directories being removed")
args = parser.parse_args()
for directory in args.dir:
remove_empty_directories(directory, args.allow_dir_removal, args.ignore_nonexistent_dir,
args.dry_run, args.quiet)
if __name__ == "__main__":
main()
docker/bin/tensorboard
Executable
+2
View File
@@ -0,0 +1,2 @@
#!/bin/bash
python -m tensorboard.main "$@"
docker/docker-compose.yml
+27
View File
@@ -0,0 +1,27 @@
version: "3"
services:
handson-ml3:
build:
context: ../
dockerfile: ./docker/Dockerfile #Dockerfile.gpu
args:
- username=devel
- userid=1000
container_name: handson-ml3
image: ageron/handson-ml3:latest #latest-gpu
restart: unless-stopped
logging:
driver: json-file
options:
max-size: 50m
ports:
- "8888:8888"
- "6006:6006"
volumes:
- ../:/home/devel/handson-ml3
command: /opt/conda/envs/homl3/bin/jupyter lab --ip='0.0.0.0' --port=8888 --no-browser
#deploy:
# resources:
# reservations:
# devices:
# - capabilities: [gpu]
docker/jupyter_notebook_config.py
+15
View File
@@ -0,0 +1,15 @@
import os
import subprocess
def export_script_and_view(model, os_path, contents_manager):
if model["type"] != "notebook":
return
dir_name, file_name = os.path.split(os_path)
file_base, file_ext = os.path.splitext(file_name)
if file_base.startswith("Untitled"):
return
export_name = file_base if file_ext == ".ipynb" else file_name
subprocess.check_call(["jupyter", "nbconvert", "--to", "script", file_name, "--output", export_name + "_script"], cwd=dir_name)
subprocess.check_call(["jupyter", "nbconvert", "--to", "html", file_name, "--output", export_name + "_view"], cwd=dir_name)
c.FileContentsManager.post_save_hook = export_script_and_view