chore: import upstream snapshot with attribution

docker/Dockerfile.awscli

@@ -0,0 +1,20 @@
+# Using the Ubuntu image (our OS)
+FROM ubuntu:latest
+# Update package manager (apt-get) 
+# and install (with the yes flag `-y`)
+# Python and Pip
+RUN apt-get update && apt-get install -y \
+    python3.8 \
+    python3-pip
+
+RUN apt-get install -y \
+        unzip \
+        curl \
+    && apt-get clean \
+    && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install \
+    && rm -rf \
+        awscliv2.zip
+
+RUN pip install pytest pytest-html requests

docker/Dockerfile.ci_benchmark

@@ -0,0 +1,30 @@
+# CI docker GPU env
+FROM nvidia/cuda:11.6.0-cudnn8-devel-ubuntu20.04
+
+ENV TZ=US
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
+RUN bash /install/ubuntu_install_build.sh
+
+# python
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
+
+ENV CONDA_ALWAYS_YES="true"
+
+ENV CONDA_ALWAYS_YES=
+
+# Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
+ENV TF_FORCE_GPU_ALLOW_GROWTH=true

docker/Dockerfile.ci_cpu

@@ -0,0 +1,41 @@
+# CI docker CPU env
+# Adapted from github.com/dmlc/tvm/docker/Dockerfile.ci_cpu
+FROM ubuntu:20.04
+
+ENV TZ=US
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
+RUN bash /install/ubuntu_install_build.sh
+
+# tcmalloc
+RUN apt-get install -y libgoogle-perftools4
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4:$LD_PRELOAD
+
+# python
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
+
+ENV CONDA_ALWAYS_YES="true"
+
+COPY install/conda_env/torch_cpu.yml /install/conda_env/torch_cpu.yml
+COPY install/conda_env/torch_cpu_pip.txt /install/conda_env/torch_cpu_pip.txt
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/torch_cpu.yml"]
+
+COPY install/conda_env/tensorflow_cpu.yml /install/conda_env/tensorflow_cpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/tensorflow_cpu.yml"]
+
+COPY install/conda_env/mxnet_cpu.yml /install/conda_env/mxnet_cpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/mxnet_cpu.yml"]
+
+ENV CONDA_ALWAYS_YES=
+
+# SSH
+RUN ["/bin/bash", "-i", "-c", "ssh-keygen -f ~/.ssh/id_rsa -N ''"]
+RUN ["/bin/bash", "-i", "-c", "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys"]
+ENTRYPOINT service ssh restart && bash

docker/Dockerfile.ci_gpu

@@ -0,0 +1,38 @@
+# CI docker GPU env
+FROM nvidia/cuda:12.1.0-cudnn8-devel-ubuntu20.04
+
+ENV TZ=US
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
+RUN bash /install/ubuntu_install_build.sh
+
+# python
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
+
+ENV CONDA_ALWAYS_YES="true"
+
+COPY install/conda_env/torch_gpu.yml /install/conda_env/torch_gpu.yml
+COPY install/conda_env/torch_gpu_pip.txt /install/conda_env/torch_gpu_pip.txt
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/torch_gpu.yml"]
+
+COPY install/conda_env/tensorflow_gpu.yml /install/conda_env/tensorflow_gpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/tensorflow_gpu.yml"]
+
+COPY install/conda_env/mxnet_gpu.yml /install/conda_env/mxnet_gpu.yml
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/mxnet_gpu.yml"]
+
+ENV CONDA_ALWAYS_YES=
+
+# Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
+ENV TF_FORCE_GPU_ALLOW_GROWTH=true

docker/Dockerfile.ci_lint

@@ -0,0 +1,16 @@
+# CI docker for lint
+# Adapted from github.com/dmlc/tvm/docker/Dockerfile.ci_lint
+
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
+RUN bash /install/ubuntu_install_python.sh
+
+RUN apt-get install -y doxygen graphviz
+
+RUN pip3 install cpplint==1.3.0 pylint==2.7.0 mypy

docker/README.md

@@ -0,0 +1,28 @@
+## Build docker image for CI
+
+### CPU image
+```bash
+docker build -t dgl-cpu -f Dockerfile.ci_cpu .
+```
+
+### GPU image
+```bash
+docker build -t dgl-gpu -f Dockerfile.ci_gpu .
+```
+
+### Lint image
+```bash
+docker build -t dgl-lint -f Dockerfile.ci_lint .
+```
+
+### CPU image for kg
+```bash
+wget https://data.dgl.ai/dataset/FB15k.zip -P install/
+docker build -t dgl-cpu:torch-1.2.0 -f Dockerfile.ci_cpu_torch_1.2.0 .
+```
+
+### GPU image for kg
+```bash
+wget https://data.dgl.ai/dataset/FB15k.zip -P install/
+docker build -t dgl-gpu:torch-1.2.0 -f Dockerfile.ci_gpu_torch_1.2.0 .
+```

docker/install/conda_env/kg_cpu.yml

@@ -0,0 +1,18 @@
+name: kg-ci
+dependencies:
+  - python=3.6.9
+  - pip
+  - pip:
+    - torch
+    - torchvision
+    - mxnet
+    - pytest
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/conda_env/kg_gpu.yml

@@ -0,0 +1,18 @@
+name: kg-ci
+dependencies:
+  - python=3.6.9
+  - pip
+  - pip:
+    - torch
+    - torchvision
+    - mxnet-cu101
+    - pytest
+  - nose
+  - numpy
+  - cython
+  - scipy
+  - networkx
+  - matplotlib
+  - nltk
+  - requests[security]
+  - tqdm

docker/install/conda_env/mxnet_cpu.yml

@@ -0,0 +1,22 @@
+name: mxnet-ci
+dependencies:
+  - python=3.7.0
+  - pip
+  - pip:
+    - mxnet==1.6.0
+    - pytest
+    - nose
+    - numpy
+    - cython==0.29
+    - scipy
+    - networkx
+    - matplotlib
+    - nltk
+    - requests[security]
+    - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/mxnet_gpu.yml

@@ -0,0 +1,22 @@
+name: mxnet-ci
+dependencies:
+  - python=3.7.0
+  - pip
+  - pip:
+    - mxnet-cu101==1.7.0
+    - pytest
+    - nose
+    - numpy
+    - cython==0.29
+    - scipy
+    - networkx
+    - matplotlib
+    - nltk
+    - requests[security]
+    - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/tensorflow_cpu.yml

@@ -0,0 +1,22 @@
+name: tensorflow-ci
+dependencies:
+  - python=3.7
+  - pip
+  - pip:
+    - tensorflow==2.3.0
+    - pytest
+    - nose
+    - numpy
+    - cython==0.29
+    - scipy
+    - networkx
+    - matplotlib
+    - nltk
+    - requests[security]
+    - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/tensorflow_gpu.yml

@@ -0,0 +1,22 @@
+name: tensorflow-ci
+dependencies:
+  - python=3.7.0
+  - pip
+  - pip:
+    - tensorflow==2.3.0
+    - pytest
+    - nose
+    - numpy
+    - cython==0.29
+    - scipy
+    - networkx
+    - matplotlib
+    - nltk
+    - requests[security]
+    - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/torch_cpu.yml

@@ -0,0 +1,7 @@
+name: pytorch-ci
+dependencies:
+  - python=3.10
+  - pip
+  - pip:
+    - --find-links https://download.pytorch.org/whl/torch_stable.html
+    - --requirement torch_cpu_pip.txt

docker/install/conda_env/torch_cpu_pip.txt

@@ -0,0 +1,24 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+cython
+filelock
+matplotlib
+networkx
+nltk
+nose
+numpy
+ogb
+pandas
+psutil
+pyarrow
+pydantic
+pytest
+pyyaml
+rdflib
+requests[security]==2.28
+scikit-learn
+scipy
+torch==2.3.0+cpu
+torcheval
+torchmetrics
+torch_geometric
+tqdm

docker/install/conda_env/torch_gpu.yml

@@ -0,0 +1,7 @@
+name: pytorch-ci
+dependencies:
+  - python=3.10
+  - pip
+  - pip:
+    - --find-links https://download.pytorch.org/whl/torch_stable.html
+    - --requirement torch_gpu_pip.txt

docker/install/conda_env/torch_gpu_pip.txt

@@ -0,0 +1,22 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+cython
+matplotlib
+networkx
+nltk
+nose
+numpy
+ogb
+pandas
+psutil
+pydantic
+pytest
+pyyaml
+rdflib
+requests[security]==2.28
+scikit-learn
+scipy
+torch==2.3.0+cu121
+torcheval
+torchmetrics
+torch_geometric
+tqdm

docker/install/ubuntu_install_antlr.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+cd /usr/local/lib
+wget -q https://www.antlr.org/download/antlr-4.7.1-complete.jar
+cd -

docker/install/ubuntu_install_build.sh

@@ -0,0 +1,11 @@
+# Install cmake with minimum required version.
+version=3.18
+build=0
+mkdir ~/temp
+cd ~/temp
+wget https://cmake.org/files/v$version/cmake-$version.$build-Linux-x86_64.sh 
+sudo mkdir /opt/cmake
+sudo sh cmake-$version.$build-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
+sudo ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
+cd ~
+rm -rf ~/temp

docker/install/ubuntu_install_conda.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+export LANG=C.UTF-8 LC_ALL=C.UTF-8
+export PATH=/opt/conda/bin:$PATH
+
+apt-get update --fix-missing && \
+    apt-get install -y wget bzip2 ca-certificates curl git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    /opt/conda/bin/conda clean -tipy && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+export TINI_VERSION=v0.16.1
+source ~/.bashrc

docker/install/ubuntu_install_core.sh

@@ -0,0 +1,6 @@
+# install libraries for building c++ core on ubuntu
+apt update && apt install -y --no-install-recommends --force-yes \
+        apt-utils git build-essential make wget unzip sudo \
+        libz-dev libxml2-dev libopenblas-dev libopencv-dev \
+        graphviz graphviz-dev libgraphviz-dev ca-certificates \
+        systemd vim openssh-client openssh-server

docker/install/ubuntu_install_java.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit -o nounset
+set -o pipefail
+
+apt-get update && apt-get install -y openjdk-8-jdk maven
+test -d "/usr/lib/jvm/java-8-openjdk-amd64/jre"
+echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" >> /etc/profile

docker/install/ubuntu_install_mxnet_cpu.sh

@@ -0,0 +1 @@
+pip3 install mxnet

docker/install/ubuntu_install_mxnet_gpu.sh

@@ -0,0 +1 @@
+pip3 install mxnet-cu90

docker/install/ubuntu_install_python.sh

@@ -0,0 +1,14 @@
+# install python and pip, don't modify this, modify install_python_package.sh
+apt-get update
+apt-get install -y python-dev python3-dev
+
+# install pip
+cd /tmp && wget https://bootstrap.pypa.io/get-pip.py
+python2 get-pip.py
+python3 get-pip.py
+
+# santiy check
+python2 --version
+python3 --version
+pip2 --version
+pip3 --version

docker/install/ubuntu_install_python_package.sh

@@ -0,0 +1,3 @@
+# install libraries for python package on ubuntu
+#pip2 install nose numpy cython scipy networkx matplotlib nltk requests[security] tqdm
+pip3 install nose numpy cython scipy networkx matplotlib nltk requests[security] tqdm

docker/install/ubuntu_install_torch.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# install torch
+pip2 install torch==1.0.1 torchvision==0.2.2
+pip3 install torch==1.0.1 torchvision==0.2.2

docker/install/ubuntu_install_torch_1.2.0.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# install torch
+pip3 install torch==1.2.0+cu92 torchvision==0.4.0+cu92 -f https://download.pytorch.org/whl/torch_stable.html

docker/pods/ci-compile-cpu.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+spec:
+  securityContext:
+    runAsUser: 0
+  containers:
+  - name: dgl-ci-cpu-compile
+    image: dgllib/dgl-ci-cpu:cu101_v220123
+    imagePullPolicy: Always
+    tty: true
+    resources:
+      requests:
+        cpu: 16
+  # affinity:
+  #   nodeAffinity:
+  #     requiredDuringSchedulingIgnoredDuringExecution:
+  #       nodeSelectorTerms:
+  #       - matchExpressions:
+  #         - key: beta.kubernetes.io/instance-type
+  #           operator: In
+  #           values:
+  #           - c5.9xlarge

docker/pods/ci-compile-gpu.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+spec:
+  securityContext:
+    runAsUser: 0
+  containers:
+  - name: dgl-ci-gpu-compile
+    image: dgllib/dgl-ci-gpu:cu101_v220123
+    imagePullPolicy: Always
+    tty: true
+    resources:
+      requests:
+        cpu: 32
+  # affinity:
+  #   nodeAffinity:
+  #     requiredDuringSchedulingIgnoredDuringExecution:
+  #       nodeSelectorTerms:
+  #       - matchExpressions:
+  #         - key: beta.kubernetes.io/instance-type
+  #           operator: In
+  #           values:
+  #           - c5.9xlarge

docker/pods/ci-cpu.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Pod
+spec:
+  securityContext:
+    runAsUser: 0
+  containers:
+  - name: dgl-ci-cpu
+    image: dgllib/dgl-ci-cpu:cu101_v220217
+    imagePullPolicy: Always
+    tty: true
+    resources:
+      requests:
+        cpu: 16
+    volumeMounts:
+      # - name: persistent-storage
+      #   mountPath: /tmp/dataset
+      - name: dshm
+        mountPath: /dev/shm    
+  volumes:
+  # - name: persistent-storage
+  #   persistentVolumeClaim:
+  #     claimName: ogb-efs-claim
+  - name: dshm
+    emptyDir:
+      medium: Memory

docker/pods/ci-gpu.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+spec:
+  securityContext:
+    runAsUser: 0
+  containers:
+    - name: dgl-ci-gpu
+      image: dgllib/dgl-ci-gpu:cu101_v220217
+      imagePullPolicy: Always
+      tty: true
+      resources:
+        limits:
+          nvidia.com/gpu: 1 # requesting 1 GPU
+      volumeMounts:
+        - name: dshm
+          mountPath: /dev/shm
+  volumes:
+  - name: dshm
+    emptyDir:
+      medium: Memory
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: beta.kubernetes.io/instance-type
+            operator: In
+            values:
+            - g4dn.2xlarge

docker/pods/ci-lint.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+spec:
+  securityContext:
+    runAsUser: 0
+  containers:
+  - name: dgl-ci-lint
+    image: dgllib/dgl-ci-lint
+    imagePullPolicy: Always
+    tty: true
+    resources:
+      requests:
+        cpu: 1
+  serviceAccountName: dglciuser