chore: import upstream snapshot with attribution

benchmarks/scripts/README.md

@@ -0,0 +1,32 @@
+Regression Test Suite
+========================
+
+### Spec of task.json
+```json
+# Note the test will be run if the name specified below is a substring of the full test name.
+# The fullname of "benchmarks/model_acc/bench_sage_ns.track_acc" will be "model_acc.bench_sage_ns.track_acc". Test will be run if it contains any keyword.
+# For example, "model_acc" will run all the tests under "model_acc" folder
+# "bench_sage" will run both "bench_sage" and "bench_sage_ns"
+# "bench_sage." will only run "bench_sage"
+# "ns" will run any tests name contains "ms"
+# "" will run all tests
+{
+    "c5.9xlarge": { # The instance type to run the test
+        "tests": [
+            "bench_sage" # The test to be run on this instance
+        ],
+        "env": {
+            "DEVICE": "cpu" # The environment variable passed to publish.sh
+        }
+    },
+    "g4dn.2xlarge": {
+        ...
+    }
+}
+```
+
+
+### Environment variable
+- `MOUNT_PATH` specify the directory in the host to be mapped into docker, if exists will map the `MOUNT_PATH`(in host) to `/tmp/dataset`(in docker)
+- `INSTANCE_TYPE` specify the current instance type
+- `DGL_REG_CONF` specify the path to `task.json`, which is relative to the repo root. If specified, must specify `INSTANCE_TYPE` also

benchmarks/scripts/build_dgl_asv.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+# Default building only with cpu
+DEVICE=${DGL_BENCH_DEVICE:-cpu}
+
+pip install -r /asv/torch_gpu_pip.txt
+
+# build
+# 'CUDA_TOOLKIT_ROOT_DIR' is always required for sparse build as torch1.13.1+cu116 is installed.
+CMAKE_VARS="-DUSE_OPENMP=ON -DBUILD_TORCH=ON -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda"
+if [[ $DEVICE == "gpu" ]]; then
+    CMAKE_VARS="-DUSE_CUDA=ON $CMAKE_VARS"
+fi
+mkdir -p build
+pushd build
+cmake $CMAKE_VARS ..
+make -j8
+popd

benchmarks/scripts/fix_ram_info.py

@@ -0,0 +1,28 @@
+import json
+from pathlib import Path
+
+
+def main():
+    result_dir = Path(__file__).parent / ".." / Path("results/")
+    for per_machine_dir in result_dir.iterdir():
+        if per_machine_dir.is_dir():
+            try:
+                machine_json = json.loads(
+                    (per_machine_dir / "machine.json").read_text()
+                )
+                ram = machine_json["ram"]
+                for f in per_machine_dir.glob("*.json"):
+                    if f.stem != "machine":
+                        result = json.loads(f.read_text())
+                        result_ram = result["params"]["ram"]
+                        if result_ram != ram:
+                            result["params"]["ram"] = ram
+                            print(f"Fix ram in {f}")
+                            f.write_text(json.dumps(result))
+                        else:
+                            print(f"Skip {f}")
+            except Exception as e:
+                print(e)
+
+
+main()

benchmarks/scripts/generate_excel.py

@@ -0,0 +1,110 @@
+import json
+from itertools import product
+from pathlib import Path
+
+import pandas as pd
+
+
+def get_branch_name_from_hash(hash):
+    import subprocess
+
+    process = subprocess.Popen(
+        ["git", "name-rev", "--name-only", hash],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = process.communicate()
+    if len(stderr) > 0:
+        return hash[:10]
+    else:
+        return stdout.decode("utf-8").strip("\n")
+
+
+def main():
+    results_path = Path("../results")
+    results_path.is_dir()
+    machines = [f for f in results_path.glob("*") if f.is_dir()]
+    output_results_dict = {}
+    for machine in machines:
+        per_machine_result = {}
+        commit_results_json_paths = [
+            f for f in machine.glob("*") if f.name != "machine.json"
+        ]
+        for commit in commit_results_json_paths:
+            with commit.open() as f:
+                commit_result = json.load(f)
+            commit_hash = commit_result["commit_hash"]
+            per_commit_result = {}
+            for test_name, result in commit_result["results"].items():
+                per_commit_result[test_name] = []
+                if result["result"] is None:
+                    for test_args in product(*result["params"]):
+                        per_commit_result[test_name].append(
+                            {"params": ", ".join(test_args), "result": None}
+                        )
+                else:
+                    for test_args, performance_number in zip(
+                        product(*result["params"]), result["result"]
+                    ):
+                        per_commit_result[test_name].append(
+                            {
+                                "params": ", ".join(test_args),
+                                "result": performance_number,
+                            }
+                        )
+            per_machine_result[commit_hash] = per_commit_result
+        output_results_dict[machine.name] = per_machine_result
+    return output_results_dict
+
+
+def dict_to_csv(output_results_dict):
+    with open("../results/benchmarks.json") as f:
+        benchmark_conf = json.load(f)
+    unit_dict = {}
+    for k, v in benchmark_conf.items():
+        if k != "version":
+            unit_dict[k] = v["unit"]
+    result_list = []
+    for machine, per_machine_result in output_results_dict.items():
+        for commit, test_cases in per_machine_result.items():
+            branch_name = get_branch_name_from_hash(commit)
+            result_column_name = "number_{}".format(branch_name)
+            # per_commit_result_list = []
+            for test_case_name, results in test_cases.items():
+                for result in results:
+                    result_list.append(
+                        {
+                            "test_name": test_case_name,
+                            "params": result["params"],
+                            "unit": unit_dict[test_case_name],
+                            "number": result["result"],
+                            "commit": branch_name,
+                            "machine": machine,
+                        }
+                    )
+    df = pd.DataFrame(result_list)
+    return df
+
+
+def side_by_side_view(df):
+    commits = df["commit"].unique().tolist()
+    full_df = df.loc[df["commit"] == commits[0]]
+    for commit in commits[1:]:
+        per_commit_df = df.loc[df["commit"] == commit]
+        full_df: pd.DataFrame = full_df.merge(
+            per_commit_df,
+            on=["test_name", "params", "machine", "unit"],
+            how="outer",
+            suffixes=(
+                "_{}".format(full_df.iloc[0]["commit"]),
+                "_{}".format(per_commit_df.iloc[0]["commit"]),
+            ),
+        )
+    full_df = full_df.loc[:, ~full_df.columns.str.startswith("commit")]
+    return full_df
+
+
+output_results_dict = main()
+df = dict_to_csv(output_results_dict)
+sbs_df = side_by_side_view(df)
+sbs_df.to_csv("result.csv")

benchmarks/scripts/install_dgl_asv.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+# install
+pushd python
+rm -rf build *.egg-info dist
+pip uninstall -y dgl
+python3 setup.py install
+popd

benchmarks/scripts/publish.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# The script launches a docker container to run ASV benchmarks. We use the same docker
+# image as our CI (i.e., dgllib/dgl-ci-gpu:conda). It performs the following steps:
+#
+#   1. Start a docker container of the given machine name. The machine name will be
+#      displayed on the generated website.
+#   2. Copy `.git` into the container. It allows ASV to determine the repository information
+#      such as commit hash, branches, etc.
+#   3. Copy this folder into the container including the ASV configuration file `asv.conf.json`.
+#      This means any changes to the files in this folder do not
+#      require a git commit. By contrast, to correctly benchmark your changes to the core
+#      library (e.g., "python/dgl"), you must call git commit first.
+#   4. It then calls the `run.sh` script inside the container. It will invoke `asv run`.
+#      You can change the command such as specifying the benchmarks to run or adding some flags.
+#   5. After benchmarking, it copies the generated `results` and `html` folders back to
+#      the host machine.
+#
+
+if [ $# -eq 2 ]; then
+    MACHINE=$1
+    DEVICE=$2
+else
+    echo "publish.sh <machine_name> <device>"
+    exit 1
+fi
+
+WS_ROOT=/asv/dgl
+docker pull public.ecr.aws/s1o7b3d9/benchmark_test:cu116_v230110
+if [ -z "$DGL_REG_CONF" ]; then
+    DOCKER_ENV_OPT="$DOCKER_ENV_OPT"
+else
+    DOCKER_ENV_OPT=" -e DGL_REG_CONF=$DGL_REG_CONF $DOCKER_ENV_OPT"
+fi
+
+if [ -z "$INSTANCE_TYPE" ]; then
+    DOCKER_ENV_OPT="$DOCKER_ENV_OPT"
+else
+    DOCKER_ENV_OPT=" -e INSTANCE_TYPE=$INSTANCE_TYPE $DOCKER_ENV_OPT"
+fi
+
+if [ -z "$MOUNT_PATH" ]; then
+    DOCKER_MOUNT_OPT=""
+else
+    DOCKER_MOUNT_OPT="-v ${MOUNT_PATH}:/tmp/dataset -v ${MOUNT_PATH}/dgl_home/:/root/.dgl/"
+fi
+
+echo $HOME
+echo "Mount Point: ${DOCKER_MOUNT_OPT}"
+echo "Env opt: ${DOCKER_ENV_OPT}"
+echo "DEVICE: ${DEVICE}"
+
+if [[ $DEVICE == "cpu" ]]; then
+    docker run --name dgl-reg \
+        --rm \
+        $DOCKER_MOUNT_OPT \
+        $DOCKER_ENV_OPT \
+        --shm-size="16g" \
+        --hostname=$MACHINE -dit public.ecr.aws/s1o7b3d9/benchmark_test:cu116_v230110 /bin/bash
+else
+    docker run --name dgl-reg \
+        --rm --gpus all \
+        $DOCKER_MOUNT_OPT \
+        $DOCKER_ENV_OPT \
+        --shm-size="16g" \
+        --hostname=$MACHINE -dit public.ecr.aws/s1o7b3d9/benchmark_test:cu116_v230110 /bin/bash
+fi
+
+pwd
+
+docker exec dgl-reg mkdir -p $WS_ROOT
+docker cp ../../.git dgl-reg:$WS_ROOT
+docker cp ../ dgl-reg:$WS_ROOT/benchmarks/
+docker cp torch_gpu_pip.txt dgl-reg:/asv
+docker exec $DOCKER_ENV_OPT dgl-reg bash $WS_ROOT/benchmarks/run.sh $DEVICE
+docker cp dgl-reg:$WS_ROOT/benchmarks/results ../
+docker cp dgl-reg:$WS_ROOT/benchmarks/html ../
+docker stop dgl-reg

benchmarks/scripts/replace_branch.py

@@ -0,0 +1,82 @@
+import argparse
+import json
+import os
+import re
+
+
+def json_minify(string, strip_space=True):
+    """
+    Based on JSON.minify.js:
+    https://github.com/getify/JSON.minify
+    Contributers:
+    - Pradyun S. Gedam (conditions and variable names changed)
+    """
+    tokenizer = re.compile(r'"|(/\*)|(\*/)|(//)|\n|\r')
+    in_string = False
+    in_multi = False
+    in_single = False
+
+    new_str = []
+    index = 0
+
+    for match in re.finditer(tokenizer, string):
+        if not (in_multi or in_single):
+            tmp = string[index : match.start()]
+            if not in_string and strip_space:
+                # replace white space as defined in standard
+                tmp = re.sub("[ \t\n\r]+", "", tmp)
+            new_str.append(tmp)
+
+        index = match.end()
+        val = match.group()
+
+        if val == '"' and not (in_multi or in_single):
+            escaped = re.search(r"(\\)*$", string[: match.start()])
+
+            # start of string or unescaped quote character to end string
+            if not in_string or (
+                escaped is None or len(escaped.group()) % 2 == 0
+            ):
+                in_string = not in_string
+            index -= 1  # include " character in next catch
+        elif not (in_string or in_multi or in_single):
+            if val == "/*":
+                in_multi = True
+            elif val == "//":
+                in_single = True
+        elif val == "*/" and in_multi and not (in_string or in_single):
+            in_multi = False
+        elif val in "\r\n" and not (in_multi or in_string) and in_single:
+            in_single = False
+        elif not (
+            (in_multi or in_single) or (val in " \r\n\t" and strip_space)
+        ):
+            new_str.append(val)
+
+    new_str.append(string[index:])
+    content = "".join(new_str)
+    content = content.replace(",]", "]")
+    content = content.replace(",}", "}")
+    return content
+
+
+def add_prefix(branch_name):
+    if "/" not in branch_name:
+        return "origin/" + branch_name
+    else:
+        return branch_name
+
+
+def change_branch(branch_str: str):
+    branches = [add_prefix(b) for b in branch_str.split(",")]
+    with open("../asv.conf.json", "r") as f:
+        ss = f.read()
+        config_json = json.loads(json_minify(ss))
+        config_json["branches"] = branches
+    with open("../asv.conf.json", "w") as f:
+        json.dump(config_json, f)
+
+
+if __name__ == "__main__":
+    if "BRANCH_STR" in os.environ:
+        change_branch(os.environ["BRANCH_STR"])

benchmarks/scripts/torch_gpu_pip.txt

@@ -0,0 +1,19 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+torch==1.13.1+cu116
+torchvision==0.14.1+cu116
+torchmetrics
+pytest
+nose
+numpy
+cython
+scipy
+networkx
+matplotlib
+nltk
+requests[security]
+tqdm
+awscli
+torchtext
+pandas
+rdflib
+ogb