chore: import upstream snapshot with attribution

2026-07-13 13:35:51 +08:00
commit c36a561cd8
2172 changed files with 455595 additions and 0 deletions
@@ -0,0 +1,33 @@
+@ECHO OFF
+SETLOCAL EnableDelayedExpansion
+
+ECHO "Current user: %USERNAME%"
+
+python --version
+
+CALL "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+CALL mkvirtualenv --system-site-packages %BUILD_TAG%
+DEL /S /Q build
+DEL /S /Q _download
+MD build
+
+SET _MSPDBSRV_ENDPOINT_=%BUILD_TAG%
+SET TMP=%WORKSPACE%\tmp
+SET TEMP=%WORKSPACE%\tmp
+SET TMPDIR=%WORKSPACE%\tmp
+
+PUSHD build
+cmake -DCMAKE_CXX_FLAGS="/DDGL_EXPORTS" -Dgtest_force_shared_crt=ON -DDMLC_FORCE_SHARED_CRT=ON -DCMAKE_CONFIGURATION_TYPES="Release" -DTORCH_PYTHON_INTERPS=python .. -G "Visual Studio 16 2019" || EXIT /B 1
+msbuild dgl.sln /m /nr:false || EXIT /B 1
+COPY /Y Release\runUnitTests.exe .
+POPD
+
+CALL workon %BUILD_TAG%
+
+PUSHD python
+DEL /S /Q build *.egg-info dist
+pip install -e . || EXIT /B 1
+POPD
+
+ENDLOCAL
+EXIT /B
@@ -0,0 +1,62 @@
+#!/bin/bash
+set -e
+. /opt/conda/etc/profile.d/conda.sh
+
+if [ $# -ne 1 ]; then
+    echo "Device argument required, can be cpu, gpu or cugraph"
+    exit -1
+fi
+
+if [[ $1 != "cpu" ]]; then
+    # CI is now running on g4dn instance. Specify target arch to avoid below
+    # error: Unknown CUDA Architecture Name 9.0a in CUDA_SELECT_NVCC_ARCH_FLAGS
+    export TORCH_CUDA_ARCH_LIST=7.5 # For dgl_sparse and tensoradaptor.
+    CMAKE_VARS="$CMAKE_VARS -DUSE_CUDA=ON -DCUDA_ARCH_NAME=Turing" # For graphbolt.
+fi
+
+# This is a semicolon-separated list of Python interpreters containing PyTorch.
+# The value here is for CI.  Replace it with your own or comment this whole
+# statement for default Python interpreter.
+if [ "$1" != "cugraph" ]; then
+    # We do not build pytorch for cugraph because currently building
+    # pytorch against all the supported cugraph versions is not supported
+    # See issue: https://github.com/rapidsai/cudf/issues/8510
+    CMAKE_VARS="$CMAKE_VARS -DTORCH_PYTHON_INTERPS=/opt/conda/envs/pytorch-ci/bin/python"
+else
+    # Disable sparse build as cugraph docker image lacks cuDNN.
+    CMAKE_VARS="$CMAKE_VARS -DBUILD_TORCH=OFF -DBUILD_SPARSE=OFF"
+fi
+
+if [ -d build ]; then
+    rm -rf build
+fi
+mkdir build
+
+rm -rf _download
+
+pushd build
+cmake $CMAKE_VARS ..
+make -j
+popd
+
+pushd python
+if [[ $1 == "cugraph" ]]; then
+    rm -rf build *.egg-info dist
+    pip uninstall -y dgl
+    # test install
+    python3 setup.py install
+    # test inplace build (for cython)
+    python3 setup.py build_ext --inplace
+else
+    for backend in pytorch mxnet tensorflow
+    do
+    conda activate "${backend}-ci"
+    rm -rf build *.egg-info dist
+    pip uninstall -y dgl
+    # test install
+    DGLBACKEND=${backend} python3 setup.py install
+    # test inplace build (for cython)
+    DGLBACKEND=${backend} python3 setup.py build_ext --inplace
+    done
+fi
+popd
@@ -0,0 +1,104 @@
+import enum
+import json
+import os
+import tempfile
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+import pytest
+import requests
+
+
+class JobStatus(enum.Enum):
+    SUCCESS = 0
+    FAIL = 1
+    SKIP = 2
+
+
+JENKINS_STATUS_MAPPING = {
+    "SUCCESS": JobStatus.SUCCESS,
+    "ABORTED": JobStatus.FAIL,
+    "FAILED": JobStatus.FAIL,
+    "IN_PROGRESS": JobStatus.FAIL,
+    "NOT_EXECUTED": JobStatus.SKIP,
+    "PAUSED_PENDING_INPUT": JobStatus.SKIP,
+    "QUEUED": JobStatus.SKIP,
+    "UNSTABLE": JobStatus.FAIL,
+}
+
+assert "BUILD_URL" in os.environ, "Are you in the Jenkins environment?"
+job_link = os.environ["BUILD_URL"]
+response = requests.get("{}wfapi".format(job_link), verify=False).json()
+domain = "{uri.scheme}://{uri.netloc}/".format(uri=urlparse(job_link))
+stages = response["stages"]
+
+final_dict = {}
+failed_nodes = []
+nodes_dict = {}
+
+
+def get_jenkins_json(path):
+    return requests.get(urljoin(domain, path), verify=False).json()
+
+
+for stage in stages:
+    link = stage["_links"]["self"]["href"]
+    stage_name = stage["name"]
+    res = requests.get(urljoin(domain, link), verify=False).json()
+    nodes = res["stageFlowNodes"]
+    for node in nodes:
+        nodes_dict[node["id"]] = node
+        nodes_dict[node["id"]]["stageName"] = stage_name
+
+
+def get_node_full_name(node, node_dict):
+    name = ""
+    while "parentNodes" in node:
+        name = name + "/" + node["name"]
+        id = node["parentNodes"][0]
+        if id in nodes_dict:
+            node = node_dict[id]
+        else:
+            break
+    return name
+
+
+for key, node in nodes_dict.items():
+    logs = get_jenkins_json(node["_links"]["log"]["href"]).get("text", "")
+    node_name = node["name"]
+    if "Post Actions" in node["stageName"]:
+        continue
+    node_status = node["status"]
+    id = node["id"]
+    full_name = get_node_full_name(node, nodes_dict)
+    final_dict["{}_{}/{}".format(id, node["stageName"], full_name)] = {
+        "status": JENKINS_STATUS_MAPPING[node_status],
+        "logs": logs,
+    }
+
+JOB_NAME = os.getenv("JOB_NAME")
+BUILD_NUMBER = os.getenv("BUILD_NUMBER")
+BUILD_ID = os.getenv("BUILD_ID")
+
+prefix = f"https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/logs_dir/"
+
+
+@pytest.mark.parametrize("test_name", final_dict)
+def test_generate_report(test_name):
+    os.makedirs("./logs_dir/", exist_ok=True)
+    tmp = tempfile.NamedTemporaryFile(
+        mode="w", delete=False, suffix=".log", dir="./logs_dir/"
+    )
+    tmp.write(final_dict[test_name]["logs"])
+    filename = Path(tmp.name).name
+    # print(final_dict[test_name]["logs"])
+    print("Log path: {}".format(prefix + filename))
+
+    if final_dict[test_name]["status"] == JobStatus.FAIL:
+        pytest.fail(
+            "Test failed. Please see the log at {}".format(prefix + filename)
+        )
+    elif final_dict[test_name]["status"] == JobStatus.SKIP:
+        pytest.skip(
+            "Test skipped. Please see the log at {}".format(prefix + filename)
+        )
@@ -0,0 +1,48 @@
+import argparse
+import os
+
+import requests
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--result",
+    type=str,
+    default="FAILURE",
+)
+args = parser.parse_args()
+
+JOB_NAME = os.getenv("JOB_NAME")
+BUILD_NUMBER = os.getenv("BUILD_NUMBER")
+BUILD_ID = os.getenv("BUILD_ID")
+COMMIT = os.getenv("GIT_COMMIT")
+
+# List of status of entire job.
+# https://javadoc.jenkins.io/hudson/model/Result.html
+if args.result == "SUCCESS":
+    status_output = "✅ CI test succeeded."
+elif args.result == "NOT_BUILT":
+    status_output = "⚪️ CI test cancelled due to overrun."
+elif args.result in ["FAILURE", "ABORTED"]:
+    status_output = "❌ CI test failed."
+    JOB_LINK = os.environ["BUILD_URL"]
+    response = requests.get("{}wfapi".format(JOB_LINK), verify=False).json()
+    for stage in response["stages"]:
+        # List of status of individual stage.
+        # https://javadoc.jenkins.io/plugin/pipeline-graph-analysis/org/jenkinsci/plugins/workflow/pipelinegraphanalysis/GenericStatus.html
+        if stage["status"] in ["FAILED", "ABORTED"]:
+            stage_name = stage["name"]
+            status_output = f"❌ CI test failed in Stage [{stage_name}]."
+            break
+else:
+    status_output = f"[Debug Only] CI test with result [{args.result}]."
+
+
+comment = f"""
+Commit ID: {COMMIT}\n
+Build ID: {BUILD_ID}\n
+Status: {status_output}\n
+Report path: [link](https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/report.html)\n
+Full logs path: [link](https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/cireport.log)
+"""
+
+print(comment)
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+. /opt/conda/etc/profile.d/conda.sh
+
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+export DGLBACKEND=$1
+export DGLTESTDEV=gpu
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=tests:${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}/_download
+export TF_FORCE_GPU_ALLOW_GROWTH=true
+
+export CUDA_VISIBLE_DEVICES=0
+
+python3 -m pip install pytest psutil pyyaml pydantic pandas rdflib ogb torchdata || fail "pip install"
+
+python3 -m pytest -v --junitxml=pytest_cugraph.xml --durations=20 tests/cugraph || fail "cugraph"
@@ -0,0 +1,6 @@
+@ECHO OFF
+SETLOCAL EnableDelayedExpansion
+
+PUSHD build
+runUnitTests.exe || EXIT /B 1
+POPD
@@ -0,0 +1,11 @@
+#!/bin/bash
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+echo $PWD
+pushd build
+ls -lh
+export LD_LIBRARY_PATH=$PWD:$LD_LIBRARY_PATH
+./runUnitTests || fail "CPP unit test"
+popd
@@ -0,0 +1,37 @@
+#!/bin/bash
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+echo $PWD
+export DGLBACKEND=pytorch
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=${PWD}/tests:${PWD}/python:$PYTHONPATH
+export LD_LIBRARY_PATH=${PWD}/build:$LD_LIBRARY_PATH
+export DIST_DGL_TEST_CPP_BIN_DIR=${PWD}/build
+export DIST_DGL_TEST_IP_CONFIG=/home/ubuntu/workspace/ip_config.txt
+export DIST_DGL_TEST_PY_BIN_DIR=${PWD}/tests/dist/python
+
+if [[ -v DIST_DGL_TEST_SSH_PORT ]]; then
+    SSH_PORT_LINE="-p $DIST_DGL_TEST_SSH_PORT";
+fi
+
+if [[ -v DIST_DGL_TEST_SSH_KEY ]]; then
+    SSH_KEY_LINE="-i $DIST_DGL_TEST_SSH_KEY";
+fi
+
+if [[ -v DIST_DGL_TEST_SSH_SETUP ]]; then
+    SSH_SETUP_LINE="$DIST_DGL_TEST_SSH_SETUP;";
+fi
+
+
+while IFS= read line
+do
+    for pkg in 'pytest' 'psutil' 'torch'
+    do
+        ret_pkg=$(ssh -o StrictHostKeyChecking=no ${line} ${SSH_PORT_LINE} ${SSH_KEY_LINE} "${SSH_SETUP_LINE}python3 -m pip list | grep -i ${pkg} ") || fail "${pkg} not installed in ${line}"
+    done
+done < ${DIST_DGL_TEST_IP_CONFIG}
+
+python3 -m pytest -v --capture=tee-sys --junitxml=pytest_dist.xml --durations=100 tests/dist/test_*.py || fail "dist across machines"
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+. /opt/conda/etc/profile.d/conda.sh
+
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+function usage {
+    echo "Usage: $0 backend device"
+}
+
+if [ $# -ne 2 ]; then
+    usage
+    fail "Error: must specify backend and device"
+fi
+
+[ $1 == "pytorch" ] || fail "Distrbuted tests run on pytorch backend only."
+[ $2 == "cpu" ] || fail "Distrbuted tests run on cpu only."
+
+export DGLBACKEND=$1
+export DGLTESTDEV=$2
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=tests:${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}/_download
+unset TORCH_ALLOW_TF32_CUBLAS_OVERRIDE
+
+export CUDA_VISIBLE_DEVICES=-1
+
+conda activate ${DGLBACKEND}-ci
+
+export PYTHONUNBUFFERED=1
+export OMP_NUM_THREADS=1
+export DMLC_LOG_DEBUG=1
+
+# Tests for distributed except test_partition.py are skipped due to glitch @2024.06.27.
+python3 -m pytest -v --capture=tee-sys --junitxml=pytest_distributed.xml --durations=100 tests/distributed/test_partition.py || fail "distributed"
+
+# Tests for tools are skipped due to glitch.
+#PYTHONPATH=tools:tools/distpartitioning:$PYTHONPATH python3 -m pytest -v --capture=tee-sys --junitxml=pytest_tools.xml --durations=100 tests/tools/*.py || fail "tools"
@@ -0,0 +1,37 @@
+@ECHO OFF
+SETLOCAL EnableDelayedExpansion
+
+SET GCN_EXAMPLE_DIR=.\examples\pytorch
+
+IF x%1x==xx (
+	ECHO Must supply CPU or GPU
+	GOTO :FAIL
+) ELSE IF x%1x==xcpux (
+	SET DEV=-1
+) ELSE IF x%1x==xgpux (
+	SET DEV=0
+	SET CUDA_VISIBLE_DEVICES=0
+) ELSE (
+	ECHO Must supply CPU or GPU
+	GOTO :FAIL
+)
+CALL workon %BUILD_TAG%
+
+SET DGLBACKEND=pytorch
+SET DGL_LIBRARY_PATH=!CD!\build
+SET PYTHONPATH=!CD!\python;!PYTHONPATH!
+SET DGL_DOWNLOAD_DIR=!CD!\_download
+
+python -m pytest -v --junitxml=pytest_backend.xml --durations=100 tests\examples || GOTO :FAIL
+
+PUSHD !GCN_EXAMPLE_DIR!
+python pagerank.py || GOTO :FAIL
+python gcn\train.py --dataset cora || GOTO :FAIL
+POPD
+ENDLOCAL
+EXIT /B
+
+:FAIL
+ECHO Example test failed
+ENDLOCAL
+EXIT /B 1
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+. /opt/conda/etc/profile.d/conda.sh
+conda activate pytorch-ci
+GCN_EXAMPLE_DIR="./examples/pytorch/"
+
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+function usage {
+    echo "Usage: $0 [cpu|gpu]"
+}
+
+# check arguments
+if [ $# -ne 1 ]; then
+    usage
+    fail "Error: must specify device"
+fi
+
+if [ "$1" == "cpu" ]; then
+    dev=-1
+elif [ "$1" == "gpu" ]; then
+    export CUDA_VISIBLE_DEVICES=0
+    dev=0
+else
+    usage
+    fail "Unknown device $1"
+fi
+
+export DGLBACKEND=pytorch
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}/_download
+
+# test
+
+python3 -m pytest -v --junitxml=pytest_backend.xml --durations=100 tests/examples || fail "sparse examples on $1"
+
+pushd $GCN_EXAMPLE_DIR> /dev/null
+
+python3 pagerank.py || fail "run pagerank.py on $1"
+python3 gcn/train.py --dataset cora || fail "run gcn/train.py on $1"
+python3 lda/lda_model.py || fail "run lda/lda_model.py on $1"
+
+popd > /dev/null
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+. /opt/conda/etc/profile.d/conda.sh
+
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+export DGLBACKEND=pytorch
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=tests:${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}/_download
+
+conda activate pytorch-ci
+
+pushd dglgo
+rm -rf build *.egg-info dist
+pip uninstall -y dglgo
+python3 setup.py install
+popd
+
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+
+# Skip go tests due to ImportError: cannot import name 'cached_property' from 'functools' in python3.7
+#python3 -m pytest -v --junitxml=pytest_go.xml --durations=100 tests/go/test_model.py || fail "go"
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# cpplint
+echo 'Checking code style of C++ codes...'
+python3 tests/lint/lint.py dgl cpp include src || exit 1
+python3 tests/lint/lint.py dgl_sparse cpp dgl_sparse/include dgl_sparse/src || exit 1
+
+# pylint
+echo 'Checking code style of python codes...'
+python3 -m pylint --reports=y -v --rcfile=tests/lint/pylintrc python/dgl || exit 1
@@ -0,0 +1,30 @@
+#!/bin/bash
+# The working directory for this script will be "tests/scripts"
+
+. /opt/conda/etc/profile.d/conda.sh
+conda activate pytorch-ci
+TUTORIAL_ROOT="./tutorials"
+
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+export MPLBACKEND=Agg
+export DGLBACKEND=pytorch
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}/_download
+
+pushd ${TUTORIAL_ROOT} > /dev/null
+# Install requirements
+pip install -r requirements.txt || fail "installing requirements"
+
+# Test
+for f in $(find . -path ./dist -prune -false -o -name "*.py" ! -name "*_mx.py")
+do
+    echo "Running tutorial ${f} ..."
+    python3 $f || fail "run ${f}"
+done
+
+popd > /dev/null
@@ -0,0 +1,21 @@
+@ECHO OFF
+SETLOCAL EnableDelayedExpansion
+
+IF x%1x==xx (
+	ECHO Specify backend
+	EXIT /B 1
+) ELSE (
+	SET BACKEND=%1
+)
+CALL workon %BUILD_TAG%
+
+SET PYTHONPATH=tests;!CD!\python;!PYTHONPATH!
+SET DGLBACKEND=!BACKEND!
+SET DGL_LIBRARY_PATH=!CD!\build
+SET DGL_DOWNLOAD_DIR=!CD!\_download
+
+python -m pip install pytest psutil pandas pyyaml pydantic rdflib torchmetrics expecttest || EXIT /B 1
+python -m pytest -v --junitxml=pytest_backend.xml --durations=100 tests\python\!DGLBACKEND! || EXIT /B 1
+python -m pytest -v --junitxml=pytest_common.xml --durations=100 tests\python\common || EXIT /B 1
+ENDLOCAL
+EXIT /B
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+. /opt/conda/etc/profile.d/conda.sh
+
+function fail {
+    echo FAIL: $@
+    exit -1
+}
+
+function usage {
+    echo "Usage: $0 backend device"
+}
+
+if [ $# -ne 2 ]; then
+    usage
+    fail "Error: must specify backend and device"
+fi
+
+export DGLBACKEND=$1
+export DGLTESTDEV=$2
+export DGL_LIBRARY_PATH=${PWD}/build
+export PYTHONPATH=tests:${PWD}/python:$PYTHONPATH
+export DGL_DOWNLOAD_DIR=${PWD}/_download
+export TF_FORCE_GPU_ALLOW_GROWTH=true
+unset TORCH_ALLOW_TF32_CUBLAS_OVERRIDE
+
+if [ $2 == "gpu" ] 
+then
+  export CUDA_VISIBLE_DEVICES=0
+else
+  export CUDA_VISIBLE_DEVICES=-1
+fi
+
+conda activate ${DGLBACKEND}-ci
+
+python3 -m pip install expecttest
+
+if [ $DGLBACKEND == "mxnet" ]
+then
+  python3 -m pytest -v --junitxml=pytest_compute.xml --durations=100 --ignore=tests/python/common/test_ffi.py tests/python/common || fail "common"
+else
+  python3 -m pytest -v --junitxml=pytest_dgl_import.xml tests/python/test_dgl_import.py || fail "dgl_import"
+  python3 -m pytest -v --junitxml=pytest_common.xml --durations=100 tests/python/common || fail "common"
+fi
+python3 -m pytest -v --junitxml=pytest_backend.xml --durations=100 tests/python/$DGLBACKEND || fail "backend-specific"