wehub-resource-sync
256 lines
7.5 KiB
Python
256 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import csv
|
|
import pickle
|
|
import re
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
|
sys.path.append(str(REPO_ROOT / "tests" / "scripts"))
|
|
sys.path.append(str(REPO_ROOT / "tests" / "scripts" / "github"))
|
|
sys.path.append(str(REPO_ROOT / "tests" / "scripts" / "jenkins"))
|
|
|
|
# Tag dictionary used to create a mapping relation to categorize PRs owning same tag.
|
|
TAG_DICT = {
|
|
"metaschedule": "MetaSchedule",
|
|
"cuda": "cuda & cutlass & tensorrt",
|
|
"cutlass": "cuda & cutlass & tensorrt",
|
|
"tensorrt": "cuda & cutlass & tensorrt",
|
|
"hexagon": "Hexagon",
|
|
"metal": "Metal",
|
|
"vulkan": "Vulkan",
|
|
"clml": "OpenCL & CLML",
|
|
"opencl": "OpenCL & CLML",
|
|
"openclml": "OpenCL & CLML",
|
|
"adreno": "Adreno",
|
|
"acl": "ArmComputeLibrary",
|
|
"rocm": "ROCm",
|
|
"crt": "CRT",
|
|
"web": "web",
|
|
"wasm": "web",
|
|
"runtime": "Runtime",
|
|
"aot": "AOT",
|
|
"arith": "Arith",
|
|
"byoc": "BYOC",
|
|
"community": "Community",
|
|
"tensorir": "TIR",
|
|
"tirx": "TIR",
|
|
"tensorflow": "Frontend",
|
|
"tflite": "Frontend",
|
|
"pytorch": "Frontend",
|
|
"torch": "Frontend",
|
|
"keras": "Frontend",
|
|
"frontend": "Frontend",
|
|
"onnx": "Frontend",
|
|
"roofline": "Misc",
|
|
"rpc": "Misc",
|
|
"tophub": "Misc",
|
|
"ux": "Misc",
|
|
"APP": "Misc",
|
|
"docker": "Docker",
|
|
"doc": "Docs",
|
|
"docs": "Docs",
|
|
"llvm": "LLVM",
|
|
"sve": "LLVM",
|
|
"ci": "CI",
|
|
"test": "CI",
|
|
"tests": "CI",
|
|
"testing": "CI",
|
|
"unittest": "CI",
|
|
"bugfix": "BugFix",
|
|
"fix": "BugFix",
|
|
"bug": "BugFix",
|
|
"hotfix": "BugFix",
|
|
"qnn": "Relay",
|
|
"quantization": "Relay",
|
|
"relax": "Relax",
|
|
"unity": "Relax",
|
|
"transform": "Relax",
|
|
"kvcache": "Relax",
|
|
"s_tir": "S-TIR",
|
|
"disco": "Disco",
|
|
"tvmscript": "TVMScript",
|
|
"tvmscripts": "TVMScript",
|
|
"tvmc": "TVMC",
|
|
"topi": "TOPI",
|
|
}
|
|
|
|
|
|
def strip_header(title: str, header: str) -> str:
|
|
pos = title.lower().find(header.lower())
|
|
if pos == -1:
|
|
return title
|
|
|
|
return title[0:pos] + title[pos + len(header) :].strip()
|
|
|
|
|
|
def sprint(*args):
|
|
print(*args, file=sys.stderr)
|
|
|
|
|
|
def create_pr_dict(cache: Path):
|
|
with open(cache, "rb") as f:
|
|
data = pickle.load(f)
|
|
|
|
sprint(data[1])
|
|
pr_dict = {}
|
|
for item in data:
|
|
prs = item["associatedPullRequests"]["nodes"]
|
|
if len(prs) != 1:
|
|
continue
|
|
|
|
pr = prs[0]
|
|
pr_dict[pr["number"]] = pr
|
|
return pr_dict
|
|
|
|
|
|
def categorize_csv_file(csv_path: str):
|
|
headings = defaultdict(lambda: defaultdict(list))
|
|
sprint("Opening CSV")
|
|
with open(csv_path) as f:
|
|
input_file = csv.DictReader(f)
|
|
|
|
i = 0
|
|
blank_cate_set = {"Misc"}
|
|
for row in input_file:
|
|
# print(row)
|
|
tags = row["pr_title_tags"].split("/")
|
|
tags = ["misc"] if len(tags) == 0 else tags
|
|
|
|
categories = map(lambda t: TAG_DICT.get(t.lower(), "Misc"), tags)
|
|
categories = list(categories)
|
|
categories = list(set(categories) - blank_cate_set)
|
|
category = "Misc" if len(categories) == 0 else categories[0]
|
|
|
|
subject = row["subject"].strip()
|
|
pr_number = row["url"].split("/")[-1]
|
|
|
|
if category == "" or subject == "":
|
|
sprint(f"Skipping {i}th pr with number: {pr_number}, row: {row}")
|
|
continue
|
|
|
|
headings[category][subject].append(pr_number)
|
|
i += 1
|
|
# if i > 30:
|
|
# break
|
|
return headings
|
|
|
|
|
|
if __name__ == "__main__":
|
|
help = "List out commits with attached PRs since a certain commit"
|
|
parser = argparse.ArgumentParser(description=help)
|
|
parser.add_argument(
|
|
"--notes", required=True, help="csv or markdown file of categorized PRs in order"
|
|
)
|
|
parser.add_argument(
|
|
"--is-pr-with-link",
|
|
required=False,
|
|
help="exported pr number with hyper-link for forum format",
|
|
)
|
|
parser.add_argument(
|
|
"--convert-with-link",
|
|
required=False,
|
|
help="make PR number in markdown file owning hyper-link",
|
|
)
|
|
args = parser.parse_args()
|
|
user = "apache"
|
|
repo = "tvm"
|
|
|
|
if args.convert_with_link:
|
|
with open(args.notes) as f:
|
|
lines = f.readlines()
|
|
formated = []
|
|
for line in lines:
|
|
match = re.search(r"#\d+", line)
|
|
if match:
|
|
pr_num_str = match.group()
|
|
pr_num_int = pr_num_str.replace("#", "")
|
|
pr_number_str = f"[#{pr_num_int}](https://github.com/apache/tvm/pull/{pr_num_int})"
|
|
line = line.replace(pr_num_str, pr_number_str)
|
|
formated.append(line)
|
|
result = "".join(formated)
|
|
print(result)
|
|
exit(0)
|
|
|
|
# 1. Create PR dict from cache file
|
|
cache = Path("out.pkl")
|
|
if not cache.exists():
|
|
sprint("run gather_prs.py first to generate out.pkl")
|
|
exit(1)
|
|
pr_dict = create_pr_dict(cache)
|
|
|
|
# 2. Categorize csv file as dict by category and subject (sub-category)
|
|
headings = categorize_csv_file(args.notes)
|
|
|
|
# 3. Summarize and sort all categories
|
|
def sorter(x):
|
|
if x == "Misc":
|
|
return 10
|
|
return 0
|
|
|
|
keys = list(headings.keys())
|
|
keys = list(sorted(keys))
|
|
keys = list(sorted(keys, key=sorter))
|
|
|
|
# 4. Generate markdown by loop categorized csv file dict
|
|
def pr_title(number, heading):
|
|
# print(f"number:{number}, heading:{heading}, len(pr_dict):{len(pr_dict)}")
|
|
try:
|
|
title = pr_dict[int(number)]["title"]
|
|
title = strip_header(title, heading)
|
|
except Exception:
|
|
sprint("The out.pkl file is not match with csv file.")
|
|
exit(1)
|
|
return title
|
|
|
|
output = ""
|
|
for key in keys:
|
|
value = headings[key]
|
|
if key == "DO NOT INCLUDE":
|
|
continue
|
|
value = dict(value)
|
|
output += f"### {key}\n"
|
|
misc = []
|
|
misc += value.get("n/a", [])
|
|
misc += value.get("Misc", [])
|
|
for pr_number in misc:
|
|
if args.is_pr_with_link:
|
|
pr_number_str = f"[#{pr_number}](https://github.com/apache/tvm/pull/{pr_number})"
|
|
else:
|
|
pr_number_str = f"#{pr_number}"
|
|
pr_str = f" * {pr_number_str} - {pr_title(pr_number, '[' + key + ']')}\n"
|
|
output += pr_str
|
|
|
|
for subheading, pr_numbers in value.items():
|
|
if subheading == "DO NOT INCLUDE":
|
|
continue
|
|
if subheading == "n/a" or subheading == "Misc":
|
|
continue
|
|
else:
|
|
output += f" * {subheading} - " + ", ".join([f"#{n}" for n in pr_numbers]) + "\n"
|
|
# print(value)
|
|
|
|
output += "\n"
|
|
|
|
# 5. Print markdown-format output
|
|
print(output)
|