wehub-resource-sync
45 lines
1.1 KiB
Python
45 lines
1.1 KiB
Python
import os
|
|
|
|
from ray.experimental.raysort.types import ByteCount, PartId, RecordCount
|
|
|
|
__DIR__ = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
# Basics
|
|
RECORD_SIZE = 100 # bytes
|
|
|
|
# Progress Tracker Actor
|
|
PROGRESS_TRACKER_ACTOR = "ProgressTrackerActor"
|
|
|
|
# Executable locations
|
|
GENSORT_PATH = os.path.join(__DIR__, "bin/gensort/64/gensort")
|
|
VALSORT_PATH = os.path.join(__DIR__, "bin/gensort/64/valsort")
|
|
|
|
# Filenames
|
|
WORK_DIR = "/tmp/raysort"
|
|
INPUT_MANIFEST_FILE = os.path.join(WORK_DIR, "input-manifest.csv")
|
|
OUTPUT_MANIFEST_FILE = os.path.join(WORK_DIR, "output-manifest.csv")
|
|
DATA_DIR_FMT = {
|
|
"input": "{mnt}/tmp/input/",
|
|
"output": "{mnt}/tmp/output/",
|
|
"temp": "{mnt}/tmp/temp/",
|
|
}
|
|
FILENAME_FMT = {
|
|
"input": "input-{part_id:08}",
|
|
"output": "output-{part_id:08}",
|
|
"temp": "temp-{part_id:08}",
|
|
}
|
|
|
|
# Prometheus config
|
|
PROM_RAY_EXPORTER_PORT = 8090
|
|
PROM_NODE_EXPORTER_PORT = 8091
|
|
|
|
|
|
# Convenience functions
|
|
def bytes_to_records(n_bytes: ByteCount) -> RecordCount:
|
|
assert n_bytes % RECORD_SIZE == 0
|
|
return int(n_bytes / RECORD_SIZE)
|
|
|
|
|
|
def merge_part_ids(reducer_id: PartId, mapper_id: PartId) -> PartId:
|
|
return reducer_id * 1_000_000 + mapper_id
|