wehub-resource-sync
70 lines
2.4 KiB
Python
70 lines
2.4 KiB
Python
import os
|
|
import argparse
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
from tqdm import tqdm
|
|
import pandas as pd
|
|
|
|
|
|
def add_args(parser: argparse.ArgumentParser):
|
|
pass
|
|
|
|
|
|
def get_metadata(**kwargs):
|
|
raise NotImplementedError("TexVerse metadata should be built from metadata.json using build_texverse_metadata.py")
|
|
|
|
|
|
def download(metadata, root, **kwargs):
|
|
raise NotImplementedError("TexVerse GLB files are already available locally. No download needed.")
|
|
|
|
|
|
def _process_instance(args):
|
|
"""Worker function for ProcessPoolExecutor (must be top-level for pickling)"""
|
|
metadatum, output_dir, func = args
|
|
try:
|
|
local_path = metadatum['local_path']
|
|
sha256 = metadatum['sha256']
|
|
file = os.path.join(output_dir, local_path)
|
|
record = func(file, sha256)
|
|
return record
|
|
except Exception as e:
|
|
print(f"Error processing object {metadatum.get('sha256', '?')}: {e}")
|
|
return None
|
|
|
|
|
|
def foreach_instance(metadata, output_dir, func, max_workers=None, desc='Processing objects', timeout=None) -> pd.DataFrame:
|
|
import os
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed, TimeoutError
|
|
from tqdm import tqdm
|
|
|
|
metadata = metadata.to_dict('records')
|
|
|
|
max_workers = max_workers or os.cpu_count()
|
|
records = []
|
|
timeout_count = 0
|
|
|
|
try:
|
|
with ProcessPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = {
|
|
executor.submit(_process_instance, (m, output_dir, func)): m['sha256']
|
|
for m in metadata
|
|
}
|
|
for future in tqdm(as_completed(futures), total=len(futures), desc=desc):
|
|
sha256 = futures[future]
|
|
try:
|
|
r = future.result(timeout=timeout)
|
|
if r is not None:
|
|
records.append(r)
|
|
except TimeoutError:
|
|
timeout_count += 1
|
|
print(f"Timeout processing object {sha256} (>{timeout}s)")
|
|
records.append({'sha256': sha256, 'error': f'Timeout (>{timeout}s)'})
|
|
except Exception as e:
|
|
print(f"Error processing object {sha256}: {e}")
|
|
except Exception as e:
|
|
print(f"Error happened during processing: {e}")
|
|
|
|
if timeout_count > 0:
|
|
print(f"Total timeout: {timeout_count} objects")
|
|
|
|
return pd.DataFrame.from_records(records)
|