chore: import upstream snapshot with attribution

ReSA/scripts/local_eval_math.sh

@@ -0,0 +1,9 @@
+export TORCH_IND_SYM_NODE_NO_SYMPY=1
+cd llm/
+torchrun --nproc_per_node=1 --nnodes=1 --master_port=29388 eval.py \
+    --limit 512 --batch_size 4 \
+    --checkpoint_dir /path/to/DeepSeek-R1-Distill-Qwen-1.5B --downstream_task math \
+    --save_feature resa_0.1_32 \
+    --output_folder /path/to/result/ \
+    --resa_rec_freq 32 \
+    --resa_sparse_ratio 0.1

ReSA/scripts/math_eval_result.sh

@@ -0,0 +1,17 @@
+tasks=("minerva_math" "gaokao2023en" "olympiadbench" "aime24" "amc23")
+# tasks=("gsm8k" "math" "svamp" "asdiv" "mawps" "carp_en" "tabmwp" "minerva_math" "gaokao2023en" "olympiadbench" "college_math" "aime24" "amc23")
+
+output_folder=$1
+result_file=$2
+
+prompt_type="r1"
+eval_num=-1
+
+export TQDM_DISABLE=1
+for task in "${tasks[@]}"; do
+    python scripts/math_eval_result_length.py \
+        --data_names $task \
+        --prompt_type $prompt_type \
+        --result_file ${output_folder}/${task}/${result_file} \
+        --eval_num $eval_num
+done

ReSA/scripts/math_eval_result_length.py

@@ -0,0 +1,89 @@
+import argparse
+from transformers import LlamaTokenizerFast
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
+from math_utils import evaluate, load_jsonl
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_names", default="gsm8k", type=str)
+    parser.add_argument("--result_file", default=None, type=str)
+    parser.add_argument("--prompt_type", default="direct", type=str)
+    parser.add_argument("--eval_num", default=-1, type=int)
+    args = parser.parse_args()
+    return args
+
+
+def eval_math_acc(args, data_name):
+    result_file = args.result_file.format(data_name=data_name)
+    print(result_file)
+    all_samples = list(load_jsonl(result_file))
+    if args.eval_num > 0:
+        all_samples = all_samples[: args.eval_num]
+    
+    tokenizer = LlamaTokenizerFast.from_pretrained('/mnt/msranlp/tianzhu/ckpt/DeepSeek-R1-Distill-Qwen-1.5B')
+    
+    avg_len = 0
+    threshold = 2048
+    below_num, above_num, below_acc, above_acc = 0, 0, 0, 0
+    for sample in all_samples:
+        length = len(tokenizer.encode(sample["code"][0]))
+        avg_len += length
+        _, result_json = evaluate(
+            samples=[sample],
+            data_name=data_name,
+            prompt_type=args.prompt_type,
+            execute=True,
+        )
+        if length <= threshold:
+            below_num += 1
+            below_acc += result_json["acc"]
+        else:
+            above_num += 1
+            above_acc += result_json["acc"]
+    total_num = below_num + above_num
+    total_acc = (below_acc + above_acc) / total_num
+    avg_len /= len(all_samples)
+    print(f"{data_name} total acc: {total_acc:.1f} ({total_num})")
+    print(
+        f"{data_name} below {threshold} acc: {int(below_acc/100)}/{below_num}/{below_acc/below_num if below_num > 0 else 0:.1f}"
+    )
+    print(
+        f"{data_name} above {threshold} acc: {int(above_acc/100)}/{above_num}/{above_acc/above_num if above_num > 0 else 0:.1f}"
+    )
+    print(f"{data_name} avg len: {avg_len:.1f}")
+    
+    print(f"{total_acc:.1f}")
+    print(f"{int(below_acc/100)}/{below_num}/{below_acc/below_num if below_num > 0 else 0:.1f}")
+    print(f"{int(above_acc/100)}/{above_num}/{above_acc/above_num if above_num > 0 else 0:.1f}")
+    print(f"{avg_len:.1f}")
+    # print(result_json)
+
+    return result_json
+
+
+def main(args):
+    data_names = args.data_names
+    data_list = data_names.split(",")
+    results = []
+    for data_name in data_list:
+        results.append(eval_math_acc(args, data_name))
+
+    # add "avg" result to data_list and results
+    data_list.append("avg")
+    results.append(
+        {
+            "acc": sum([result["acc"] for result in results]) / len(results),
+        }
+    )
+
+    # print all results
+    pad = max([len(data_name) for data_name in data_list])
+    print("\t".join(data_name.ljust(pad, " ") for data_name in data_list))
+    print("\t".join([f"{result['acc']:.1f}".ljust(pad, " ") for result in results]))
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

ReSA/scripts/setup_math_eval.sh

@@ -0,0 +1,9 @@
+CURRENT_DIR=$(pwd)
+cd /tmp/
+git clone https://github.com/QwenLM/Qwen2.5-Math.git
+cd Qwen2.5-Math/evaluation/latex2sympy
+pip install -e .
+cd $CURRENT_DIR
+
+pip install fairscale tensorboard optimum transformers tokenizers datasets einops opt_einsum iopath numpy==1.23.0 tiktoken boto3 sentencepiece scikit-learn huggingface-hub ninja wandb rouge-score xopen
+pip install tqdm python_dateutil sympy==1.12 antlr4-python3-runtime==4.11.1 word2number Pebble timeout-decorator