chore: import upstream snapshot with attribution

tests/setup/__main__.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+
+from .data.create_sample_jsonl import create_sample_jsonl
+from .models.create_hf_model import create_hf_model
+
+print("Setup test data and models...")
+
+parser = argparse.ArgumentParser("Setup test data and models.")
+parser.add_argument("--save_dir", required=True, help="Root save directory for artifacts")
+parser.add_argument("--overwrite", action="store_true", help="Overwrite existing files and directories")
+args = parser.parse_args()
+
+print(f"Arguments are: {vars(args)}")
+
+os.makedirs(args.save_dir, exist_ok=True)
+
+create_sample_jsonl(
+    output_file=os.path.join(args.save_dir, "test_quantization", "test.json"),
+    overwrite=args.overwrite,
+)
+
+create_hf_model(
+    model_name_or_path="/home/TestData/nlp/megatron_llama/llama-ci-hf",
+    output_dir=os.path.join(args.save_dir, "megatron_llama/llama-ci-hf-tiny"),
+    config_updates={"hidden_size": 256, "num_attention_heads": 4, "num_hidden_layers": 2, "num_key_value_heads": 4},
+    overwrite=args.overwrite,
+)
+print("Setup done.")

tests/setup/data/create_sample_jsonl.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+
+"""
+Create sample JSONL file for functional testing. Each line contains a dictionary
+with a single element "text" for storing data.
+"""
+
+
+def create_sample_jsonl(output_file: str, overwrite: bool = False):
+    """Create sample JSONL."""
+    if os.path.isfile(output_file) and not overwrite:
+        print(f"File {output_file} exists and overwrite flag is not set so exiting.")
+        return
+
+    texts = [
+        "Sample data for functional tests",
+        "Once upon a time, in the middle of a dense forest, there was a small house, where lived a pretty little girl "
+        "named Little Red Riding Hood.",
+        "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore "
+        "magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea "
+        "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat "
+        "nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit "
+        "anim id est laborum...",
+        "Next please!",
+        "¡H E L L O   W O R L D!",
+        "Korzystając z okazji chciałbym pozdrowić całą moją rodzinę i przyjaciół",
+    ]
+    print(f"Writing {len(texts)} line(s) to {output_file}...")
+    os.makedirs(os.path.dirname(os.path.abspath(output_file)), exist_ok=True)
+    with open(output_file, mode="w", encoding="utf-8") as f:
+        for text in texts:
+            json.dump({"text": text}, f)
+            f.write("\n")
+    print("OK.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Create sample JSONL file.")
+    parser.add_argument("--output_file", required=True, help="Output file name")
+    parser.add_argument("--overwrite", action="store_true", help="Overwrite file if it exists")
+    args = parser.parse_args()
+    create_sample_jsonl(args.output_file, args.overwrite)

tests/setup/data/create_sample_lambada.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+
+"""
+Create a dataset with five Lambada test examples for functional testing. Each line
+contains a dictionary with a "text_before_last_word" and "last_word" keys.
+"""
+
+
+def create_sample_lambada(output_file: str, overwrite: bool = False):
+    """Create JSON file with a few Lambada examples."""
+    if os.path.isfile(output_file) and not overwrite:
+        print(f"File {output_file} exists and overwrite flag is not set so exiting.")
+        return
+
+    texts = [
+        {
+            "text_before_last_word": "In my palm is a clear stone , and inside it is a small ivory statuette . A guardian angel .\n\n\" Figured if you re going to be out at night getting hit by cars , you might as well have some backup .\"\n\n I look at him , feeling stunned . Like this is some sort of sign . But as I stare at Harlin , his mouth curved in a confident grin , I don t care about",
+            "last_word": "signs",
+        },
+        {
+            "text_before_last_word": "Give me a minute to change and I ll meet you at the docks .\" She d forced those words through her teeth .\n\n\" No need to change . We won t be that long .\"\n\n Shane gripped her arm and started leading her to the dock .\n\n\" I can make it there on my own ,",
+            "last_word": "Shane",
+        },
+        {
+            "text_before_last_word": "\" Only one source I know of that would be likely to cough up enough money to finance a phony sleep research facility and pay people big bucks to solve crimes in their dreams ,\" Farrell concluded dryly .\n\n\" What can I say ?\" Ellis unfolded his arms and widened his hands . \" Your tax dollars at work .\"\n\n Before Farrell could respond , Leila s voice rose from inside the house .\n\n\" No insurance ?\" she wailed . \" What do you mean you don t have any",
+            "last_word": "insurance",
+        },
+        {
+            "text_before_last_word": "Helen s heart broke a little in the face of Miss Mabel s selfless courage . She thought that because she was old , her life was of less value than the others . For all Helen knew , Miss Mabel had a lot more years to live than she did . \" Not going to happen ,\" replied",
+            "last_word": "Helen",
+        },
+        {
+            "text_before_last_word": "Preston had been the last person to wear those chains , and I knew what I d see and feel if they were slipped onto my skin the Reaper s unending hatred of me . I d felt enough of that emotion already in the amphitheater . I didn t want to feel anymore .\n\n\" Don t put those on me ,\" I whispered . \" Please .\"\n\n Sergei looked at me , surprised by my low , raspy please , but he put down the",
+            "last_word": "chains",
+        },
+    ]
+
+    print(f"Writing {len(texts)} line(s) to {output_file}...")
+    os.makedirs(os.path.dirname(os.path.abspath(output_file)), exist_ok=True)
+    with open(output_file, mode="w", encoding="utf-8") as f:
+        json.dump(texts, f)
+    print("OK.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Create a sample from Lambada test dataset.")
+    parser.add_argument("--output_file", required=True, help="Output file name")
+    parser.add_argument("--overwrite", action="store_true", help="Overwrite file if it exists")
+    args = parser.parse_args()
+    create_sample_lambada(args.output_file, args.overwrite)

tests/setup/models/create_hf_model.py

@@ -0,0 +1,102 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+
+from typing import Any, Dict, Optional
+
+import transformers
+
+"""
+Create a randomly initialized HuggingFace model for testing purposes.
+
+Model can be specified by name or path for creating its config and tokenizer using
+HuggingFace transformers AutoConfig and AutoTokenizer functions.
+
+Parameter config_updates can be used to override specific model config fields to make
+it smaller, for example, by changing number of layers or hidden layers dimensionality,
+making it adequate for testing purposes. This parameter should be specified as
+a dictionary that can be parsed using json.loads method.
+
+Example usage for Llama2 model (requires HF login):
+```
+python tests/setup/models/create_tiny_hf_model.py \
+  --model_name_or_path meta-llama/Llama-2-7b-hf \
+  --output_dir tiny_llama2_hf \
+  --config_updates '{"hidden_size": 128, "num_attention_heads": 4, "num_hidden_layers": 2, "num_key_value_heads": 4}'
+```
+"""
+
+
+def get_hf_model_class(hf_config):
+    """Get HuggingFace model class from config."""
+    if len(hf_config.architectures) > 1:
+        print(f"More than one model architecture available, choosing 1st: {hf_config.architectures}")
+    model_name = hf_config.architectures[0]
+    model_class = getattr(transformers, model_name)
+    return model_class
+
+
+def create_hf_model(
+    model_name_or_path: str, output_dir: str, config_updates: Optional[Dict[str, Any]] = None, overwrite: bool = False
+):
+    """Create HuggingFace model with optional config updates."""
+    if os.path.isdir(output_dir) and not overwrite:
+        print(f"Output directory {output_dir} exists and overwrite flag is not set so exiting.")
+        return
+
+    hf_config = transformers.AutoConfig.from_pretrained(model_name_or_path)
+    tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path)
+    model_class = get_hf_model_class(hf_config)
+
+    if config_updates is not None:
+        hf_config.update(config_updates)
+    print(hf_config)
+
+    model = model_class(hf_config)
+    print(model)
+
+    os.makedirs(output_dir, exist_ok=True)
+    print(f"Saving model to {output_dir}...")
+    tokenizer.save_pretrained(output_dir)
+    model.save_pretrained(output_dir)
+    print("OK.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Create a HuggingFace model (random initialization) for testing purposes.")
+    parser.add_argument(
+        "--model_name_or_path",
+        required=True,
+        help="Model name or local path with model config and tokenizer",
+    )
+    parser.add_argument(
+        "--output_dir",
+        required=True,
+        help="Output directory",
+    )
+    parser.add_argument(
+        "--config_updates",
+        type=json.loads,
+        help="Parameter updates in JSON format to overwrite for model config",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action="store_true",
+        help="Overwrite file if it exists",
+    )
+    args = parser.parse_args()
+    create_hf_model(args.model_name_or_path, args.output_dir, args.config_updates, args.overwrite)