chore: import upstream snapshot with attribution

tests/python/shared/utils/config.py

@@ -0,0 +1,98 @@
+# Copyright (C) CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from pathlib import Path
+
+import requests
+from cvat_sdk.api_client import ApiClient, Configuration
+from cvat_sdk.core.client import Client, Config
+
+ROOT_DIR = next(dir.parent for dir in Path(__file__).parents if dir.name == "utils")
+ASSETS_DIR = (ROOT_DIR / "assets").resolve()
+SHARE_DIR = (ROOT_DIR.parents[1] / "mounted_file_share").resolve()
+# Suppress the warning from Bandit about hardcoded passwords
+USER_PASS = "!Q@W#E$R"  # nosec
+BASE_URL = "http://localhost:8080"
+API_URL = BASE_URL + "/api/"
+
+# MiniIO settings
+MINIO_KEY = "minio_access_key"
+MINIO_SECRET_KEY = "minio_secret_key"  # nosec
+MINIO_ENDPOINT_URL = "http://localhost:9000"
+IMPORT_EXPORT_BUCKET_ID = 3
+
+
+def _to_query_params(**kwargs):
+    return "&".join([f"{k}={v}" for k, v in kwargs.items()])
+
+
+def get_server_url(endpoint, **kwargs):
+    return BASE_URL + "/" + endpoint + "?" + _to_query_params(**kwargs)
+
+
+def get_api_url(endpoint, **kwargs):
+    return API_URL + endpoint + "?" + _to_query_params(**kwargs)
+
+
+def get_method(username, endpoint, **kwargs):
+    return requests.get(get_api_url(endpoint, **kwargs), auth=(username, USER_PASS))
+
+
+def options_method(username, endpoint, **kwargs):
+    return requests.options(get_api_url(endpoint, **kwargs), auth=(username, USER_PASS))
+
+
+def delete_method(username, endpoint, **kwargs):
+    return requests.delete(get_api_url(endpoint, **kwargs), auth=(username, USER_PASS))
+
+
+def patch_method(username, endpoint, data, **kwargs):
+    return requests.patch(get_api_url(endpoint, **kwargs), json=data, auth=(username, USER_PASS))
+
+
+def post_method(username, endpoint, data, **kwargs):
+    return requests.post(get_api_url(endpoint, **kwargs), json=data, auth=(username, USER_PASS))
+
+
+def post_files_method(username, endpoint, data, files, **kwargs):
+    return requests.post(
+        get_api_url(endpoint, **kwargs), data=data, files=files, auth=(username, USER_PASS)
+    )
+
+
+def put_method(username, endpoint, data, **kwargs):
+    return requests.put(get_api_url(endpoint, **kwargs), json=data, auth=(username, USER_PASS))
+
+
+def server_get(username, endpoint, **kwargs):
+    return requests.get(get_server_url(endpoint, **kwargs), auth=(username, USER_PASS))
+
+
+def make_api_client(
+    user: str | None = None,
+    *,
+    password: str | None = None,
+    access_token: str | None = None,
+) -> ApiClient:
+    assert (
+        sum([bool(access_token), bool(user)]) <= 1
+    ), "Expected only one of the 'access_token' and 'user' args"
+
+    configuration = Configuration(host=BASE_URL)
+    if access_token:
+        configuration.access_token = access_token
+    else:
+        configuration.username = user
+        configuration.password = password or USER_PASS
+
+    return ApiClient(configuration=configuration)
+
+
+@contextmanager
+def make_sdk_client(user: str, *, password: str = None) -> Generator[Client, None, None]:
+    with Client(BASE_URL, config=Config(status_check_period=0.01)) as client:
+        client.login((user, password or USER_PASS))
+        yield client

tests/python/shared/utils/dump_objects.py

@@ -0,0 +1,119 @@
+# Copyright (C) CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+"""
+This script lists resources on the CVAT server
+and saves them to JSON files in the `assets` directory.
+Before running it, start the test instance by running:
+
+    pytest tests/python --start-services
+
+The script determines which endpoints to query by looking at the set of existing JSON files.
+For example, if `tasks.json` exists, the script will overwrite it with output of `GET /api/tasks`.
+Underscores in the file name are replaced with slashes in the URL path. If the default path for an
+asset file is different, it can be overridden via the --asset-path argument.
+In addition, `annotations.json` is always saved.
+"""
+
+import argparse
+import json
+from datetime import timezone
+from http import HTTPStatus
+from pathlib import Path
+from typing import Any
+
+from config import ASSETS_DIR, get_method
+from dateutil.parser import ParserError, parse
+
+
+def clean_list_response(data: dict[str, Any]) -> dict[str, Any]:
+    # truncate milliseconds to 3 digit precision to align with data.json
+    # "2023-03-30T09:37:31.615123Z" ->
+    # "2023-03-30T09:37:31.615000Z"
+
+    for result in data["results"]:
+        for k, v in result.items():
+            if not isinstance(v, str):
+                continue
+
+            try:
+                parsed_date = parse(v)
+            except ParserError:
+                continue
+
+            parsed_date = parsed_date.replace(
+                microsecond=parsed_date.microsecond - (parsed_date.microsecond % 1000)
+            )
+            result[k] = parsed_date.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+
+    return data
+
+
+def _parse_asset_url_path(s: str) -> tuple[str, str]:
+    asset_filename, url_path = s.lower().rsplit(":", maxsplit=1)
+    return asset_filename, url_path
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--assets-dir", type=Path, default=ASSETS_DIR)
+    parser.add_argument(
+        "--asset-path",
+        dest="asset_url_paths",
+        default=[],
+        action="append",
+        type=_parse_asset_url_path,
+        help="Repeatable, an override for the default inferred URL path for an asset. "
+        "Format: '<asset filename without extension>:<url path after api/>'",
+    )
+    args = parser.parse_args()
+
+    assets_dir: Path = args.assets_dir
+
+    asset_url_paths: dict[str, str] = dict(args.asset_url_paths)
+
+    annotations = {}
+    access_tokens = {"user": {}}
+
+    for dump_path in assets_dir.glob("*.json"):
+        asset_name = dump_path.stem
+        endpoint = asset_url_paths.get(asset_name, asset_name.replace("_", "/"))
+
+        if asset_name in ("annotations", "access_tokens"):
+            continue  # this will be handled at the end
+
+        response = get_method("admin1", endpoint, page_size="all")
+
+        with open(dump_path, "w") as f:
+            json.dump(clean_list_response(response.json()), f, indent=2, sort_keys=True)
+
+        if endpoint in ["jobs", "tasks"]:
+            obj_type = endpoint.removesuffix("s")
+            annotations[obj_type] = {}
+            for obj in response.json()["results"]:
+                oid = obj["id"]
+
+                response = get_method("admin1", f"{endpoint}/{oid}/annotations")
+                if response.status_code == HTTPStatus.OK:
+                    annotations[obj_type][oid] = response.json()
+
+        if endpoint == "users":
+            obj_type = endpoint.removesuffix("s")
+            for user in response.json()["results"]:
+                response = get_method(
+                    user["username"], "auth/access_tokens", page_size=100, sort="id"
+                )
+                if response.status_code == HTTPStatus.OK:
+                    access_tokens[obj_type][user["username"]] = response.json()["results"]
+
+    for filename, data in [
+        ("annotations.json", annotations),
+        ("access_tokens.json", access_tokens),
+    ]:
+        with open(assets_dir / filename, "w") as f:
+            json.dump(data, f, indent=2, sort_keys=True)
+
+
+if __name__ == "__main__":
+    main()

tests/python/shared/utils/dump_test_db.py

@@ -0,0 +1,458 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+"""
+Dump the test CVAT database to JSON with stable ordering.
+
+The default ``manage.py dumpdata`` output is not deterministic: record order
+within a model depends on the underlying ``SELECT`` order, which can differ
+between runs even when DB contents are identical. This causes spurious
+diffs in ``tests/python/shared/assets/cvat_db/data.json`` after each dump,
+which then need to be reviewed and reverted by hand.
+
+This script wraps ``dumpdata`` and post-processes its output to make the
+result stable:
+
+* records are sorted alphabetically by ``(model, pk)`` (the
+  ``loaddata_sorted`` management command reorders by FK dependency at
+  load time, so the on-disk order can be arbitrary);
+* field keys within each record are sorted alphabetically;
+* known unordered many-to-many fields (e.g. ``auth.user.groups``) are
+  sorted as well.
+
+It can also preserve volatile timestamp fields (``last_login``,
+``*_updated_date``, ``last_used_date`` and similar) from the existing
+output file. When preservation is enabled (the default), an unchanged
+record keeps its previous timestamp values, so re-dumping after a
+no-op test run produces zero diff. Genuinely new records get whatever
+timestamp the server wrote.
+
+Usage:
+
+    # default: run dumpdata in the test container and write the
+    # normalized output to tests/python/shared/assets/cvat_db/data.json,
+    # preserving volatile fields from the existing file.
+    python tests/python/shared/utils/dump_test_db.py
+
+    # read pre-existing dump from a file (or stdin via "-") instead of
+    # running docker exec
+    python tests/python/shared/utils/dump_test_db.py --from-file dump.json
+
+    # also refresh volatile fields (don't preserve from existing output)
+    python tests/python/shared/utils/dump_test_db.py --refresh-volatile
+
+    # write to a custom location / stdout
+    python tests/python/shared/utils/dump_test_db.py -o /tmp/data.json
+    python tests/python/shared/utils/dump_test_db.py -o -
+
+    # extend the built-in volatile/sortable maps and dumpdata excludes
+    # from a JSON file (e.g. when running against a Django project that
+    # adds models beyond the OSS apps covered by this script or you want
+    # to customize the dump).
+    python tests/python/shared/utils/dump_test_db.py \\
+        --extra-config path/to/dump_extra_config.json
+
+The extra config file has up to three optional sections:
+
+    {
+      "extra_volatile_fields":      { "<model>": ["<field>", ...], ... },
+      "extra_sortable_list_fields": { "<model>": ["<field>", ...], ... },
+      "extra_excludes":             ["<app_or_app.model>", ...]
+    }
+
+Field maps union with the built-in entries per model;
+``extra_excludes`` append to ``DUMPDATA_EXCLUDES`` (and are ignored
+when ``--from-file`` is used, since the dump is already produced).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from subprocess import PIPE, run
+from typing import Any
+
+DEFAULT_CONTAINER_NAME = "test_cvat_server_1"
+
+DEFAULT_OUTPUT = Path(__file__).resolve().parents[2] / "shared" / "assets" / "cvat_db" / "data.json"
+
+STDOUT_SENTINEL = "-"
+
+DUMPDATA_EXCLUDES = (
+    "admin",
+    "auth.permission",
+    "authtoken",
+    "contenttypes",
+    "django_rq",
+    "sessions",
+)
+
+# Fields whose value changes on every dump even when no test-relevant state
+# has changed (auto_now timestamps, last-login markers, etc.). When
+# preservation is enabled the script copies these values from the existing
+# data.json so that re-dumping an unchanged DB produces no diff.
+VOLATILE_FIELDS: dict[str, frozenset[str]] = {
+    "auth.user": frozenset({"last_login"}),
+    "engine.profile": frozenset({"last_activity_date"}),
+    "quality_control.qualityreport": frozenset(
+        {"assignee_last_updated", "gt_last_updated", "target_last_updated"}
+    ),
+    "access_tokens.accesstoken": frozenset({"last_used_date"}),
+}
+
+
+# Any field whose name is exactly ``updated_date`` or ends with
+# ``_updated_date`` is treated as volatile for every model. ``auto_now``
+# timestamps drift on every save and the exact value is not relevant for
+# tests, so we don't track the per-model list of such fields explicitly.
+def _is_blanket_volatile(field_name: str) -> bool:
+    return field_name == "updated_date" or field_name.endswith("_updated_date")
+
+
+# Fields holding many-to-many references whose element order is not
+# semantically meaningful. Sorting them stabilizes the dump even if the
+# m2m join table returns rows in a different order between runs.
+SORTABLE_LIST_FIELDS: dict[str, frozenset[str]] = {
+    "auth.user": frozenset({"groups", "user_permissions"}),
+    "auth.group": frozenset({"permissions"}),
+}
+
+
+# ``DjangoJSONEncoder`` truncates microseconds to milliseconds: a value
+# with ``microsecond`` in [1, 999] is encoded as ``.000Z``. After
+# ``loaddata`` parses that back to ``microsecond=0``, the next ``dumpdata``
+# omits the fraction entirely and writes a bare ``Z`` instead, producing a
+# spurious diff. Drop the redundant zero milliseconds up-front so the
+# round-trip is idempotent.
+ZERO_MS_PATTERN = r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\.000(Z|[+-]\d{2}:\d{2})$"
+
+_DEFAULT = object()
+
+
+def _strip_zero_milliseconds(value: Any) -> Any:
+    if not isinstance(value, str):
+        return value
+    return re.sub(ZERO_MS_PATTERN, r"\1\2", value)
+
+
+def _pk_sort_key(pk: Any) -> tuple:
+    # `pk` is usually an int but can be a list of natural-key parts.
+    # Wrap into a tuple of (type_tag, value) so heterogeneous pks order
+    # consistently without TypeError.
+    if isinstance(pk, (int, float)):
+        return (0, pk)
+    if isinstance(pk, list):
+        return (1, tuple(_pk_sort_key(p) for p in pk))
+    return (2, str(pk))
+
+
+def _normalize_record(
+    record: dict[str, Any],
+    *,
+    sortable_list_fields: dict[str, frozenset[str]] | Any = _DEFAULT,
+) -> dict[str, Any]:
+    if sortable_list_fields is _DEFAULT:
+        sortable_list_fields = SORTABLE_LIST_FIELDS
+
+    model = record.get("model", "")
+    fields = record.get("fields", {})
+    sortable = sortable_list_fields.get(model, frozenset())
+
+    new_fields: dict[str, Any] = {}
+    for key in sorted(fields):
+        value = fields[key]
+        if key in sortable and isinstance(value, list):
+            try:
+                value = sorted(value, key=lambda v: json.dumps(v, sort_keys=True))
+            except TypeError:
+                pass
+        value = _strip_zero_milliseconds(value)
+        new_fields[key] = value
+
+    return {"model": model, "pk": record.get("pk"), "fields": new_fields}
+
+
+def _preserve_volatile(
+    records: list[dict[str, Any]],
+    reference: list[dict[str, Any]],
+    *,
+    volatile_fields: dict[str, frozenset[str]] | Any = _DEFAULT,
+) -> list[dict[str, Any]]:
+    if volatile_fields is _DEFAULT:
+        volatile_fields = VOLATILE_FIELDS
+
+    ref_index = {
+        (r.get("model", ""), _pk_sort_key(r.get("pk"))): r.get("fields", {}) for r in reference
+    }
+
+    for record in records:
+        model = record.get("model", "")
+        ref_fields = ref_index.get((model, _pk_sort_key(record.get("pk"))))
+        if ref_fields is None:
+            continue
+
+        explicit = volatile_fields.get(model, frozenset())
+        fields = record.get("fields", {})
+        for name in fields:
+            if name not in ref_fields:
+                continue
+            if name in explicit or _is_blanket_volatile(name):
+                fields[name] = ref_fields[name]
+
+    return records
+
+
+def normalize(
+    records: list[dict[str, Any]],
+    *,
+    reference: list[dict[str, Any]] | None = None,
+    volatile_fields: dict[str, frozenset[str]] | Any = _DEFAULT,
+    sortable_list_fields: dict[str, frozenset[str]] | Any = _DEFAULT,
+) -> list[dict[str, Any]]:
+    """Return ``records`` sorted and field-normalized, with volatile fields
+    optionally preserved from ``reference``."""
+
+    if volatile_fields is _DEFAULT:
+        volatile_fields = VOLATILE_FIELDS
+
+    if sortable_list_fields is _DEFAULT:
+        sortable_list_fields = _DEFAULT
+
+    normalized = [_normalize_record(r, sortable_list_fields=sortable_list_fields) for r in records]
+
+    if reference is not None:
+        _preserve_volatile(normalized, reference=reference, volatile_fields=volatile_fields)
+
+    # Sort alphabetically by model name (then by pk) so the committed file
+    # has a layout that doesn't depend on Django's dependency sort. The
+    # ``loaddata_sorted`` management command reorders records by dependency
+    # at load time, so any total ordering is fine here.
+    normalized.sort(key=lambda r: (r.get("model", ""), _pk_sort_key(r.get("pk"))))
+    return normalized
+
+
+def dump_from_container(
+    container: str = DEFAULT_CONTAINER_NAME,
+    *,
+    excludes: tuple[str, ...] | list[str] = DUMPDATA_EXCLUDES,
+) -> list[dict[str, Any]]:
+    cmd = [
+        "docker",
+        "exec",
+        container,
+        "python",
+        "manage.py",
+        "dumpdata",
+        "--indent",
+        "2",
+        "--natural-foreign",
+    ]
+    for ex in excludes:
+        cmd.extend(["--exclude", ex])
+
+    proc = run(cmd, check=True, stdout=PIPE)  # nosec
+    return json.loads(proc.stdout)
+
+
+EXTRA_CONFIG_KEYS = ("extra_volatile_fields", "extra_sortable_list_fields", "extra_excludes")
+
+# Conventional Django ``app_label.modelname`` shape: lowercase identifier on
+# each side, joined by a single dot. Mismatch is a warning (not an error)
+# because the script doesn't load Django and can't authoritatively validate
+# model names — but the convention catches typos like ``engine.Task``.
+_MODEL_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*$")
+
+
+class _ConfigError(ValueError):
+    """Raised when ``--extra-config`` content is malformed."""
+
+
+def _require_non_empty_str(value: Any, where: str) -> None:
+    if not isinstance(value, str):
+        raise _ConfigError(f"{where}: expected string, got {type(value).__name__}")
+    if not value:
+        raise _ConfigError(f"{where}: expected non-empty string")
+
+
+def _validate_field_map(section: Any, where: str) -> None:
+    if not isinstance(section, dict):
+        raise _ConfigError(f"{where}: expected object, got {type(section).__name__}")
+    for model, fields in section.items():
+        _require_non_empty_str(model, f"{where} key")
+        if not _MODEL_NAME_RE.match(model):
+            print(
+                f"warning: {where}.{model!r} does not match the conventional "
+                "<app_label>.<model_lowercase> shape",
+                file=sys.stderr,
+            )
+        if not isinstance(fields, list):
+            raise _ConfigError(f"{where}.{model}: expected list, got {type(fields).__name__}")
+        seen: set[str] = set()
+        for i, field in enumerate(fields):
+            _require_non_empty_str(field, f"{where}.{model}[{i}]")
+            if field in seen:
+                raise _ConfigError(f"{where}.{model}[{i}]: duplicate field {field!r}")
+            seen.add(field)
+
+
+def _validate_excludes(section: Any, where: str) -> None:
+    if not isinstance(section, list):
+        raise _ConfigError(f"{where}: expected list, got {type(section).__name__}")
+    seen: set[str] = set()
+    for i, item in enumerate(section):
+        _require_non_empty_str(item, f"{where}[{i}]")
+        if item in seen:
+            raise _ConfigError(f"{where}[{i}]: duplicate exclude {item!r}")
+        seen.add(item)
+
+
+def _load_extra_config(path: Path) -> dict[str, Any]:
+    """Read and validate an extension config file.
+
+    See ``EXTRA_CONFIG_KEYS`` for the allowed top-level sections; missing
+    sections default to empty. Malformed content raises ``_ConfigError``
+    with a JSON-pointer-style path so the offending entry is easy to find.
+    """
+    raw = json.loads(path.read_text())
+    if not isinstance(raw, dict):
+        raise _ConfigError(f"{path}: top-level must be an object, got {type(raw).__name__}")
+    unknown = sorted(set(raw) - set(EXTRA_CONFIG_KEYS))
+    if unknown:
+        raise _ConfigError(
+            f"{path}: unknown top-level keys: {unknown}. " f"valid keys: {list(EXTRA_CONFIG_KEYS)}"
+        )
+
+    extra_volatile = raw.get("extra_volatile_fields", {})
+    extra_sortable = raw.get("extra_sortable_list_fields", {})
+    extra_excludes = raw.get("extra_excludes", [])
+
+    _validate_field_map(extra_volatile, "extra_volatile_fields")
+    _validate_field_map(extra_sortable, "extra_sortable_list_fields")
+    _validate_excludes(extra_excludes, "extra_excludes")
+
+    return {
+        "extra_volatile_fields": extra_volatile,
+        "extra_sortable_list_fields": extra_sortable,
+        "extra_excludes": extra_excludes,
+    }
+
+
+def _merge_field_map(
+    base: dict[str, frozenset[str]],
+    extra: dict[str, list[str]],
+) -> dict[str, frozenset[str]]:
+    merged = dict(base)
+    for model, fields in extra.items():
+        merged[model] = merged.get(model, frozenset()) | frozenset(fields)
+    return merged
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default=str(DEFAULT_OUTPUT),
+        help=f'Output file (default: %(default)s). Use "{STDOUT_SENTINEL}" for stdout.',
+    )
+
+    parser.add_argument(
+        "--from-file",
+        type=str,
+        help="Read pre-existing dumpdata JSON from this file instead of running docker exec. "
+        'Use "-" to read from stdin.',
+    )
+    parser.add_argument(
+        "--container",
+        default=DEFAULT_CONTAINER_NAME,
+        help="Docker container to dump from (default: %(default)s).",
+    )
+    parser.add_argument(
+        "--refresh-volatile",
+        action="store_true",
+        help="Take volatile timestamp fields from the new dump instead of "
+        "preserving them from the existing output file.",
+    )
+    parser.add_argument(
+        "--extra-config",
+        type=Path,
+        help="Path to a JSON file extending the built-in volatile/sortable "
+        "maps and dumpdata excludes. Sections (all optional): "
+        "extra_volatile_fields, extra_sortable_list_fields, extra_excludes. "
+        "Field maps union with the built-in entries per model; "
+        "extra_excludes append to the built-in DUMPDATA_EXCLUDES (and are "
+        "ignored when --from-file is used).",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parsed_args = _build_parser().parse_args(argv)
+
+    if parsed_args.extra_config:
+        try:
+            extra = _load_extra_config(parsed_args.extra_config)
+        except _ConfigError as exc:
+            print(f"error: invalid --extra-config: {exc}", file=sys.stderr)
+            return 2
+    else:
+        extra = {
+            "extra_volatile_fields": {},
+            "extra_sortable_list_fields": {},
+            "extra_excludes": [],
+        }
+    volatile_fields = _merge_field_map(VOLATILE_FIELDS, extra["extra_volatile_fields"])
+    sortable_list_fields = _merge_field_map(
+        SORTABLE_LIST_FIELDS, extra["extra_sortable_list_fields"]
+    )
+    excludes = list(DUMPDATA_EXCLUDES) + [
+        ex for ex in extra["extra_excludes"] if ex not in DUMPDATA_EXCLUDES
+    ]
+    if extra["extra_excludes"] and parsed_args.from_file:
+        print(
+            "warning: extra_excludes is ignored when reading the dump from "
+            "--from-file (the dump is already produced).",
+            file=sys.stderr,
+        )
+
+    if parsed_args.from_file == "-":
+        records = json.loads(sys.stdin.read())
+    elif parsed_args.from_file:
+        records = json.loads(Path(parsed_args.from_file).read_text())
+    else:
+        records = dump_from_container(parsed_args.container, excludes=excludes)
+
+    reference_dump: list[dict[str, Any]] | None = None
+    output_path: Path | None = (
+        None if parsed_args.output == STDOUT_SENTINEL else Path(parsed_args.output)
+    )
+    if not parsed_args.refresh_volatile and output_path is not None and output_path.exists():
+        reference_dump = json.loads(output_path.read_text())
+
+    normalized = normalize(
+        records,
+        reference=reference_dump,
+        volatile_fields=volatile_fields,
+        sortable_list_fields=sortable_list_fields,
+    )
+    serialized = json.dumps(normalized, indent=2, ensure_ascii=False)
+
+    if output_path is None:
+        print(serialized)
+    else:
+        output_path.write_text(serialized + "\n")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/python/shared/utils/helpers.py

@@ -0,0 +1,131 @@
+# Copyright (C) CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import io
+import subprocess
+from collections.abc import Generator
+from fractions import Fraction
+from io import BytesIO
+from pathlib import Path
+
+import av
+import av.video.reformatter
+import librosa
+import numpy as np
+from PIL import Image
+
+from shared.fixtures.init import get_server_image_tag
+
+
+def generate_image_file(filename="image.png", size=(100, 50), color=(0, 0, 0)):
+    f = BytesIO()
+    f.name = filename
+    image = Image.new("RGB", size=size, color=color)
+    image.save(f)
+    f.seek(0)
+
+    return f
+
+
+def generate_image_files(
+    count: int,
+    *,
+    prefixes: list[str] | None = None,
+    filenames: list[str] | None = None,
+    sizes: list[tuple[int, int]] | None = None,
+) -> list[BytesIO]:
+    assert not (prefixes and filenames), "prefixes cannot be used together with filenames"
+    assert not prefixes or len(prefixes) == count
+    assert not filenames or len(filenames) == count
+
+    images = []
+    for i in range(count):
+        prefix = prefixes[i] if prefixes else ""
+        filename = f"{prefix}{i}.jpeg" if not filenames else filenames[i]
+        image = generate_image_file(
+            filename, color=(i, i, i), **({"size": sizes[i]}) if sizes else {}
+        )
+        images.append(image)
+
+    return images
+
+
+def generate_video_file(
+    num_frames: int,
+    *,
+    size: tuple[int, int] = (100, 50),
+    invalid_keyframes: bool = False,
+) -> BytesIO:
+    f = BytesIO()
+    f.name = "video.mkv"
+    chapters = [
+        {
+            "id": 0,
+            "start": 0,
+            "end": 100,
+            "time_base": Fraction(1, 1000),
+            "metadata": {"title": "Intro"},
+        }
+    ]
+
+    with av.open(f, "w") as container:
+        container.set_chapters(chapters)
+        stream = container.add_stream("mjpeg", rate=60)
+        stream.width = size[0]
+        stream.height = size[1]
+        stream.color_range = av.video.reformatter.ColorRange.JPEG
+
+        for i in range(num_frames):
+            frame = av.VideoFrame.from_image(Image.new("RGB", size=size, color=(i, i, i)))
+            for packet in stream.encode(frame):
+                if invalid_keyframes:
+                    # Specify pts/dts values that result in 0 valid keyframes
+                    packet.pts = 10
+                    packet.dts = 10
+
+                container.mux(packet)
+
+    f.seek(0)
+
+    return f
+
+
+def read_video_file(file: BytesIO) -> Generator[Image.Image, None, None]:
+    file.seek(0)
+
+    with av.open(file) as container:
+        video_stream = container.streams.video[0]
+
+        for frame in container.decode(video_stream):
+            yield frame.to_image()
+
+
+def generate_manifest(path: str) -> None:
+    command = [
+        "docker",
+        "run",
+        "--rm",
+        "-u",
+        "root:root",
+        "-v",
+        f"{path}:/local",
+        "--entrypoint",
+        "python3",
+        get_server_image_tag(),
+        "utils/dataset_manifest/create.py",
+        "--output-dir",
+        "/local",
+        "/local",
+    ]
+    try:
+        subprocess.check_output(command, stderr=subprocess.PIPE)
+    except subprocess.CalledProcessError as e:
+        print(e.stderr.decode("utf-8"))
+        raise
+
+
+def read_audio_pcm(
+    f: Path | io.IOBase, *, offset_ms: int = 0, rate: int = 8000
+) -> tuple[np.ndarray, float]:
+    return librosa.load(f, mono=True, sr=rate, offset=offset_ms / 1000)

tests/python/shared/utils/resource_import_export.py

@@ -0,0 +1,220 @@
+import functools
+import json
+from abc import ABC, abstractmethod
+from contextlib import ExitStack
+from http import HTTPStatus
+from time import sleep
+from typing import Any, TypeVar
+
+import pytest
+
+T = TypeVar("T")
+
+from shared.utils.config import get_method, post_method
+
+FILENAME_TEMPLATE = "cvat/{}/{}.zip"
+EXPORT_FORMAT = "CVAT for images 1.1"
+IMPORT_FORMAT = "CVAT 1.1"
+
+
+def _make_custom_resource_params(resource: str, obj: str, cloud_storage_id: int) -> dict[str, Any]:
+    return {
+        "filename": FILENAME_TEMPLATE.format(obj, resource),
+        "location": "cloud_storage",
+        "cloud_storage_id": cloud_storage_id,
+    }
+
+
+def _make_default_resource_params(resource: str, obj: str) -> dict[str, Any]:
+    return {
+        "filename": FILENAME_TEMPLATE.format(obj, resource),
+    }
+
+
+def _make_export_resource_params(
+    resource: str, is_default: bool = True, **kwargs
+) -> dict[str, Any]:
+    func = _make_default_resource_params if is_default else _make_custom_resource_params
+    params = func(resource, **kwargs)
+    if resource != "backup":
+        params["format"] = EXPORT_FORMAT
+        params["save_images"] = resource == "dataset"
+
+    return params
+
+
+def _make_import_resource_params(
+    resource: str, is_default: bool = True, **kwargs
+) -> dict[str, Any]:
+    func = _make_default_resource_params if is_default else _make_custom_resource_params
+    params = func(resource, **kwargs)
+    if resource != "backup":
+        params["format"] = IMPORT_FORMAT
+    return params
+
+
+# FUTURE-TODO: reuse common logic from rest_api/utils
+def _wait_request(
+    user: str,
+    request_id: str,
+    *,
+    sleep_interval: float = 0.1,
+    number_of_checks: int = 100,
+):
+    for _ in range(number_of_checks):
+        sleep(sleep_interval)
+        response = get_method(user, f"requests/{request_id}")
+        assert response.status_code == HTTPStatus.OK
+
+        request_details = json.loads(response.content)
+        status = request_details["status"]
+        assert status in {"started", "queued", "finished", "failed"}
+        if status in {"finished", "failed"}:
+            return
+
+
+class _CloudStorageResourceTest(ABC):
+    @staticmethod
+    @abstractmethod
+    def _make_client():
+        pass
+
+    @pytest.fixture(autouse=True)
+    def setup(self, admin_user: str):
+        self.user = admin_user
+        self.client = self._make_client()
+        self.exit_stack = ExitStack()
+        with self.exit_stack:
+            yield
+
+    def _ensure_file_created(self, func: T, storage: dict[str, Any]) -> T:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            filename = kwargs["filename"]
+            bucket = storage["resource"]
+
+            # check that file doesn't exist on the bucket
+            assert not self.client.file_exists(bucket=bucket, filename=filename)
+
+            func(*args, **kwargs)
+
+            # check that file exists on the bucket
+            assert self.client.file_exists(bucket=bucket, filename=filename)
+
+        return wrapper
+
+    def _export_resource_to_cloud_storage(
+        self,
+        obj_id: int,
+        obj: str,
+        resource: str,
+        *,
+        user: str,
+        _expect_status: HTTPStatus = HTTPStatus.ACCEPTED,
+        **kwargs,
+    ):
+
+        # initialize the export process
+        response = post_method(
+            user,
+            f"{obj}/{obj_id}/{resource if resource != 'annotations' else 'dataset'}/export",
+            data=None,
+            **kwargs,
+        )
+        assert response.status_code == _expect_status
+
+        if _expect_status == HTTPStatus.FORBIDDEN:
+            return
+
+        rq_id = json.loads(response.content).get("rq_id")
+        assert rq_id, "The rq_id was not found in server request"
+
+        _wait_request(user, rq_id)
+
+    def _import_resource_from_cloud_storage(
+        self, url: str, *, user: str, _expect_status: HTTPStatus = HTTPStatus.ACCEPTED, **kwargs
+    ) -> None:
+        response = post_method(user, url, data=None, **kwargs)
+        status = response.status_code
+
+        assert status == _expect_status, status
+        if status == HTTPStatus.FORBIDDEN:
+            return
+
+        rq_id = response.json().get("rq_id")
+        assert rq_id, "The rq_id parameter was not found in the server response"
+
+        _wait_request(user, rq_id)
+
+    def _import_annotations_from_cloud_storage(
+        self,
+        obj_id,
+        obj,
+        *,
+        user,
+        _expect_status: HTTPStatus = HTTPStatus.ACCEPTED,
+        _check_uploaded: bool = True,
+        **kwargs,
+    ):
+        url = f"{obj}/{obj_id}/annotations"
+        self._import_resource_from_cloud_storage(
+            url, user=user, _expect_status=_expect_status, **kwargs
+        )
+
+        if _expect_status == HTTPStatus.FORBIDDEN:
+            return
+
+        if _check_uploaded:
+            response = get_method(user, url)
+            assert response.status_code == HTTPStatus.OK
+
+            annotations = response.json()
+
+            assert len(annotations["shapes"])
+
+    def _import_backup_from_cloud_storage(
+        self, obj, *, user, _expect_status: HTTPStatus = HTTPStatus.ACCEPTED, **kwargs
+    ):
+        self._import_resource_from_cloud_storage(
+            f"{obj}/backup", user=user, _expect_status=_expect_status, **kwargs
+        )
+
+    def _import_dataset_from_cloud_storage(
+        self, obj_id, obj, *, user, _expect_status: HTTPStatus = HTTPStatus.ACCEPTED, **kwargs
+    ):
+        self._import_resource_from_cloud_storage(
+            f"{obj}/{obj_id}/dataset", user=user, _expect_status=_expect_status, **kwargs
+        )
+
+    def _import_resource(self, cloud_storage: dict[str, Any], resource_type: str, *args, **kwargs):
+        methods = {
+            "annotations": self._import_annotations_from_cloud_storage,
+            "dataset": self._import_dataset_from_cloud_storage,
+            "backup": self._import_backup_from_cloud_storage,
+        }
+
+        org_id = cloud_storage["organization"]
+        if org_id:
+            kwargs.setdefault("org_id", org_id)
+
+        kwargs.setdefault("user", self.user)
+
+        return methods[resource_type](*args, **kwargs)
+
+    def _export_resource(self, cloud_storage: dict[str, Any], *args, **kwargs):
+        org_id = cloud_storage["organization"]
+        if org_id:
+            kwargs.setdefault("org_id", org_id)
+
+        kwargs.setdefault("user", self.user)
+
+        export_callback = self._ensure_file_created(
+            self._export_resource_to_cloud_storage, storage=cloud_storage
+        )
+        export_callback(*args, **kwargs)
+
+        self.exit_stack.callback(
+            self.client.remove_file,
+            bucket=cloud_storage["resource"],
+            filename=kwargs["filename"],
+        )

tests/python/shared/utils/s3.py

@@ -0,0 +1,68 @@
+# Copyright (C) CVAT.ai Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from io import BytesIO
+
+import boto3
+from botocore.exceptions import ClientError
+
+from shared.utils.config import MINIO_ENDPOINT_URL, MINIO_KEY, MINIO_SECRET_KEY
+
+
+class S3Client:
+    def __init__(
+        self, endpoint_url: str, *, access_key: str, secret_key: str, bucket: str | None = None
+    ) -> None:
+        self.client = self._make_boto_client(
+            endpoint_url=endpoint_url, access_key=access_key, secret_key=secret_key
+        )
+        self.bucket = bucket
+
+    @staticmethod
+    def _make_boto_client(endpoint_url: str, *, access_key: str, secret_key: str):
+        s3 = boto3.resource(
+            "s3",
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            endpoint_url=endpoint_url,
+        )
+        return s3.meta.client
+
+    def create_file(self, filename: str, data: bytes = b"", *, bucket: str | None = None):
+        bucket = bucket or self.bucket
+        assert bucket
+        self.client.put_object(Body=data, Bucket=bucket, Key=filename)
+
+    def remove_file(self, filename: str, *, bucket: str | None = None):
+        bucket = bucket or self.bucket
+        assert bucket
+        self.client.delete_object(Bucket=bucket, Key=filename)
+
+    def file_exists(self, filename: str, *, bucket: str | None = None) -> bool:
+        bucket = bucket or self.bucket
+        assert bucket
+        try:
+            self.client.head_object(Bucket=bucket, Key=filename)
+            return True
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "404":
+                return False
+            else:
+                raise
+
+    def download_fileobj(self, key: str, *, bucket: str | None = None) -> bytes:
+        bucket = bucket or self.bucket
+        assert bucket
+        with BytesIO() as data:
+            self.client.download_fileobj(Bucket=bucket, Key=key, Fileobj=data)
+            return data.getvalue()
+
+
+def make_client(*, bucket: str | None = None) -> S3Client:
+    return S3Client(
+        endpoint_url=MINIO_ENDPOINT_URL,
+        access_key=MINIO_KEY,
+        secret_key=MINIO_SECRET_KEY,
+        bucket=bucket,
+    )