commit 10df941303a292b0f75ca6151b1781b53a8a578d Author: wehub-resource-sync Date: Mon Jul 13 13:32:23 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.bandit b/.bandit new file mode 100644 index 0000000..cc9255a --- /dev/null +++ b/.bandit @@ -0,0 +1,7 @@ +[bandit] +# B101 : assert_used +# B102 : exec_used +# B404 : import_subprocess +# B406 : import_xml_sax +skips: B101,B102,B404,B406 +exclude: **/tests/**,tests diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..e2bc834 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,38 @@ +[run] +branch = true + +source = + cvat/apps/ + cvat-sdk/ + cvat-cli/ + utils/dataset_manifest + +omit = + cvat/settings/* + */tests/* + */test_* + */_test_* + */migrations/* + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if\s+[\w\.()]+\.isEnabledFor\(log\.DEBUG\): + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +# don't fail on the code that can be found +ignore_errors = true + +skip_empty = true diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2bd68c0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +/.git +/.env +/.vscode +/.github +/.husky +/share +/data +/media +/keys +**/node_modules +/static +/serverless +/site +/helm-chart +/dev +/changelog.d +/cvat-cli +/profiles diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..783dea0 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# EditorConfig helps developers define and maintain consistent +# coding styles between different editors and IDEs +# editorconfig.org + +root = true + +[*] + +# Change these settings to your own preference +indent_style = space +indent_size = 4 + +# We recommend you to keep these unchanged +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{yml,yaml}] + +indent_size = 2 diff --git a/.eslintrc.cjs b/.eslintrc.cjs new file mode 100644 index 0000000..f884f9c --- /dev/null +++ b/.eslintrc.cjs @@ -0,0 +1,88 @@ +// Copyright (C) 2018-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +module.exports = { + root: true, + env: { + node: true, + browser: true, + es2020: true, + }, + parserOptions: { + sourceType: 'module', + parser: '@typescript-eslint/parser', + }, + ignorePatterns: [ + '.eslintrc.cjs', + 'lint-staged.config.js', + 'site/**', + 'webpack.config.cjs', + ], + plugins: ['@typescript-eslint', '@stylistic', 'security', 'no-unsanitized', 'import'], + extends: [ + 'eslint:recommended', 'plugin:security/recommended', 'plugin:no-unsanitized/DOM', + 'airbnb-base', 'plugin:import/errors', 'plugin:import/warnings', + 'plugin:import/typescript', 'plugin:@typescript-eslint/recommended', 'airbnb-typescript/base', + ], + rules: { + // 'header/header': [2, 'line', [{ + // pattern: ' {1}Copyright \\(C\\) (?:20\\d{2}-)?2022 Intel Corporation', + // template: ' Copyright (C) 2022 Intel Corporation' + // }, '', ' SPDX-License-Identifier: MIT']], + 'no-plusplus': 0, + 'no-continue': 0, + 'no-console': 0, + 'no-restricted-syntax': [0, { selector: 'ForOfStatement' }], + 'no-await-in-loop': 0, + '@stylistic/indent': ['error', 4, { 'SwitchCase': 1 }], + 'max-len': ['error', { code: 120, ignoreStrings: true }], + 'func-names': 0, + 'valid-typeof': 0, + 'quotes': ['error', 'single', { "avoidEscape": true }], + 'lines-between-class-members': 'off', + '@stylistic/lines-between-class-members': 0, + '@typescript-eslint/lines-between-class-members': 'off', + 'class-methods-use-this': 0, + 'no-underscore-dangle': ['error', { allowAfterThis: true }], + 'max-classes-per-file': 0, + 'operator-linebreak': ['error', 'after'], + 'newline-per-chained-call': 0, + 'global-require': 0, + 'arrow-parens': ['error', 'always'], + 'security/detect-object-injection': 0, // the rule is relevant for user input data on the node.js environment + 'import/order': ['error', {'groups': ['builtin', 'external', 'internal']}], + 'import/no-unresolved': 'off', + 'import/prefer-default-export': 0, // works incorrect with interfaces + 'no-useless-assignment': 'off', + 'preserve-caught-error': 'off', + + 'react/jsx-indent-props': 0, // new rule, breaks current styling + 'react/jsx-indent': 0, // new rule, conflicts with eslint@typescript-eslint/indent eslint@indent, breaks current styling + 'function-paren-newline': 0, // new rule, breaks current styling + '@typescript-eslint/default-param-last': 0, // does not really work with redux reducers + '@typescript-eslint/ban-ts-comment': 0, + '@typescript-eslint/no-explicit-any': 0, + '@typescript-eslint/explicit-function-return-type': ['warn', { allowExpressions: true }], + '@typescript-eslint/explicit-module-boundary-types': 'off', + '@typescript-eslint/no-empty-object-type': [ + 'error', + { + allowInterfaces: 'always', + allowObjectTypes: 'never', + }, + ], + '@typescript-eslint/no-unsafe-function-type': 'error', + '@typescript-eslint/no-restricted-types': [ + 'error', + { + types: { + object: { + message: 'Use a more specific object shape, Record, or unknown instead of object.', + }, + }, + }, + ], + }, +}; diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7c0b591 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,29 @@ +* text=auto whitespace=trailing-space,space-before-tab,-indent-with-non-tab,tab-in-indent,tabwidth=4 + +.git* text export-ignore + +*.txt text +*.htm text +*.html text +*.js text +*.py text +*.css text +*.md text +*.yml text +Dockerfile text +LICENSE text +*.conf text +*.mimetypes text +*.sh text eol=lf + +*.avi binary +*.bmp binary +*.exr binary +*.ico binary +*.jpeg binary +*.jpg binary +*.png binary +*.gif binary +*.ttf binary +*.pdf binary + diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..c6e5cfb --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,59 @@ +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, they will +# be requested for review when someone opens a pull request. +* @nmanovic + +# Order is important; the last matching pattern takes the most +# precedence. When someone opens a pull request that only +# modifies components below, only the list of owners and not +# the global owner(s) will be requested for a review. + +# Component: Server +/cvat/ @SpecLad +/cvat/apps/dataset_manager/ @zhiltsov-max +/cvat/apps/consensus/ @zhiltsov-max +/cvat/apps/quality_control/ @zhiltsov-max + +# Component: CVAT SDK/CLI +/cvat-sdk/ @SpecLad +/cvat/schema.yml @SpecLad +/cvat-cli/ @SpecLad + +# Component: Documentation +/site/ @bsekachev +/CHANGELOG.md @bsekachev +/README.md @bsekachev + +# Component: CVAT UI +/cvat-ui/ @bsekachev +/cvat-data/ @bsekachev +/cvat-canvas/ @bsekachev +/cvat-canvas3d/ @bsekachev +/cvat-core/ @bsekachev + +# Advanced components (e.g. analytics) +/components/ @azhavoro + +# Component: Tests +/tests/ @bsekachev +/tests/python/ @zhiltsov-max + +# Component: Serverless functions +/serverless/ @bsekachev + +# Infrastructure +Dockerfile* @azhavoro +docker-compose* @azhavoro +.* @azhavoro +*.js @bsekachev +*.conf @azhavoro +*.sh @azhavoro +/supervisord/ @azhavoro +/helm-chart/ @azhavoro +/utils/ @SpecLad +/.github/ @SpecLad +/dev/ @SpecLad +.regal/ @SpecLad +/LICENSE @nmanovic diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml new file mode 100644 index 0000000..66797a8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -0,0 +1,65 @@ +name: "\U0001F41E Bug Report" +description: Create a bug report to help us fix it +labels: ["bug"] +body: +- type: checkboxes + attributes: + label: Actions before raising this issue + options: + - label: I searched the existing issues and did not find anything similar. + required: true + - label: I read/searched [the docs](https://docs.cvat.ai/docs/) + required: true + +- type: textarea + attributes: + label: Steps to Reproduce + description: Provide a link to a live example or an unambiguous set of steps to reproduce this bug. Include code to reproduce, if relevant. + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + validations: + required: false +- type: textarea + attributes: + label: Expected Behavior + description: A concise description of what you expected to happen. + validations: + required: false +- type: textarea + attributes: + label: Possible Solution + description: | + Not obligatory, but suggest a fix/reason for the bug, or ideas on how to implement the addition or change + validations: + required: false +- type: textarea + attributes: + label: Context + description: | + How has this issue affected you? What are you trying to accomplish? + Providing context helps us come up with a solution that is most useful in the real world! + + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: false +- type: textarea + attributes: + label: Environment + description: | + Include relevant details about the environment you experienced + placeholder: | + - Git hash commit (`git log -1`): + - Docker version `docker version` (e.g. Docker 17.0.05): + - Are you using Docker Swarm or Kubernetes? + - Operating System and version (e.g. Linux, Windows, MacOS): + - Code example or link to GitHub repo or gist to reproduce problem: + - Other diagnostic information / logs: +
+ Logs from `cvat` container +
+ render: Markdown + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 0000000..b06919d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,41 @@ +name: "\U0001F680 Feature Request" +description: Suggest an idea for this project +labels: ["enhancement"] +body: +- type: checkboxes + attributes: + label: Actions before raising this issue + options: + - label: I searched the existing issues and did not find anything similar. + required: true + - label: I read/searched [the docs](https://docs.cvat.ai/docs/) + required: true +- type: textarea + attributes: + label: Is your feature request related to a problem? Please describe. + description: A clear and concise description of what the problem is. + placeholder: I'm always frustrated when I have to press this small button. + validations: + required: false +- type: textarea + attributes: + label: Describe the solution you'd like + description: A clear and concise description of what you want to happen. + placeholder: Make this button bigger + validations: + required: false +- type: textarea + attributes: + label: Describe alternatives you've considered + description: A clear and concise description of any alternative solutions or features you've considered. + placeholder: I wanted to buy bigger display, but it would be too expensive. + validations: + required: false +- type: textarea + attributes: + label: Additional context + description: | + Add any other context or screenshots about the feature request here. + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..ae4841f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,34 @@ + + + + +### Motivation and context + + +### How has this been tested? + + +### Checklist + +- [ ] I submit my changes into the `develop` branch +- [ ] I have created a changelog fragment +- [ ] I have updated the documentation accordingly +- [ ] I have added tests to cover my changes +- [ ] I have linked related issues (see [GitHub docs]( + https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) + +### License + +- [ ] I submit _my code changes_ under the same [MIT License]( + https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. + Feel free to contact the maintainers if that's a concern. diff --git a/.github/actions/setup-yarn/action.yml b/.github/actions/setup-yarn/action.yml new file mode 100644 index 0000000..fe60cbf --- /dev/null +++ b/.github/actions/setup-yarn/action.yml @@ -0,0 +1,62 @@ +name: Setup Yarn +description: Set up Node.js, enable Yarn via Corepack, cache dependencies, and install packages + +inputs: + node-version: + description: Node.js version to install + required: false + default: 22.x + node-modules-path: + description: Paths to cache for node_modules + required: false + default: node_modules + install-command: + description: Command used to install dependencies + required: false + default: yarn install --immutable + +runs: + using: composite + steps: + - uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + - name: Enable corepack + shell: bash + run: corepack enable yarn + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + shell: bash + run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT + + - name: Cache yarn cache + uses: actions/cache@v4 + id: cache-yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Get yarn version + id: yarn-version + shell: bash + run: echo "version=$(node --version)" >> $GITHUB_OUTPUT + + - name: Cache node_modules + uses: actions/cache@v4 + id: cache-node-modules + with: + path: ${{ inputs.node-modules-path }} + key: ${{ runner.os }}-${{ steps.yarn-version.outputs.version }}-nodemodules-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-${{ steps.yarn-version.outputs.version }}-nodemodules- + + - name: Install dependencies + if: | + steps.cache-yarn-cache.outputs.cache-hit != 'true' || + steps.cache-node-modules.outputs.cache-hit != 'true' + shell: bash + run: ${{ inputs.install-command }} diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..aa7fd1c --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,33 @@ +comment: + layout: "header, diff, components" + +component_management: + individual_components: + - component_id: cvat-ui + name: cvat-ui + paths: + - cvat-canvas/** + - cvat-canvas3d/** + - cvat-core/** + - cvat-data/** + - cvat-ui/** + - component_id: cvat-server + name: cvat-server + paths: + - cvat/** + - cvat-cli/** + - cvat-sdk/** + - utils/** + +codecov: + require_ci_to_pass: yes + notify: + wait_for_ci: yes + +coverage: + status: + patch: false + project: + default: + target: auto + threshold: 5% \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c86d993 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,35 @@ +version: 2 + +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + + # Python dependencies in cvat/requirements are managed by pip-compile-multi. + # Dependabot should update source *.in files; generated *.txt files are + # regenerated by a separate workflow. + - package-ecosystem: "pip" + directory: "/utils/dataset_manifest" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + exclude-paths: + - "*.txt" + + # Dependabot resolves the full CVAT requirements graph for each candidate + # update, while generated lock files are refreshed by the workflow below. + - package-ecosystem: "pip" + directory: "/cvat/requirements" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + exclude-paths: + - "*.txt" + - "all.in" diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml new file mode 100644 index 0000000..ed9db9f --- /dev/null +++ b/.github/workflows/cache.yml @@ -0,0 +1,63 @@ +name: Cache +on: + push: + branches: + - 'develop' + +jobs: + get-sha: + uses: ./.github/workflows/search-cache.yml + + Caching_CVAT: + needs: get-sha + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/cache@v6 + id: server-cache-action + with: + path: /tmp/cvat_cache_server + key: ${{ runner.os }}-build-server-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-build-server-${{ needs.get-sha.outputs.sha }} + ${{ runner.os }}-build-server- + + - uses: actions/cache@v6 + id: ui-cache-action + with: + path: /tmp/cvat_cache_ui + key: ${{ runner.os }}-build-ui-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-build-ui-${{ needs.get-sha.outputs.sha }} + ${{ runner.os }}-build-ui- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Caching CVAT Server + uses: docker/build-push-action@v7 + with: + context: . + file: ./Dockerfile + cache-from: type=local,src=/tmp/cvat_cache_server + cache-to: type=local,dest=/tmp/cvat_cache_server-new + + - name: Caching CVAT UI + uses: docker/build-push-action@v7 + with: + context: . + file: ./Dockerfile.ui + cache-from: type=local,src=/tmp/cvat_cache_ui + cache-to: type=local,dest=/tmp/cvat_cache_ui-new + + - name: Moving cache + run: | + rm -rf /tmp/cvat_cache_server + mv /tmp/cvat_cache_server-new /tmp/cvat_cache_server + + rm -rf /tmp/cvat_cache_ui + mv /tmp/cvat_cache_ui-new /tmp/cvat_cache_ui diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..b9a31e6 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "develop", master, release-* ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "develop" ] + schedule: + - cron: '27 19 * * 4' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v4 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/comment.yml b/.github/workflows/comment.yml new file mode 100644 index 0000000..1a362d7 --- /dev/null +++ b/.github/workflows/comment.yml @@ -0,0 +1,89 @@ +name: Comment +on: + issue_comment: + types: [created] + +env: + WORKFLOW_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + +jobs: + verify_author: + if: contains(github.event.issue.html_url, '/pull') && + contains(github.event.comment.body, '/check') + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.get-ref.outputs.ref }} + cid: ${{ steps.send-status.outputs.cid }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Check author of comment + id: check-author + run: | + PERM=$(gh api repos/${{ github.repository }}/collaborators/${{ github.event.comment.user.login }}/permission | jq -r '.permission') + if [[ $PERM == "write" || $PERM == "maintain" || $PERM == "admin" ]]; + then + ALLOW="true" + fi + echo "allow=${ALLOW}" >> $GITHUB_OUTPUT + + - name: Verify that author of comment is collaborator + if: steps.check-author.outputs.allow == '' + uses: actions/github-script@v9 + with: + script: | + core.setFailed('User that send comment with /check command is not collaborator') + + - name: Get branch name + id: get-ref + run: | + SHA=$(gh api /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} | jq -r '.head.sha') + echo "ref=${SHA}" >> $GITHUB_OUTPUT + + - name: Send comment. Test are executing + id: send-status + run: | + BODY=":hourglass: Tests are executing, see more information [here](${{ env.WORKFLOW_RUN_URL }})" + BODY=$BODY"\n :warning: Cancel [this](${{ env.WORKFLOW_RUN_URL }}) workflow manually first, if you want to restart full check" + BODY=$(echo -e $BODY) + + COMMENT_ID=$(gh api --method POST \ + /repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments \ + -f body="${BODY}" | jq '.id') + + echo "cid=${COMMENT_ID}" >> $GITHUB_OUTPUT + + run-full: + needs: verify_author + uses: ./.github/workflows/full.yml + with: + ref: ${{ needs.verify_author.outputs.ref }} + + send_status: + runs-on: ubuntu-latest + needs: [run-full, verify_author] + if: needs.run-full.result != 'skipped' && always() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Send status in comments + run: | + BODY="" + + if [[ "${{ needs.run-full.result }}" == "failure" ]] + then + BODY=":x: Some checks failed" + elif [[ "${{ needs.run-full.result }}" == "success" ]] + then + BODY=":heavy_check_mark: All checks completed successfully" + elif [[ "${{ needs.run-full.result }}" == "cancelled" ]] + then + BODY=":no_entry_sign: Workflows has been canceled" + fi + + BODY=$BODY"\n :page_facing_up: See logs [here](${WORKFLOW_RUN_URL})" + BODY=$(echo -e $BODY) + + gh api --method PATCH \ + /repos/${{ github.repository }}/issues/comments/${{ needs.verify_author.outputs.cid }} \ + -f body="${BODY}" \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..d731af2 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,65 @@ +name: Docs +on: + push: + branches: + - 'master' + - 'develop' + pull_request: + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate_github_pages: + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + fetch-depth: 0 + + - name: Generate CVAT SDK + run: | + pip3 install --user -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + - name: Setup Hugo + run: | + wget https://github.com/gohugoio/hugo/releases/download/v0.110.0/hugo_extended_0.110.0_Linux-64bit.tar.gz + (mkdir hugo_extended_0.110.0_Linux-64bit && tar -xf hugo_extended_0.110.0_Linux-64bit.tar.gz -C hugo_extended_0.110.0_Linux-64bit) + + mkdir hugo + cp hugo_extended_0.110.0_Linux-64bit/hugo hugo/hugo-0.110 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: '22.x' + + - name: Install npm packages + working-directory: ./site + run: | + npm ci + + - name: Build docs + run: | + pip install -r site/requirements.txt + python site/process_sdk_docs.py + PATH="$PWD/hugo:$PATH" python site/build_docs.py + env: + BASE_URL: https://docs.cvat.ai + HUGO_ENV: production + GOOGLE_TAG_ID: G-GVSBK1DNK5 + + - name: Deploy + if: github.ref == 'refs/heads/develop' + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./public + force_orphan: true + cname: docs.cvat.ai diff --git a/.github/workflows/finalize-release.yml b/.github/workflows/finalize-release.yml new file mode 100644 index 0000000..0fcc0ca --- /dev/null +++ b/.github/workflows/finalize-release.yml @@ -0,0 +1,109 @@ +name: Finalize release +on: + workflow_dispatch: +jobs: + main: + runs-on: ubuntu-latest + steps: + - name: Discover pending release + id: discover + env: + GH_TOKEN: "${{ github.token }}" + run: | + gh --repo="${{ github.repository }}" \ + pr list --base master --state open --json number,headRefName \ + --jq 'map(select(.headRefName | startswith("release-")))' \ + > /tmp/release-prs.json + + if jq -e 'length < 1' /tmp/release-prs.json > /dev/null; then + echo "No open release pull requests found." + exit 1 + elif jq -e 'length > 1' /tmp/release-prs.json > /dev/null; then + echo "Multiple open release pull requests found:" + jq -r '.[] | "https://github.com/${{ github.repository }}/pull/\(.number)"' /tmp/release-prs.json + exit 1 + fi + + jq -r '.[] | "prNumber=\(.number)", "version=\(.headRefName | ltrimstr("release-"))"' \ + /tmp/release-prs.json >> "$GITHUB_OUTPUT" + + # When you use the default github.token to make changes in the repository, + # it does not trigger further GitHub pipelines. We want to trigger CI for + # the pull request and artifact building for the release, so we have to use + # an app token. + - name: Generate authentication token + id: gen-token + uses: actions/create-github-app-token@v3 + with: + client-id: "${{ secrets.CVAT_BOT_CLIENT_ID }}" + private-key: "${{ secrets.CVAT_BOT_PRIVATE_KEY }}" + + - name: Install dependencies + run: + sudo apt-get install -y pandoc + + - uses: actions/checkout@v7 + with: + ref: "release-${{ steps.discover.outputs.version }}" + + - name: Verify that the release is new + env: + NEW_VERSION: "${{ steps.discover.outputs.version }}" + run: | + if git ls-remote --exit-code origin "refs/tags/v$NEW_VERSION" > /dev/null; then + echo "Release v$NEW_VERSION already exists" + exit 1 + fi + + # Do post-release tasks before publishing the release. If anything goes wrong, + # the dev-release-* branch can be deleted, and the whole process restarted again; + # whereas we can't unmerge the release PR. + + - name: Create post-release branch + run: + git checkout -b "dev-release-${{ steps.discover.outputs.version }}" + + - name: Bump version + run: + ./dev/update_version.py --patch + + - name: Commit post-release changes + run: | + git -c user.name='cvat-bot[bot]' -c user.email='147643061+cvat-bot[bot]@users.noreply.github.com' \ + commit -a -m "Update ${{ github.ref_name }} after v${{ steps.discover.outputs.version }}" + + - name: Push post-release branch + run: + git push -u origin "dev-release-${{ steps.discover.outputs.version }}" + + - name: Create post-release pull request + env: + GH_TOKEN: "${{ steps.gen-token.outputs.token }}" + run: | + gh pr create \ + --base="${{ github.ref_name }}" \ + --title="Update ${{ github.ref_name }} after v${{ steps.discover.outputs.version }}" \ + --body="" + + # Now publish the release. + + - name: Merge release pull request + env: + GH_TOKEN: "${{ steps.gen-token.outputs.token }}" + run: + gh pr merge --merge "${{ steps.discover.outputs.prNumber }}" --delete-branch + + - name: Create release + env: + GH_TOKEN: "${{ steps.gen-token.outputs.token }}" + NEW_VERSION: "${{ steps.discover.outputs.version }}" + run: | + # We could grab the release notes from the PR description, but it could + # be outdated if any changes were made on the release branch. So instead, + # just re-extract them from the changelog again. + + ./dev/gh_release_notes.sh \ + | gh release create "v$NEW_VERSION" \ + --target=master \ + --title="v$NEW_VERSION" \ + --notes-file=- diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml new file mode 100644 index 0000000..53ff09a --- /dev/null +++ b/.github/workflows/full.yml @@ -0,0 +1,354 @@ +name: Full +on: + workflow_call: + inputs: + ref: + type: string + required: true + workflow_dispatch: + inputs: + ref: + type: string + required: true + +env: + WORKFLOW_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + CYPRESS_VERIFY_TIMEOUT: 180000 # https://docs.cypress.io/guides/guides/command-line#cypress-verify + CVAT_VERSION: "local" + +jobs: + search_cache: + uses: ./.github/workflows/search-cache.yml + + build: + needs: search_cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: CVAT server. Getting cache from the default branch + uses: actions/cache@v6 + with: + path: /tmp/cvat_cache_server + key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }} + + - name: CVAT UI. Getting cache from the default branch + uses: actions/cache@v6 + with: + path: /tmp/cvat_cache_ui + key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Create artifact directories + run: | + mkdir /tmp/cvat_server + mkdir /tmp/cvat_ui + mkdir /tmp/cvat_sdk + + - name: CVAT server. Build and push + uses: docker/build-push-action@v7 + with: + cache-from: type=local,src=/tmp/cvat_cache_server + context: . + file: Dockerfile + tags: cvat/server:${{ env.CVAT_VERSION }} + outputs: type=docker,dest=/tmp/cvat_server/image.tar + + - name: CVAT UI. Build and push + uses: docker/build-push-action@v7 + with: + cache-from: type=local,src=/tmp/cvat_cache_ui + context: . + file: Dockerfile.ui + tags: cvat/ui:${{ env.CVAT_VERSION }} + outputs: type=docker,dest=/tmp/cvat_ui/image.tar + + - name: CVAT SDK. Build + run: | + pip3 install --user -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + cp -r cvat-sdk/* /tmp/cvat_sdk/ + + - name: Upload CVAT server artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_server + path: /tmp/cvat_server/image.tar + + - name: Upload CVAT UI artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_ui + path: /tmp/cvat_ui/image.tar + + - name: Upload CVAT SDK artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_sdk + path: /tmp/cvat_sdk/ + + rest_api_testing: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI images + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Download CVAT SDK package + uses: actions/download-artifact@v8 + with: + name: cvat_sdk + path: /tmp/cvat_sdk/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a + + - name: Verify API schema + id: verify_schema + run: | + docker run --rm cvat/server:${CVAT_VERSION} bash \ + -c 'python manage.py spectacular' > cvat/schema-expected.yml + + if ! git diff --no-index cvat/schema.yml cvat/schema-expected.yml; then + echo + echo 'API schema has changed! Please update cvat/schema.yml:' + echo + echo ' docker run --rm cvat/server:dev bash \' + echo " -c 'python manage.py spectacular' > cvat/schema.yml" + exit 1 + fi + + - name: Verify migrations + run: | + docker run --rm cvat/server:${CVAT_VERSION} bash \ + -c 'python manage.py makemigrations --check' + + - name: Generate SDK + run: | + pip3 install -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + - name: Install SDK + run: | + pip3 install -r ./tests/python/requirements.txt \ + -e './cvat-sdk[masks,pytorch]' -e ./cvat-cli \ + --extra-index-url https://download.pytorch.org/whl/cpu + + - name: Running REST API and SDK tests + id: run_tests + run: | + pytest tests/python/ + CVAT_ALLOW_STATIC_CACHE="true" pytest -k "TestTaskData" tests/python + + - name: Creating a log file from cvat containers + if: failure() && steps.run_tests.conclusion == 'failure' + env: + LOGS_DIR: "${{ github.workspace }}/rest_api_testing" + run: | + mkdir $LOGS_DIR + docker logs test_cvat_server_1 > $LOGS_DIR/cvat_server.log + docker logs test_cvat_worker_export_1 > $LOGS_DIR/cvat_worker_export.log + docker logs test_cvat_worker_import_1 > $LOGS_DIR/cvat_worker_import.log + docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() && steps.run_tests.conclusion == 'failure' + uses: actions/upload-artifact@v7 + with: + name: rest_api_container_logs + path: "${{ github.workspace }}/rest_api_testing" + + unit_testing: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Load Docker server image + run: | + docker load --input /tmp/cvat_server/image.tar + docker image ls -a + + - name: Running OPA tests + run: | + python cvat/apps/iam/rules/tests/generate_tests.py + + docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \ + cvat_opa test cvat/apps/*/rules + + - name: Running unit tests + env: + HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} + CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" + run: | + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db + + max_tries=12 + while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done + + docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml \ + run --env PYTHONDEVMODE=1 cvat_ci /bin/bash \ + -c 'python manage.py test cvat/apps -v 2' + + - name: Creating a log file from cvat containers + if: failure() + env: + LOGS_DIR: "${{ github.workspace }}/unit_testing" + run: | + mkdir $LOGS_DIR + docker logs cvat_server > $LOGS_DIR/cvat_server.log + docker logs cvat_opa 2> $LOGS_DIR/cvat_opa.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: unit_tests_container_logs + path: "${{ github.workspace }}/unit_testing" + + e2e_testing: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_tasks4', + 'actions_objects', 'actions_objects2', 'actions_users', + 'actions_projects_models', 'canvas3d_functionality', + 'issues_prs', 'issues_prs2', 'features', 'features2', + 'audio_workspace'] + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - uses: actions/setup-node@v6 + with: + node-version: '22.x' + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI image + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a + + - name: Run CVAT instance + run: | + docker compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + -f components/serverless/docker-compose.serverless.yml \ + -f tests/docker-compose.minio.yml \ + -f tests/docker-compose.file_share.yml up -d + + - name: Waiting for server + env: + API_ABOUT_PAGE: "localhost:8080/api/server/about" + run: | + max_tries=60 + status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE}) + while [[ $status_code != "200" && max_tries -gt 0 ]] + do + echo Number of attempts left: $max_tries + echo Status code of response: $status_code + + sleep 5 + status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE}) + (( max_tries-- )) + done + + - name: Run E2E tests + env: + DJANGO_SU_NAME: 'admin' + DJANGO_SU_EMAIL: 'admin@localhost.company' + DJANGO_SU_PASSWORD: '12qwaszx' + run: | + docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell" + cd ./tests + corepack enable yarn + yarn --immutable + + if [[ ${{ matrix.specs }} == canvas3d_* ]]; then + npx cypress run \ + --headed \ + --browser chrome \ + --env coverage=false \ + --config-file cypress_canvas3d.config.js \ + --spec 'cypress/e2e/setup/setup_canvas3d.js,cypress/e2e/canvas3d_*/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + elif [[ ${{ matrix.specs }} == audio_* ]]; then + npx cypress run \ + --headed \ + --browser chrome \ + --env coverage=false \ + --config-file cypress_audio.config.js \ + --spec 'cypress/e2e/setup/setup_audio.js,cypress/e2e/audio_workspace/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + else + npx cypress run \ + --browser chrome \ + --env coverage=false \ + --spec 'cypress/e2e/setup/setup.js,cypress/e2e/setup/setup_project.js,cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + fi + + - name: Creating a log file from "cvat" container logs + if: failure() + run: | + docker logs cvat_server > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: e2e_container_logs_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log + + - name: Uploading cypress screenshots as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cypress_screenshots_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cypress/screenshots diff --git a/.github/workflows/generate-allure-report.yml b/.github/workflows/generate-allure-report.yml new file mode 100644 index 0000000..d2b0387 --- /dev/null +++ b/.github/workflows/generate-allure-report.yml @@ -0,0 +1,68 @@ +name: Generate Allure Report + +on: + workflow_call: + + secrets: + AWS_ALLURE_REPORTS_ROLE: + required: true +env: + S3_BUCKET: cvat-allure-reports + +jobs: + generate_report: + name: Generate Allure Report and Upload to S3 + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Install Allure CLI (.deb) + run: | + sudo apt-get update + sudo apt-get install -y default-jre-headless + wget https://github.com/allure-framework/allure2/releases/download/2.34.0/allure_2.34.0-1_all.deb + sudo dpkg -i allure_2.34.0-1_all.deb + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.AWS_ALLURE_REPORTS_ROLE }} + aws-region: eu-west-1 + + - name: Set a timestamp + id: timestampid + run: | + echo "timestamp=$(date --utc +%Y%m%d_%H%M%SZ)" >> "$GITHUB_OUTPUT" + + - name: Download Allure results from artifacts + uses: actions/download-artifact@v8 + with: + pattern: allure-results* + merge-multiple: true + path: merged-allure-results + + - name: Download previous history from S3 + run: aws s3 cp s3://${{ env.S3_BUCKET }}/history/ ./merged-allure-results/history --recursive || echo "No history found" + + - name: Generate Allure Report + run: allure generate ./merged-allure-results --clean -o allure-report + + - name: Backup updated history to S3 + run: aws s3 cp ./allure-report/history s3://${{ env.S3_BUCKET }}/history/ --recursive + + - name: Deploy report to S3 (timestamped folder) + run: aws s3 cp ./allure-report s3://${{ env.S3_BUCKET }}/report/${{ steps.timestampid.outputs.timestamp }}/ --recursive + + - name: Update latest report alias + run: | + aws s3 rm s3://${{ env.S3_BUCKET }}/report/latest/ --recursive || true + aws s3 cp ./allure-report s3://${{ env.S3_BUCKET }}/report/latest/ --recursive + + - name: Write report URL to summary + run: | + echo "### 🧪 [Allure Report](http://${{ env.S3_BUCKET }}.s3-website.eu-west-1.amazonaws.com/report/${{ steps.timestampid.outputs.timestamp }}/index.html)" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml new file mode 100644 index 0000000..e0510f3 --- /dev/null +++ b/.github/workflows/linters.yml @@ -0,0 +1,189 @@ +name: linters +on: pull_request +jobs: + bandit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Run checks + run: | + pipx install $(grep "^bandit" ./dev/requirements.txt) + + echo "Bandit version: "$(bandit --version | head -1) + bandit -a file --ini .bandit --recursive . + + black: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Run checks + run: | + pipx install $(grep "^black" ./dev/requirements.txt) + + echo "Black version: $(black --version)" + + black --check --diff . + + hadolint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Run checks + env: + HADOLINT: '${{ github.workspace }}/hadolint' + HADOLINT_VER: 2.12.0 + VERIFICATION_LEVEL: error + run: | + curl -sLo "$HADOLINT" "https://github.com/hadolint/hadolint/releases/download/v$HADOLINT_VER/hadolint-Linux-x86_64" + chmod +x "$HADOLINT" + echo "hadolint version: $("$HADOLINT" --version)" + git ls-files -z 'Dockerfile*' '*/Dockerfile*' | xargs -0 "$HADOLINT" -t "$VERIFICATION_LEVEL" + + isort: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Run checks + run: | + pipx install $(grep "^isort" ./dev/requirements.txt) + + echo "isort version: $(isort --version-number)" + + isort --check --diff --resolve-all-configs . + + pylint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - id: setup-pipx + shell: bash + run: | + # custom cached dir + cache_home_dir="$HOME/.local/pipx" + cache_bin_dir="$HOME/.local/pipx_bin" + + # add cached dir to PATH + echo "$cache_bin_dir" >> $GITHUB_PATH + + echo "cache_home_dir=$cache_home_dir" >> $GITHUB_OUTPUT + echo "cache_bin_dir=$cache_bin_dir" >> $GITHUB_OUTPUT + echo "version=$(pipx --version)" >> $GITHUB_OUTPUT + echo "python_version=$(python3 --version)" >> $GITHUB_OUTPUT + + - id: cache-pipx + uses: actions/cache@v6 + with: + path: | + ${{ steps.setup-pipx.outputs.cache_home_dir }} + ${{ steps.setup-pipx.outputs.cache_bin_dir }} + # cache based on pipx version, python version, and both requirements files + # nb! we need to keep python version in the key because packages might be not compatible between python versions + key: pipx-pylint-${{ steps.setup-pipx.outputs.version}}-${{ steps.setup-pipx.outputs.python_version}}-${{ hashFiles('./dev/requirements.txt', './cvat/requirements/base.txt') }} + + - name: Install pylint + if: steps.cache-pipx.outputs.cache-hit != 'true' + shell: bash + env: + # nb! install in our custom cached dir, using /opt will not work because of permission issues + PIPX_HOME: ${{ steps.setup-pipx.outputs.cache_home_dir }} + PIPX_BIN_DIR: ${{ steps.setup-pipx.outputs.cache_bin_dir }} + run: | + pipx install $(grep "^pylint==" ./dev/requirements.txt) + + pipx inject pylint \ + $(grep "^pylint-.\+==" ./dev/requirements.txt) \ + $(grep "^django==" ./cvat/requirements/base.txt) + + - name: Run checks + run: | + echo "Pylint version: "$(pylint --version | head -1) + pylint -j0 . + + regal: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Setup Regal + uses: open-policy-agent/setup-regal@761188c3b435761fa254beca508a44875619648f # v2.0.0 + with: + version: v0.21.3 + + - run: regal lint --format=github cvat/apps/*/rules + + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Run checks + run: | + pipx install $(grep "^typos==" ./dev/requirements.txt) + + echo "Typos version: $(typos --version )" + typos + + eslint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: ./.github/actions/setup-yarn + with: + node-version: 22.x + node-modules-path: | + node_modules + tests/node_modules + install-command: | + yarn install --immutable + (cd tests && yarn install --immutable) + + - name: Run checks + run: | + echo "ESLint version: "$(yarn run eslint --version) + yarn run eslint . + + typescript: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: ./.github/actions/setup-yarn + with: + node-version: 22.x + + - name: Run TypeScript checks for cvat frontend code + run: yarn run type-check + + remark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: ./.github/actions/setup-yarn + with: + node-version: 22.x + + - name: Run checks + run: | + echo "Remark version: "`npx remark --version` + npx remark --quiet --frail -i .remarkignore . + + stylelint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: ./.github/actions/setup-yarn + with: + node-version: 22.x + + - name: Run checks + run: | + echo "StyleLint version: "$(yarn run stylelint --version) + yarn run stylelint '**/*.css' '**/*.scss' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..5ec13b3 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,625 @@ +name: CI +on: + push: + branches: + - 'master' + - 'develop' + pull_request: + types: [ready_for_review, opened, synchronize, reopened] + paths-ignore: + - 'site/**' + - '**/*.md' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CYPRESS_VERIFY_TIMEOUT: 180000 # https://docs.cypress.io/guides/guides/command-line#cypress-verify + CVAT_VERSION: "local" + +jobs: + search_cache: + if: | + github.event.pull_request.draft == false && + !startsWith(github.event.pull_request.title, '[WIP]') && + !startsWith(github.event.pull_request.title, '[Dependent]') + uses: ./.github/workflows/search-cache.yml + + build: + needs: search_cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Verify version consistency + run: ./dev/update_version.py --verify-current + + - name: Check changelog fragments + run: ./dev/check_changelog_fragments.py + + - name: CVAT server. Getting cache from the default branch + uses: actions/cache@v6 + with: + path: /tmp/cvat_cache_server + key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }} + + - name: CVAT UI. Getting cache from the default branch + uses: actions/cache@v6 + with: + path: /tmp/cvat_cache_ui + key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Create artifact directories + run: | + mkdir /tmp/cvat_server + mkdir /tmp/cvat_ui + mkdir /tmp/cvat_sdk + + - name: CVAT server. Build and push + uses: docker/build-push-action@v7 + with: + build-args: | + "COVERAGE_PROCESS_START=.coveragerc" + cache-from: type=local,src=/tmp/cvat_cache_server + context: . + file: Dockerfile + tags: cvat/server:${{ env.CVAT_VERSION }} + outputs: type=docker,dest=/tmp/cvat_server/image.tar + + - name: Instrumentation of the code then rebuilding the CVAT UI + run: | + corepack enable yarn + yarn --immutable + yarn run coverage + + - name: CVAT UI. Build and push + uses: docker/build-push-action@v7 + with: + cache-from: type=local,src=/tmp/cvat_cache_ui + context: . + file: Dockerfile.ui + tags: cvat/ui:${{ env.CVAT_VERSION }} + outputs: type=docker,dest=/tmp/cvat_ui/image.tar + + - name: CVAT SDK. Build + run: | + pip3 install --user -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + cp -r cvat-sdk/* /tmp/cvat_sdk/ + + - name: Verify API schema + id: verify_schema + run: | + docker load --input /tmp/cvat_server/image.tar + docker run --rm "cvat/server:${CVAT_VERSION}" bash \ + -c 'python manage.py spectacular' > cvat/schema-expected.yml + + if ! git diff --no-index cvat/schema.yml cvat/schema-expected.yml; then + echo + echo 'API schema has changed! Please update cvat/schema.yml:' + echo + echo ' docker run --rm cvat/server:dev bash \' + echo " -c 'python manage.py spectacular' > cvat/schema.yml" + exit 1 + fi + + - name: Verify migrations + run: | + docker run --rm "cvat/server:${CVAT_VERSION}" bash \ + -c 'python manage.py makemigrations --check' + + - name: Verify redis migration filenames + run: | + docker run --rm "cvat/server:${CVAT_VERSION}" bash \ + -c 'python manage.py checkredismigrations' + + - name: Upload CVAT server artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_server + path: /tmp/cvat_server/image.tar + + - name: Upload CVAT UI artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_ui + path: /tmp/cvat_ui/image.tar + + - name: Upload CVAT SDK artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_sdk + path: /tmp/cvat_sdk/ + + rest_api_testing: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - pattern: "test_task_data" + allow_cache: 'true' + - pattern: "test_task_data" + allow_cache: 'no' + - pattern: "not test_task_data" + allow_cache: 'no' + - pattern: "not test_task_data" + allow_cache: 'true' + name: rest_api_testing [${{ matrix.pattern }}, cache-${{ matrix.allow_cache }}] + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI images + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a + + - name: Generate SDK + run: | + pip3 install -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + - name: Install SDK + run: | + pip3 install -r ./tests/python/requirements.txt \ + -e './cvat-sdk[masks,pytorch]' -e ./cvat-cli \ + --extra-index-url https://download.pytorch.org/whl/cpu + + - name: Run REST API and SDK tests + id: run_tests + env: + COVERAGE_PROCESS_START: ".coveragerc" + CVAT_ALLOW_STATIC_CACHE: ${{ matrix.allow_cache }} + run: | + pytest tests/python/ -k "${{ matrix.pattern }}" --cov --cov-report=json --alluredir=tests/python/allure-results + for COVERAGE_FILE in `find -name "coverage*.json" -type f -printf "%f\n"`; do mv ${COVERAGE_FILE} "${COVERAGE_FILE%%.*}_0.json"; done + + + - name: Uploading code coverage results as an artifact + uses: actions/upload-artifact@v7 + with: + name: ${{format( + 'coverage_results_rest_api-{0}-cache-{1}', + matrix.pattern, matrix.allow_cache + )}} + path: | + coverage*.json + + - name: Creating a log file from cvat containers + if: failure() && steps.run_tests.conclusion == 'failure' + env: + LOGS_DIR: "${{ github.workspace }}/rest_api_testing" + run: | + mkdir $LOGS_DIR + docker logs test_cvat_server_1 > $LOGS_DIR/cvat_server.log + docker logs test_cvat_worker_export_1 > $LOGS_DIR/cvat_worker_export.log + docker logs test_cvat_worker_import_1 > $LOGS_DIR/cvat_worker_import.log + docker logs test_cvat_worker_annotation_1 > $LOGS_DIR/cvat_worker_annotation.log + docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log + # all remaining workers (any one can fail) + docker logs test_cvat_worker_utils_1 > $LOGS_DIR/cvat_worker_utils.log + docker logs test_cvat_worker_annotation_1 > $LOGS_DIR/cvat_worker_annotation.log + docker logs test_cvat_worker_quality_reports_1 > $LOGS_DIR/cvat_worker_quality_reports.log + docker logs test_cvat_worker_consensus_1 > $LOGS_DIR/cvat_worker_consensus.log + docker logs test_cvat_worker_webhooks_1 > $LOGS_DIR/cvat_worker_webhooks.log + docker logs test_cvat_worker_export_1 > $LOGS_DIR/cvat_worker_export.log + docker logs test_cvat_redis_inmem_1 > $LOGS_DIR/cvat_redis_inmem.log + docker logs test_cvat_redis_ondisk_1 > $LOGS_DIR/cvat_redis_ondisk.log + + + + - name: Creating a log file from cvat containers + if: failure() && steps.run_tests.conclusion == 'failure' + uses: actions/upload-artifact@v7 + with: + name: ${{format( + 'rest_api_container_logs-{0}-cache-{1}', + matrix.pattern, matrix.allow_cache + )}} + path: "${{ github.workspace }}/rest_api_testing" + + - name: Upload allure results + if: always() + uses: actions/upload-artifact@v7 + with: + name: ${{format( + 'allure-results-tests-api-{0}-cache-{1}', + matrix.pattern,matrix.allow_cache + )}} + path: tests/python/allure-results + + unit_testing: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Load Docker server image + run: | + docker load --input /tmp/cvat_server/image.tar + docker image ls -a + + - name: Running OPA tests + run: | + python cvat/apps/iam/rules/tests/generate_tests.py + + docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \ + cvat_opa test cvat/apps/*/rules + + - name: Running unit tests + env: + HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} + CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" + run: | + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db + + max_tries=12 + while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done + + docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml \ + run --env PYTHONDEVMODE=1 cvat_ci /bin/bash \ + -c 'coverage run -a manage.py test -v 2 cvat/apps && coverage json && mv coverage.json ${CONTAINER_COVERAGE_DATA_DIR}/unit_tests_coverage.json' + + - name: Uploading code coverage results as an artifact + uses: actions/upload-artifact@v7 + with: + name: coverage_results_unit_tests + path: | + ${{ github.workspace }}/coverage-final.json + ${{ github.workspace }}/unit_tests_coverage.json + + - name: Creating a log file from cvat containers + if: failure() + env: + LOGS_DIR: "${{ github.workspace }}/unit_testing" + run: | + mkdir $LOGS_DIR + docker logs cvat_server > $LOGS_DIR/cvat_server.log + docker logs cvat_opa 2> $LOGS_DIR/cvat_opa.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: unit_tests_container_logs + path: "${{ github.workspace }}/unit_testing" + + e2e_testing: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_tasks4', + 'actions_objects', 'actions_objects2', 'actions_users', + 'actions_projects_models', 'canvas3d_functionality', + 'issues_prs', 'issues_prs2', 'features', 'features2', + 'audio_workspace'] + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v6 + with: + node-version: '22.x' + # NOTE: corepack is not bundled after Node 25 + # https://nodejs.org/docs/v22.18.0/api/corepack.html#corepack + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI image + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a + + - name: Run CVAT instance + run: | + docker compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + -f components/serverless/docker-compose.serverless.yml \ + -f tests/docker-compose.minio.yml \ + -f tests/docker-compose.file_share.yml up -d + + - name: Waiting for server + env: + API_ABOUT_PAGE: "localhost:8080/api/server/about" + run: | + max_tries=60 + status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE}) + while [[ $status_code != "200" && max_tries -gt 0 ]] + do + echo Number of attempts left: $max_tries + echo Status code of response: $status_code + + sleep 5 + status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE}) + (( max_tries-- )) + done + + - name: Run E2E tests + env: + DJANGO_SU_NAME: 'admin' + DJANGO_SU_EMAIL: 'admin@localhost.company' + DJANGO_SU_PASSWORD: '12qwaszx' + + run: | + docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell" + cd ./tests + corepack enable yarn + yarn --immutable + + if [[ ${{ matrix.specs }} == canvas3d_* ]]; then + npx cypress run \ + --headed \ + --browser chrome \ + --config-file cypress_canvas3d.config.js \ + --spec 'cypress/e2e/setup/setup_canvas3d.js,cypress/e2e/canvas3d_*/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + elif [[ ${{ matrix.specs }} == audio_* ]]; then + npx cypress run \ + --headed \ + --browser chrome \ + --config-file cypress_audio.config.js \ + --spec 'cypress/e2e/setup/setup_audio.js,cypress/e2e/audio_workspace/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + else + npx cypress run \ + --browser chrome \ + --spec 'cypress/e2e/setup/setup.js,cypress/e2e/setup/setup_project.js,cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + fi + mv coverage/coverage-final.json coverage/${{ matrix.specs }}_coverage.json + + - name: Uploading code coverage results as an artifact + uses: actions/upload-artifact@v7 + with: + name: coverage_results_e2e_${{ matrix.specs }} + path: | + tests/coverage/${{ matrix.specs }}_coverage.json + + - name: Creating a log file from "cvat" container logs + if: failure() + run: | + docker logs cvat_server > ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log + docker logs cvat_worker_export > ${{ github.workspace }}/tests/cvat_worker_export_${{ matrix.specs }}.log + docker logs cvat_worker_import > ${{ github.workspace }}/tests/cvat_worker_import_${{ matrix.specs }}.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: e2e_container_logs_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cvat_${{ matrix.specs }}.log + + - name: Uploading cypress screenshots as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cypress_screenshots_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cypress/screenshots + + - name: Uploading cypress videos as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cypress_videos_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cypress/videos + + - name: Upload Allure results as an artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: allure-results-e2e-${{ matrix.specs }} + path: tests/allure-results-e2e + + helm_rest_api_testing: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Start minikube + uses: medyagh/setup-minikube@latest + with: + cpus: max + memory: max + + - name: Try the cluster + run: kubectl get pods -A + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI images + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Load images + run: | + eval $(minikube -p minikube docker-env) + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a + + - uses: azure/setup-helm@v5 + + - name: Update Helm chart dependencies + working-directory: helm-chart + run: | + helm dependency update + + - name: Deploy to minikube + run: | + printf " service:\n externalIPs:\n - $(minikube ip)\n" >> helm-chart/test.values.yaml + helm upgrade release-${{ github.run_id }}-${{ github.run_attempt }} --install helm-chart \ + -f helm-chart/cvat.values.yaml \ + -f helm-chart/test.values.yaml \ + --set cvat.backend.tag=${{ env.CVAT_VERSION }} \ + --set cvat.frontend.tag=${{ env.CVAT_VERSION }} + + - name: Update test config + run: | + sed -i -e 's$http://localhost:8080$http://cvat.local:80$g' tests/python/shared/utils/config.py + find tests/python/shared/assets/ -type f -name '*.json' | xargs sed -i -e 's$http://localhost:8080$http://cvat.local$g' + echo "$(minikube ip) cvat.local" | sudo tee -a /etc/hosts + + - name: Generate SDK + run: | + pip3 install --user -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + - name: Install test requirements + run: | + pip3 install --user cvat-sdk/ cvat-cli/ -r tests/python/requirements.txt + + - name: Wait for CVAT to be ready + run: | + max_tries=60 + while [[ $(kubectl get pods -l component=server -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" && max_tries -gt 0 ]]; do echo "waiting for CVAT pod" && (( max_tries-- )) && sleep 5; done + while [[ $(kubectl get pods -l app.kubernetes.io/name=postgresql -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" && max_tries -gt 0 ]]; do echo "waiting for DB pod" && (( max_tries-- )) && sleep 5; done + while [[ $(curl -s -o /tmp/server_response -w "%{http_code}" cvat.local/api/server/about) != "200" && max_tries -gt 0 ]]; do echo "waiting for CVAT" && (( max_tries-- )) && sleep 5; done + kubectl get pods + kubectl logs $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}') + + - name: REST API and SDK tests + # We don't have external services in Helm tests, so we ignore corresponding cases + # They are still tested without Helm + run: | + kubectl cp tests/mounted_file_share $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/home/django/share/ + kubectl exec $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}') -- bash -c "mv /home/django/share/mounted_file_share/* /home/django/share/ && rm -r /home/django/share/mounted_file_share" + pytest --timeout 30 --platform=kube -m "not with_external_services" tests/python --log-level INFO + + - name: Creating a log file from "cvat" container logs + if: failure() + env: + LOGS_DIR: "${{ github.workspace }}/rest_api_testing" + run: | + mkdir ${LOGS_DIR} + for backend_pod in $(kubectl get pods -l tier=backend -o 'jsonpath={.items[*].metadata.name}'); do + kubectl logs ${backend_pod} >${LOGS_DIR}/${backend_pod}.log + done + kubectl logs $(kubectl get pods -l app.kubernetes.io/name=traefik -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/traefik.log + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: helm_rest_api_container_logs + path: "${{ github.workspace }}/rest_api_testing" + + publish_dev_images: + if: github.ref == 'refs/heads/develop' + needs: [rest_api_testing, unit_testing, e2e_testing] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Download CVAT server images + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI images + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push to Docker Hub + env: + SERVER_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/server + UI_IMAGE_REPO: ${{ secrets.DOCKERHUB_WORKSPACE }}/ui + run: | + docker tag "cvat/server:${CVAT_VERSION}" "${SERVER_IMAGE_REPO}:dev" + docker push "${SERVER_IMAGE_REPO}:dev" + + docker tag "cvat/ui:${CVAT_VERSION}" "${UI_IMAGE_REPO}:dev" + docker push "${UI_IMAGE_REPO}:dev" + + generate_report: + name: Generate Allure Report and Upload to S3 + needs: [rest_api_testing, e2e_testing] + if: github.event_name == 'push' && github.ref == 'refs/heads/develop' && github.repository == 'cvat.ai/cvat' + permissions: + id-token: write + contents: read + uses: ./.github/workflows/generate-allure-report.yml + secrets: + AWS_ALLURE_REPORTS_ROLE: ${{ secrets.AWS_ALLURE_REPORTS_ROLE }} + + codecov: + runs-on: ubuntu-latest + needs: [unit_testing, e2e_testing, rest_api_testing] + steps: + - uses: actions/checkout@v7 + + - name: Merge coverage artifacts + uses: actions/upload-artifact/merge@v7 + with: + name: coverage_results + pattern: coverage_results_* + delete-merged: true + + - name: Downloading coverage results + uses: actions/download-artifact@v8 + with: + name: coverage_results + + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 0000000..124dab3 --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,76 @@ +name: Prepare release +on: + workflow_dispatch: + inputs: + newVersion: + description: "Version number for the new release" + required: true + default: X.Y.Z +jobs: + main: + runs-on: ubuntu-latest + steps: + - name: Validate version number + env: + NEW_VERSION: "${{ inputs.newVersion }}" + run: | + if ! [[ "$NEW_VERSION" =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "Invalid version number" + exit 1 + fi + + # When you use the default github.token to make changes in the repository, + # it does not trigger further GitHub pipelines. We want to trigger CI for + # the pull request, so we have to use an app token. + - name: Generate authentication token + id: gen-token + uses: actions/create-github-app-token@v3 + with: + client-id: "${{ secrets.CVAT_BOT_CLIENT_ID }}" + private-key: "${{ secrets.CVAT_BOT_PRIVATE_KEY }}" + + - name: Install dependencies + run: + sudo apt-get install -y pandoc + + - uses: actions/checkout@v7 + + - name: Verify that the release is new + run: | + if git ls-remote --exit-code origin refs/tags/v${{ inputs.newVersion }} > /dev/null; then + echo "Release v${{ inputs.newVersion }} already exists" + exit 1 + fi + + - name: Create release branch + run: + git checkout -b "release-${{ inputs.newVersion }}" + + - name: Collect changelog + run: | + if ls changelog.d/*.md >/dev/null 2>&1; then + pipx run scriv collect --version='${{ inputs.newVersion }}' + fi + + - name: Set the new version + run: + ./dev/update_version.py --set="${{ inputs.newVersion }}" + + - name: Commit release preparation changes + run: | + git -c user.name='cvat-bot[bot]' -c user.email='147643061+cvat-bot[bot]@users.noreply.github.com' \ + commit -a -m "Prepare release v${{ inputs.newVersion }}" + + - name: Push release branch + run: + git push -u origin "release-${{ inputs.newVersion }}" + + - name: Create release pull request + env: + GH_TOKEN: "${{ steps.gen-token.outputs.token }}" + run: | + ./dev/gh_release_notes.sh \ + | gh pr create \ + --base=master \ + --title="Release v${{ inputs.newVersion }}" \ + --body-file=- diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml new file mode 100644 index 0000000..dc94c78 --- /dev/null +++ b/.github/workflows/publish-artifacts.yml @@ -0,0 +1,84 @@ +name: Publish artifacts +on: + release: + types: [released] + +jobs: + docker-images: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Build images + run: | + CVAT_VERSION=latest CLAM_AV=yes docker compose -f docker-compose.yml -f docker-compose.dev.yml build + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push to Docker Hub + env: + DOCKERHUB_WORKSPACE: ${{ secrets.DOCKERHUB_WORKSPACE }} + SERVER_IMAGE_REPO: 'server' + UI_IMAGE_REPO: 'ui' + run: | + docker tag "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest" + + docker tag "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest" + + python-packages: + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v7 + + - name: Generate SDK + run: | + pip3 install --user -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + - name: Build packages + run: | + for d in cvat-sdk cvat-cli; do + pipx run --spec=build pyproject-build --outdir=dist "$d" + done + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + + helm: + runs-on: ubuntu-latest + needs: docker-images + env: + DOCKERHUB_WORKSPACE: ${{ secrets.DOCKERHUB_WORKSPACE }} + steps: + - uses: actions/checkout@v7 + + - name: Set up Helm + uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 + with: + version: v3.19.0 + + - name: Log in to Docker Hub + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Package Helm chart + working-directory: helm-chart + run: | + helm package . --dependency-update --destination ../ + + - name: Push Helm chart to Docker Hub + run: | + helm push cvat-*.tgz oci://registry-1.docker.io/${DOCKERHUB_WORKSPACE} diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml new file mode 100644 index 0000000..b75cebf --- /dev/null +++ b/.github/workflows/schedule.yml @@ -0,0 +1,317 @@ +name: CI-nightly +on: + schedule: + - cron: '0 22 * * *' + workflow_dispatch: + +env: + CYPRESS_VERIFY_TIMEOUT: 180000 # https://docs.cypress.io/guides/guides/command-line#cypress-verify + CVAT_VERSION: "local" + +jobs: + check_updates: + runs-on: ubuntu-latest + env: + REPO: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + last_commit_time: ${{ steps.check_updates.outputs.last_commit_time }} + last_night_time: ${{ steps.check_updates.outputs.last_night_time }} + steps: + - id: check_updates + run: | + default_branch=$(gh api /repos/$REPO | jq -r '.default_branch') + + last_commit_date=$(gh api /repos/${REPO}/branches/${default_branch} | jq -r '.commit.commit.author.date') + + last_night_date=$(gh api /repos/${REPO}/actions/workflows/schedule.yml/runs | \ + jq -r '.workflow_runs[]? | select((.status == "completed")) | .updated_at' \ + | sort | tail -1) + + last_night_time=$(date +%s -d $last_night_date) + last_commit_time=$(date +%s -d $last_commit_date) + + echo Last CI-nightly workflow run time: $last_night_date + echo Last commit time in develop branch: $last_commit_date + + echo "last_commit_time=${last_commit_time}" >> $GITHUB_OUTPUT + echo "last_night_time=${last_night_time}" >> $GITHUB_OUTPUT + + search_cache: + needs: check_updates + uses: ./.github/workflows/search-cache.yml + + build: + needs: search_cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: CVAT server. Getting cache from the default branch + uses: actions/cache@v6 + with: + path: /tmp/cvat_cache_server + key: ${{ runner.os }}-build-server-${{ needs.search_cache.outputs.sha }} + + - name: CVAT UI. Getting cache from the default branch + uses: actions/cache@v6 + with: + path: /tmp/cvat_cache_ui + key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Create artifact directories + run: | + mkdir /tmp/cvat_server + mkdir /tmp/cvat_ui + mkdir /tmp/cvat_sdk + + - name: CVAT server. Build and push + uses: docker/build-push-action@v7 + with: + cache-from: type=local,src=/tmp/cvat_cache_server + context: . + file: Dockerfile + tags: cvat/server:${{ env.CVAT_VERSION }} + outputs: type=docker,dest=/tmp/cvat_server/image.tar + + - name: CVAT UI. Build and push + uses: docker/build-push-action@v7 + with: + cache-from: type=local,src=/tmp/cvat_cache_ui + context: . + file: Dockerfile.ui + tags: cvat/ui:${{ env.CVAT_VERSION }} + outputs: type=docker,dest=/tmp/cvat_ui/image.tar + + - name: Upload CVAT server artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_server + path: /tmp/cvat_server/image.tar + + - name: Upload CVAT UI artifact + uses: actions/upload-artifact@v7 + with: + name: cvat_ui + path: /tmp/cvat_ui/image.tar + + unit_testing: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: '3.10' + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI images + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a + + - name: OPA tests + run: | + python cvat/apps/iam/rules/tests/generate_tests.py + + docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \ + cvat_opa test cvat/apps/*/rules + + - name: Generate SDK + run: | + pip3 install -r cvat-sdk/gen/requirements.txt + ./cvat-sdk/gen/generate.sh + + - name: Install test requirements + run: | + pip3 install -e ./cvat-sdk -e ./cvat-cli -r ./tests/python/requirements.txt + + - name: REST API and SDK tests + run: | + echo 'Run 1: without cache' + pytest tests/python/ --alluredir=tests/python/allure-results + pytest tests/python/ --stop-services + + echo 'Run 2: with cache' + CVAT_ALLOW_STATIC_CACHE="true" pytest tests/python + pytest tests/python/ --stop-services + + - name: Upload allure results for REST API tests + if: always() + uses: actions/upload-artifact@v7 + with: + name: allure-results-rest-api + path: tests/python/allure-results + + - name: Unit tests + env: + HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} + CONTAINER_COVERAGE_DATA_DIR: "/coverage_data" + run: | + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d cvat_opa cvat_server cvat_db + max_tries=12 + while [[ $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health?bundles) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done + + docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml \ + run --env PYTHONDEVMODE=1 cvat_ci /bin/bash \ + -c 'python manage.py test cvat/apps -v 2' + + docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml down -v + + e2e_testing: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + specs: ['actions_tasks', 'actions_tasks2', 'actions_tasks3', 'actions_tasks4', + 'actions_objects', 'actions_objects2', 'actions_users', + 'actions_projects_models', 'canvas3d_functionality', + 'issues_prs', 'issues_prs2', 'features'] + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v6 + with: + node-version: '22.x' + + - name: Download CVAT server image + uses: actions/download-artifact@v8 + with: + name: cvat_server + path: /tmp/cvat_server/ + + - name: Download CVAT UI image + uses: actions/download-artifact@v8 + with: + name: cvat_ui + path: /tmp/cvat_ui/ + + - name: Load Docker images + run: | + docker load --input /tmp/cvat_server/image.tar + docker load --input /tmp/cvat_ui/image.tar + docker image ls -a + + - name: Run CVAT instance + run: | + docker compose \ + -f docker-compose.yml \ + -f docker-compose.dev.yml \ + -f tests/docker-compose.file_share.yml \ + -f tests/docker-compose.minio.yml \ + -f components/serverless/docker-compose.serverless.yml up -d + + - name: Waiting for server + id: wait-server + env: + API_ABOUT_PAGE: "localhost:8080/api/server/about" + run: | + max_tries=60 + status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE}) + while [[ $status_code != "200" && max_tries -gt 0 ]] + do + echo Number of attempts left: $max_tries + echo Status code of response: $status_code + + sleep 5 + status_code=$(curl -s -o /tmp/server_response -w "%{http_code}" ${API_ABOUT_PAGE}) + (( max_tries-- )) + done + + if [[ $status_code != "200" ]]; then + echo Response from server is incorrect, output: + cat /tmp/server_response + fi + echo "status_code=${status_code}" >> $GITHUB_OUTPUT + + - name: Fail on bad response from server + if: steps.wait-server.outputs.status_code != '200' + uses: actions/github-script@v9 + with: + script: | + core.setFailed('Workflow failed: incorrect response from server. See logs artifact to get more info') + + - name: Add user for tests + env: + DJANGO_SU_NAME: "admin" + DJANGO_SU_EMAIL: "admin@localhost.company" + DJANGO_SU_PASSWORD: "12qwaszx" + run: | + docker exec -i cvat_server /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell" + + - name: Run tests + run: | + cd ./tests + corepack enable yarn + yarn --immutable + + shopt -s extglob + if [[ ${{ matrix.specs }} == canvas3d_* ]]; then + npx cypress run \ + --headed \ + --browser chrome \ + --config-file cypress_canvas3d.config.js \ + --spec 'cypress/e2e/setup/setup_canvas3d.js,cypress/e2e/canvas3d_*/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + else + npx cypress run \ + --browser chrome \ + --spec 'cypress/e2e/setup/setup.js,cypress/e2e/setup/setup_project.js,cypress/e2e/${{ matrix.specs }}/**/*.js,cypress/e2e/remove_users_tasks_projects_organizations.js' + fi + + - name: Creating a log file from "cvat" container logs + if: failure() + run: | + docker logs cvat_server > ${{ github.workspace }}/tests/cvat.log + + - name: Uploading cypress screenshots as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cypress_screenshots_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cypress/screenshots + + - name: Uploading cypress videos as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cypress_videos_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cypress/videos + + - name: Uploading "cvat" container logs as an artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: cvat_container_logs_${{ matrix.specs }} + path: ${{ github.workspace }}/tests/cvat.log + - name: Upload Allure results as an artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: allure-results-e2e-${{ matrix.specs }} + path: tests/allure-results-e2e + + generate_report: + name: Generate Allure Report and Upload to S3 + needs: [unit_testing, e2e_testing] + if: always() + + uses: ./.github/workflows/generate-allure-report.yml + secrets: + AWS_ALLURE_REPORTS_ROLE: ${{ secrets.AWS_ALLURE_REPORTS_ROLE }} diff --git a/.github/workflows/search-cache.yml b/.github/workflows/search-cache.yml new file mode 100644 index 0000000..d19b434 --- /dev/null +++ b/.github/workflows/search-cache.yml @@ -0,0 +1,40 @@ +name: Search Cache +on: + workflow_call: + outputs: + sha: + value: ${{ jobs.search_cache.outputs.sha }} + +# if: | +# github.event.pull_request.draft == false && +# !startsWith(github.event.pull_request.title, '[WIP]') && +# !startsWith(github.event.pull_request.title, '[Dependent]') + +jobs: + search_cache: + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.get-sha.outputs.sha}} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + steps: + - name: Getting SHA with cache from the default branch + id: get-sha + run: | + DEFAULT_BRANCH=$(gh api /repos/$REPO | jq -r '.default_branch') + for sha in $(gh api "/repos/$REPO/commits?per_page=100&sha=$DEFAULT_BRANCH" | jq -r '.[].sha'); + do + RUN_status=$(gh api /repos/${REPO}/actions/workflows/cache.yml/runs | \ + jq -r ".workflow_runs[]? | select((.head_sha == \"${sha}\") and (.conclusion == \"success\")) | .status") + + if [[ ${RUN_status} == "completed" ]]; then + SHA=$sha + break + fi + done + + echo Default branch is ${DEFAULT_BRANCH} + echo Workflow will try to get cache from commit: ${SHA} + + echo "sha=${SHA}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/trigger-cvat-models-images-build.yml b/.github/workflows/trigger-cvat-models-images-build.yml new file mode 100644 index 0000000..2a1534e --- /dev/null +++ b/.github/workflows/trigger-cvat-models-images-build.yml @@ -0,0 +1,30 @@ +name: Trigger Agent Image Builds + +on: + workflow_dispatch: + release: + types: [released] + +jobs: + dispatch: + runs-on: ubuntu-latest + steps: + - name: Generate authentication token + id: gen-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CVAT_BOT_CLIENT_ID }} + private-key: ${{ secrets.CVAT_BOT_PRIVATE_KEY }} + repositories: "cvat-models" + + - name: Trigger build in cvat-models + env: + GH_TOKEN: "${{ steps.gen-token.outputs.token }}" + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/cvat-ai/cvat-models/dispatches \ + -d '{"event_type":"build-agent-images","client_payload":{"tag_name":"${{ github.event.release.tag_name }}"}}' diff --git a/.github/workflows/trigger-dependent-repo-update.yml b/.github/workflows/trigger-dependent-repo-update.yml new file mode 100644 index 0000000..9523916 --- /dev/null +++ b/.github/workflows/trigger-dependent-repo-update.yml @@ -0,0 +1,30 @@ +name: Trigger dependent repo update workflow + +on: + workflow_dispatch: + push: + branches: [ develop ] + +jobs: + trigger: + runs-on: ubuntu-latest + steps: + - name: Generate authentication token + id: gen-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CVAT_BOT_CLIENT_ID }} + private-key: ${{ secrets.CVAT_BOT_PRIVATE_KEY }} + repositories: ${{ secrets.CVAT_DEPENDENT_REPO }} + + - name: Trigger private repository workflow + env: + GH_TOKEN: "${{ steps.gen-token.outputs.token }}" + run: | + gh api \ + --method POST \ + /repos/cvat-ai/${{ secrets.CVAT_DEPENDENT_REPO }}/dispatches \ + -f 'event_type=upstream_updated' \ + -F "client_payload[triggeredBy]=${{ github.repository }}" \ + -F "client_payload[commitSha]=${{ github.sha }}" \ + -F "client_payload[branch]=${{ github.ref }}" diff --git a/.github/workflows/update-python-requirements.yml b/.github/workflows/update-python-requirements.yml new file mode 100644 index 0000000..61e11dd --- /dev/null +++ b/.github/workflows/update-python-requirements.yml @@ -0,0 +1,91 @@ +name: Update Python requirements + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "cvat/requirements/*.in" + - "utils/dataset_manifest/requirements.in" + +permissions: + contents: read + pull-requests: read + +jobs: + update: + # Only run on Dependabot branches from this repository. Regenerating + # requirements can execute package build code, so fork PRs must not run this + # workflow with access to the CVAT bot token. + if: > + github.event.pull_request.user.login == 'dependabot[bot]' && + github.event.pull_request.head.repo.full_name == github.repository && + startsWith(github.event.pull_request.head.ref, 'dependabot/pip/') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + cargo \ + g++ \ + gcc \ + libhdf5-dev \ + libldap2-dev \ + libmp3lame-dev \ + libsasl2-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libxmlsec1-openssl \ + make \ + nasm \ + pkg-config \ + python3-dev + + - name: Install pip-compile-multi + run: python -m pip install pip-compile-multi + + - name: Regenerate Python requirements + run: cvat/requirements/regenerate.sh --no-upgrade + + - name: Check generated requirements changes + id: changes + run: | + if git diff --quiet -- cvat/requirements/*.txt utils/dataset_manifest/requirements.txt; then + echo "No generated requirements changes" + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + # The default github.token does not trigger follow-up CI on push. Use the + # same app token as the release workflow so the updated Dependabot PR runs + # the normal validation pipeline. + - name: Generate authentication token + if: steps.changes.outputs.changed == 'true' + id: gen-token + uses: actions/create-github-app-token@v3 + with: + client-id: "${{ secrets.CVAT_BOT_CLIENT_ID }}" + private-key: "${{ secrets.CVAT_BOT_PRIVATE_KEY }}" + + - name: Commit regenerated requirements + if: steps.changes.outputs.changed == 'true' + env: + GH_TOKEN: "${{ steps.gen-token.outputs.token }}" + run: | + git config user.name "cvat-bot[bot]" + git config user.email "147643061+cvat-bot[bot]@users.noreply.github.com" + git add cvat/requirements/*.txt utils/dataset_manifest/requirements.txt + git commit -m "Regenerate Python requirements" + git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" + git push origin HEAD:${{ github.event.pull_request.head.ref }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f2947e --- /dev/null +++ b/.gitignore @@ -0,0 +1,79 @@ +# Project Specific +/data/ +/events/ +/share/ +/static/ +/db.sqlite3 +/keys +/logs +/profiles + +# Ignore temporary files +docker-compose.override.yml +__pycache__ +*.pyc +._* +.coverage +.husky/ +.python-version +/tmp*cvat/ +/temp*/ + +# Ignore generated test files +docker-compose.tests.yml + +# Ignore npm logs file +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +.DS_Store + +#Ignore Cypress tests temp files +/tests/cypress/fixtures +/tests/cypress/screenshots +.idea/ + +#Ignore helm-related files +/helm-chart/Chart.lock +/helm-chart/values.*.yaml +/helm-chart/charts/* + +#Ignore website temp files +/site/public/ +/site/resources/ +/site/node_modules/ +/site/tech-doc-hugo +/site/.hugo_build.lock + +# Ignore all the installed packages +node_modules +/*env*/ +/.*env* +perfkit.egg-info/ + +# Ignore all js dists +cvat-data/dist +cvat-core/dist +cvat-canvas/dist +cvat-canvas3d/dist +cvat-ui/dist + +# produced by prepare in the root package.json script +.husky + +# produced by cvat/apps/iam/rules/tests/generate_tests.py +/cvat/apps/*/rules/*_test.gen.rego + +# produced by allure report +/**/allure-report* +/**/allure-results* + +# https://yarnpkg.com/getting-started/qa#which-files-should-be-gitignored +**/.pnp.* +**/.yarn/* +**/!.yarn/patches +**/!.yarn/plugins +**/!.yarn/releases +**/!.yarn/sdks +**/!.yarn/versions diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ae36fe0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "site/themes/docsy"] + path = site/themes/docsy + url = https://github.com/google/docsy diff --git a/.nycrc b/.nycrc new file mode 100644 index 0000000..8c68499 --- /dev/null +++ b/.nycrc @@ -0,0 +1,19 @@ +{ + "all": true, + "compact": false, + "extension": [ + ".js", + ".jsx", + ".ts", + ".tsx" + ], + "exclude": [ + "**/3rdparty/*", + "**/tests/*", + "cvat-ui/src/actions/boundaries-actions.ts", + "cvat-ui/src/utils/platform-checker.ts" + ], + "parser-plugins": [ + "typescript" + ] +} diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..54f42aa --- /dev/null +++ b/.prettierignore @@ -0,0 +1,10 @@ +.*/ +3rdparty/ +node_modules/ +dist/ +data/ +datumaro/ +keys/ +logs/ +static/ +templates/ diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..2bd8f20 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,27 @@ +{ + "arrowParens": "always", + "bracketSpacing": true, + "embeddedLanguageFormatting": "auto", + "htmlWhitespaceSensitivity": "css", + "insertPragma": false, + "jsxBracketSameLine": false, + "jsxSingleQuote": true, + "printWidth": 120, + "proseWrap": "preserve", + "quoteProps": "as-needed", + "requirePragma": false, + "semi": true, + "singleQuote": true, + "tabWidth": 4, + "trailingComma": "all", + "useTabs": false, + "vueIndentScriptAndStyle": false, + "overrides": [ + { + "files": ["*.json", "*.yml", "*.yaml", "*.md"], + "options": { + "tabWidth": 2 + } + } + ] +} diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..68f8dfe --- /dev/null +++ b/.pylintrc @@ -0,0 +1,986 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + cvat-sdk/cvat_sdk/api_client|cvat-sdk/build|node_modules + + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. The default value ignores Emacs file +# locks +ignore-patterns= + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins=pylint_django + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.10 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=abstract-method, + arguments-out-of-order, + arguments-renamed, + assert-on-string-literal, + assigning-non-slot, + assignment-from-none, + astroid-error, + attribute-defined-outside-init, + await-outside-async, + bad-classmethod-argument, + bad-configuration-section, + bad-file-encoding, + bad-inline-option, + bad-mcs-classmethod-argument, + bad-mcs-method-argument, + bad-open-mode, + bad-plugin-value, + bad-reversed-sequence, + bad-staticmethod-argument, + bad-str-strip-call, + bad-string-format-type, + bad-thread-instantiation, + bidirectional-unicode, + boolean-datetime, + broad-exception-caught, + broad-exception-raised, + c-extension-no-member, + cell-var-from-loop, + chained-comparison, + class-variable-slots-conflict, + comparison-of-constants, + comparison-with-callable, + comparison-with-itself, + condition-evals-to-constant, + config-parse-error, + consider-iterating-dictionary, + consider-math-not-float, + consider-merging-isinstance, + consider-swap-variables, + consider-using-dict-comprehension, + consider-using-dict-items, + consider-using-f-string, + consider-using-from-import, + consider-using-generator, + consider-using-get, + consider-using-in, + consider-using-join, + consider-using-max-builtin, + consider-using-min-builtin, + consider-using-set-comprehension, + consider-using-sys-exit, + consider-using-ternary, + consider-using-with, + cyclic-import, + deprecated-argument, + deprecated-class, + deprecated-decorator, + deprecated-method, + deprecated-module, + deprecated-pragma, + dict-iter-missing-items, + disallowed-name, + django-not-available-placeholder, + django-not-available, + django-not-configured, + django-settings-module-not-found, + duplicate-code, + duplicate-string-formatting-argument, + duplicate-value, + empty-docstring, + eval-used, + f-string-without-interpolation, + fatal, + file-ignored, + fixme, + forgotten-debug-statement, + format-string-without-interpolation, + global-statement, + hard-coded-auth-user, + http-response-with-content-type-json, + http-response-with-json-dumps, + implicit-str-concat, + import-error, + import-outside-toplevel, + import-self, + imported-auth-user, + inconsistent-quotes, + inconsistent-return-statements, + invalid-all-format, + invalid-bool-returned, + invalid-bytes-returned, + invalid-character-backspace, + invalid-character-carriage-return, + invalid-character-esc, + invalid-character-nul, + invalid-character-sub, + invalid-character-zero-width-space, + invalid-characters-in-docstring, + invalid-class-object, + invalid-enum-extension, + invalid-envvar-default, + invalid-envvar-value, + invalid-format-returned, + invalid-getnewargs-ex-returned, + invalid-getnewargs-returned, + invalid-hash-returned, + invalid-index-returned, + invalid-length-hint-returned, + invalid-length-returned, + invalid-metaclass, + invalid-name, + invalid-overridden-method, + invalid-repr-returned, + invalid-str-returned, + invalid-unary-operand-type, + invalid-unicode-codec, + isinstance-second-argument-not-valid-type, + keyword-arg-before-vararg, + line-too-long, + literal-comparison, + locally-disabled, + logging-format-interpolation, + logging-fstring-interpolation, + logging-not-lazy, + method-cache-max-size-none, + method-check-failed, + misplaced-format-function, + missing-class-docstring, + missing-function-docstring, + missing-module-docstring, + missing-parentheses-for-call-in-test, + mixed-line-endings, + model-has-unicode, + model-missing-unicode, + model-no-explicit-unicode, + model-unicode-not-callable, + modelform-uses-exclude, + modified-iterating-dict, + modified-iterating-list, + modified-iterating-set, + multiple-imports, + multiple-statements, + nan-comparison, + no-else-break, + no-else-continue, + no-else-raise, + no-else-return, + no-member, + no-name-in-module, + no-self-argument, + non-ascii-file-name, + non-ascii-module-import, + non-ascii-name, + non-str-assignment-to-dunder-name, + not-a-mapping, + not-an-iterable, + not-async-context-manager, + not-context-manager, + overridden-final-method, + parse-error, + possibly-unused-variable, + possibly-used-before-assignment, + potential-index-error, + preferred-module, + property-with-parameters, + protected-access, + raise-missing-from, + raising-format-tuple, + raw-checker-failed, + redeclared-assigned-name, + redefined-argument-from-local, + redefined-outer-name, + redefined-slots-in-subclass, + redundant-content-type-for-json-response, + redundant-u-string-prefix, + redundant-unittest-assert, + relative-beyond-top-level, + return-arg-in-generator, + self-assigning-variable, + self-cls-assignment, + shallow-copy-environ, + simplifiable-condition, + simplifiable-if-expression, + simplify-boolean-expression, + single-string-used-for-slots, + stop-iteration-return, + subclassed-final-class, + subprocess-popen-preexec-fn, + subprocess-run-check, + super-init-not-called, + super-with-arguments, + super-without-brackets, + suppressed-message, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-positional-arguments, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + trailing-comma-tuple, + trailing-newlines, + try-except-raise, + typevar-double-variance, + typevar-name-incorrect-variance, + typevar-name-mismatch, + unbalanced-tuple-unpacking, + undefined-loop-variable, + unexpected-line-ending-format, + ungrouped-imports, + unhashable-member, + unknown-option-value, + unnecessary-comprehension, + unnecessary-dict-index-lookup, + unnecessary-direct-lambda-call, + unnecessary-dunder-call, + unnecessary-ellipsis, + unnecessary-lambda-assignment, + unnecessary-list-index-lookup, + unnecessary-negation, + unpacking-non-sequence, + unrecognized-inline-option, + unrecognized-option, + unspecified-encoding, + unsubscriptable-object, + unsupported-assignment-operation, + unsupported-binary-operation, + unsupported-delete-operation, + unsupported-membership-test, + unused-argument, + unused-format-string-argument, + unused-private-member, + unused-wildcard-import, + use-a-generator, + use-dict-literal, + use-implicit-booleaness-not-comparison, + use-implicit-booleaness-not-len, + use-list-literal, + use-maxsplit-arg, + use-sequence-for-iteration, + use-symbolic-message-instead, + used-prior-global-declaration, + useless-import-alias, + useless-option-value, + useless-parent-delegation, + useless-return, + useless-suppression, + useless-with-lock, + using-constant-test, + using-f-string-in-unsupported-version, + using-final-decorator-in-unsupported-version, + wrong-exception-operation, + wrong-import-order, + wrong-import-position, + wrong-spelling-in-comment, + wrong-spelling-in-docstring, + yield-inside-async-function, + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=abstract-class-instantiated, + access-member-before-definition, + anomalous-backslash-in-string, + anomalous-unicode-escape-in-string, + arguments-differ, + assert-on-tuple, + assignment-from-no-return, + async-context-manager-with-regular-with, + bad-except-order, + bad-exception-cause, + bad-format-character, + bad-format-string-key, + bad-format-string, + bad-indentation, + bad-super-call, + bare-except, + bare-name-capture-pattern, + binary-op-exception, + break-in-finally, + catching-non-exception, + confusing-with-statement, + consider-using-enumerate, + continue-in-finally, + dangerous-default-value, + duplicate-argument-name, + duplicate-bases, + duplicate-except, + duplicate-key, + exec-used, + expression-not-assigned, + format-combined-specification, + format-needs-mapping, + function-redefined, + global-at-module-level, + global-variable-not-assigned, + global-variable-undefined, + inconsistent-mro, + inherit-non-class, + init-is-generator, + invalid-all-object, + invalid-format-index, + invalid-match-args-definition, + invalid-sequence-index, + invalid-slice-index, + invalid-slots-object, + invalid-slots, + invalid-star-assignment-target, + logging-format-truncated, + logging-too-few-args, + logging-too-many-args, + logging-unsupported-format, + lost-exception, + match-class-bind-self, + match-class-positional-attributes, + method-hidden, + misplaced-bare-raise, + misplaced-future, + missing-final-newline, + missing-format-argument-key, + missing-format-attribute, + missing-format-string-key, + missing-kwoa, + mixed-format-string, + multiple-class-sub-patterns, + no-classmethod-decorator, + no-method-argument, + no-staticmethod-decorator, + no-value-for-parameter, + non-iterator-returned, + non-parent-init-called, + nonexistent-operator, + nonlocal-and-global, + nonlocal-without-binding, + not-callable, + not-in-loop, + notimplemented-raised, + pointless-statement, + pointless-string-statement, + raising-bad-type, + raising-non-exception, + redefined-builtin, + redundant-keyword-arg, + reimported, + repeated-keyword, + return-in-init, + return-outside-function, + signature-differs, + simplifiable-if-statement, + singleton-comparison, + star-needs-assignment-target, + superfluous-parens, + syntax-error, + too-few-format-args, + too-many-format-args, + too-many-function-args, + too-many-positional-sub-patterns, + too-many-star-expressions, + trailing-whitespace, + truncated-format-string, + undefined-all-variable, + undefined-variable, + unexpected-keyword-arg, + unexpected-special-method-signature, + unidiomatic-typecheck, + unnecessary-lambda, + unnecessary-pass, + unnecessary-semicolon, + unreachable, + unused-format-string-key, + unused-import, + unused-variable, + used-before-assignment, + useless-else-on-loop, + useless-object-inheritance, + wildcard-import, + yield-outside-function, + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.* + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=no + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.Exception + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[DJANGO FOREIGN KEYS REFERENCED BY STRINGS] + +# A module containing Django settings to be used while linting. +#django-settings-module= diff --git a/.regal/config.yaml b/.regal/config.yaml new file mode 100644 index 0000000..610778c --- /dev/null +++ b/.regal/config.yaml @@ -0,0 +1,40 @@ +ignore: + files: + - "*_test.gen.rego" +rules: + custom: + naming-convention: + conventions: + - pattern: '^[a-z_]+$|^[A-Z_]+$' + targets: + - rule + - function + - variable + level: error + idiomatic: + no-defined-entrypoint: + # This would likely be the allow rule in each package + # Not critical though, so ignoring for the time being + level: ignore + style: + avoid-get-and-list-prefix: + # Mainly a style preference + # https://docs.styra.com/regal/rules/style/avoid-get-and-list-prefix + level: ignore + external-reference: + # https://docs.styra.com/regal/rules/style/external-reference + level: ignore + opa-fmt: + # https://docs.styra.com/regal/rules/style/opa-fmt + level: ignore + prefer-snake-case: + # Disabled in favor of custom naming-convention rule above + level: ignore + rule-length: + # Many rules longer than the default limit of 30 lines + level: error + max-rule-length: 60 + imports: + unresolved-import: + # one of "error", "warning", "ignore" + level: error diff --git a/.remarkignore b/.remarkignore new file mode 100644 index 0000000..00bd38c --- /dev/null +++ b/.remarkignore @@ -0,0 +1,4 @@ +cvat-sdk/docs/ +cvat-sdk/README.md +.env/ +site/themes/ diff --git a/.remarkrc.js b/.remarkrc.js new file mode 100644 index 0000000..81bdce3 --- /dev/null +++ b/.remarkrc.js @@ -0,0 +1,18 @@ +exports.settings = { bullet: '*', paddedTable: false }; + +exports.plugins = [ + 'remark-frontmatter', + 'remark-gfm', + 'remark-preset-lint-recommended', + 'remark-preset-lint-consistent', + ['remark-lint-list-item-indent', 'one'], + ['remark-lint-no-dead-urls', false], // Does not work because of github protection system + ['remark-lint-maximum-line-length', 120], + ['remark-lint-maximum-heading-length', 120], + ['remark-lint-strong-marker', '*'], + ['remark-lint-emphasis-marker', '_'], + ['remark-lint-unordered-list-marker-style', '-'], + ['remark-lint-ordered-list-marker-style', '.'], + ['remark-lint-no-file-name-irregular-characters', false], + ['remark-lint-list-item-spacing', false], +]; diff --git a/.stylelintrc.json b/.stylelintrc.json new file mode 100644 index 0000000..f16c1b3 --- /dev/null +++ b/.stylelintrc.json @@ -0,0 +1,11 @@ +{ + "extends": "stylelint-config-standard-scss", + "rules": { + "scss/comment-no-empty": null, + "value-keyword-case": null, + "color-function-notation": ["legacy"], + "scss/at-extend-no-missing-placeholder": null, + "no-descending-specificity": null + }, + "ignoreFiles": ["**/*.js", "**/*.ts", "**/*.py"] +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..a9d0cec --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,427 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "REST API tests: Attach to server", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 9090 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/django/" + }, + { + "localRoot": "${workspaceFolder}/.env", + "remoteRoot": "/opt/venv", + } + ], + "justMyCode": false, + }, + { + "name": "REST API tests: Attach to RQ annotation worker", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 9091 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/django/" + }, + { + "localRoot": "${workspaceFolder}/.env", + "remoteRoot": "/opt/venv", + } + ], + "justMyCode": false, + }, + { + "name": "REST API tests: Attach to RQ export worker", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 9092 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/django/" + }, + { + "localRoot": "${workspaceFolder}/.env", + "remoteRoot": "/opt/venv", + } + ], + "justMyCode": false, + }, + { + "name": "REST API tests: Attach to RQ import worker", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 9093 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/django/" + }, + { + "localRoot": "${workspaceFolder}/.env", + "remoteRoot": "/opt/venv", + } + ], + "justMyCode": false, + }, + { + "name": "REST API tests: Attach to RQ quality reports worker", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 9094 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/django/" + }, + { + "localRoot": "${workspaceFolder}/.env", + "remoteRoot": "/opt/venv", + } + ], + "justMyCode": false, + }, + { + "name": "REST API tests: Attach to RQ consensus worker", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "127.0.0.1", + "port": 9096 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/home/django/" + }, + { + "localRoot": "${workspaceFolder}/.env", + "remoteRoot": "/opt/venv", + } + ], + "justMyCode": false, + }, + { + "type": "pwa-chrome", + "request": "launch", + "preLaunchTask": "npm: start - cvat-ui", + "name": "ui.js: debug", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}/cvat-ui", + "sourceMaps": true, + "sourceMapPathOverrides": { + "webpack://cvat/./*": "${workspaceFolder}/cvat-core/*", + "webpack:///./*": "${webRoot}/*", + "webpack:///src/*": "${webRoot}/*", + "webpack:///*": "*", + "webpack:///./~/*": "${webRoot}/node_modules/*" + }, + "smartStep": true, + }, + { + "type": "node", + "request": "launch", + "name": "ui.js: test", + "cwd": "${workspaceFolder}/tests", + "runtimeExecutable": "${workspaceFolder}/tests/node_modules/.bin/cypress", + "args": [ + "run", + "--headless", + "--browser", + "chrome" + ], + "outputCapture": "std", + "console": "internalConsole" + }, + { + "name": "server: django", + "type": "debugpy", + "request": "launch", + "stopOnEntry": false, + "justMyCode": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/manage.py", + "env": { + "CVAT_SERVERLESS": "1", + "ALLOWED_HOSTS": "*", + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": "8282", + }, + "args": [ + "runserver", + "--noreload", + "--insecure", + "127.0.0.1:7000" + ], + "django": true, + "cwd": "${workspaceFolder}", + "console": "internalConsole", + }, + { + "name": "server: chrome", + "type": "pwa-chrome", + "request": "launch", + "url": "http://localhost:7000/", + "disableNetworkCache": true, + "trace": true, + "showAsyncStacks": true, + "pathMapping": { + "/static/engine/": "${workspaceFolder}/cvat/apps/engine/static/engine/", + "/static/dashboard/": "${workspaceFolder}/cvat/apps/dashboard/static/dashboard/", + } + }, + { + "name": "server: RQ - worker", + "type": "debugpy", + "request": "launch", + "stopOnEntry": false, + "justMyCode": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/manage.py", + "args": [ + "rqworker", + "--worker-class=cvat.rqworker.SimpleWorker", + // high priority + "chunks", + // normal priority + "annotation", + "consensus", + "export", + "import", + "quality_reports", + "webhooks", + // low priority + "cleaning", + ], + "django": true, + "cwd": "${workspaceFolder}", + "env": { + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": "8282" + }, + "console": "internalConsole" + }, + { + "name": "server: RQ - scheduler", + "type": "debugpy", + "request": "launch", + "stopOnEntry": false, + "justMyCode": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/rqscheduler.py", + "django": true, + "cwd": "${workspaceFolder}", + "args": [ + "-i", "1" + ], + "env": { + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": "8282" + }, + "console": "internalConsole" + }, + { + "name": "server: migrate", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/manage.py", + "args": [ + "migrate" + ], + "django": true, + "cwd": "${workspaceFolder}", + "env": {}, + "console": "internalConsole" + }, + { + "name": "server: sync periodic jobs", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/manage.py", + "args": [ + "syncperiodicjobs" + ], + "django": true, + "cwd": "${workspaceFolder}", + "env": {}, + "console": "internalConsole" + }, + { + "name": "server: tests", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/manage.py", + "args": [ + "test", + "--settings", + "cvat.settings.testing", + "cvat/apps", + "cvat-cli/" + ], + "django": true, + "cwd": "${workspaceFolder}", + "env": {}, + "console": "internalConsole" + }, + { + "name": "server: REST API tests", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "module": "pytest", + "args": [ + "--verbose", + "--no-cov", // vscode debugger might not work otherwise + "tests/python/rest_api/" + ], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "sdk: tests", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "module": "pytest", + "args": [ + "tests/python/sdk/" + ], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "cli: tests", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "module": "pytest", + "args": [ + "tests/python/cli/" + ], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "api client: Postprocess generator output", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/cvat-sdk/gen/postprocess.py", + "args": [ + "--schema", "${workspaceFolder}/cvat/schema.yml", + "--input-path", "${workspaceFolder}/cvat-sdk/cvat_sdk/" + ], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "docs: Postprocess SDK docs", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/site/process_sdk_docs.py", + "args": [ + "--input-dir", "${workspaceFolder}/cvat-sdk/docs/", + "--site-root", "${workspaceFolder}/site/", + ], + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "docs: Build docs", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/site/build_docs.py", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal" + }, + { + "name": "server: Generate REST API Schema", + "type": "debugpy", + "request": "launch", + "justMyCode": false, + "stopOnEntry": false, + "python": "${command:python.interpreterPath}", + "program": "${workspaceFolder}/manage.py", + "args": [ + "spectacular", + "--file", + "${workspaceFolder}/cvat/schema.yml" + ], + "django": true, + "cwd": "${workspaceFolder}", + "env": {}, + "console": "internalConsole" + }, + { + "name": "core.js: debug", + "type": "node", + "request": "launch", + "cwd": "${workspaceFolder}/cvat-core", + "runtimeExecutable": "node", + "runtimeArgs": [ + "--nolazy", + "--inspect-brk=9230", + "src/api.js" + ], + "port": 9230 + } + ], + "compounds": [ + { + "name": "server: debug", + "configurations": [ + "server: django", + "server: RQ - worker", + "server: RQ - scheduler", + ], + "stopAll": true, + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..03ec388 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,42 @@ +{ + "eslint.probe": [ + "javascript", + "typescript", + "typescriptreact" + ], + "eslint.onIgnoredFiles": "warn", + "eslint.workingDirectories": [ + { + "directory": "${cwd}", + }, + { + "pattern": "cvat-*" + }, + { + "directory": "tests", + "!cwd": true + } + ], + "npm.exclude": "**/.env/**", + "files.trimTrailingWhitespace": true, + "python.analysis.exclude": [ + // VS Code defaults + "**/node_modules", + "**/__pycache__", + ".git", + + "cvat-cli/build", + "cvat-sdk/build", + ], + "python.defaultInterpreterPath": "${workspaceFolder}/.env/", + "python.testing.pytestArgs": [ + "--rootdir","${workspaceFolder}/tests/" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.testing.pytestPath": "${workspaceFolder}/.env/bin/pytest", + "python.testing.cwd": "${workspaceFolder}/tests", + "cSpell.words": [ + "crowdsourcing" + ] +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..3e42457 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,38 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "type": "npm", + "script": "start", + "path": "cvat-ui/", + "label": "npm: start - cvat-ui", + "detail": "webpack-dev-server --env.API_URL=http://localhost:7000 --config ./webpack.config.js --mode=development", + "promptOnClose": true, + "isBackground": true, + "problemMatcher": { + "owner": "webpack", + "severity": "error", + "fileLocation": "absolute", + "pattern": [ + { + "regexp": "ERROR in (.*)", + "file": 1 + }, + { + "regexp": "\\((\\d+),(\\d+)\\):(.*)", + "line": 1, + "column": 2, + "message": 3 + } + ], + "background": { + "activeOnStart": true, + "beginsPattern": "webpack-dev-server", + "endsPattern": "compiled" + } + } + } + ] +} diff --git a/.yarnrc.yml b/.yarnrc.yml new file mode 100644 index 0000000..3186f3f --- /dev/null +++ b/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8a429b9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5760 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + + + + + +## \[2.69.0\] - 2026-06-22 + +### Added + +- \[SDK\] `TrackingFunction` is now exported from `cvat_sdk.auto_annotation` + () + +- Added configurable startup arguments and process counts for backend + workers, including support for running webhook workers through `rqworker-pool` + with a configurable number of processes (numWorkers) +- numProcs is now passed as value, should not use additionalEnv anymore. + NUMPROCS from values override additionalEnv value. + () + +### Changed + +- SSRF mitigations are no longer applied to backing cloud storage requests + () + +- \[SDK\] The `x_organization` parameter passed to individual API calls + now takes precedence over the client-level `organization_slug`. Previously, + `organization_slug` would silently override it, causing those calls to + operate in the wrong organization context + () + +- Updated backend worker liveness probes. Now it validates the expected + worker count from chart values, so both regular workers and webhook worker that uses `rqworker-pool` + can be checked correctly. + () + +- Updated nuclio subchart version + dashboard version + () + +- \[Server API\] Export and backup webhook payloads now report successful + requests with the `succeeded` status instead of `completed` + () + +- Allowed importing COCO and COCO Keypoints annotations without the `iscrowd` field. + () + +### Fixed + +- \[SDK\] Fixed `Project.get_tasks()`, `Project.get_labels()`, `Task.get_jobs()`, + `Task.get_labels()`, `Job.get_issues()`, `Job.get_labels()`, and `Issue.get_comments()` + methods returning an empty list when called on organization resources without + an explicit organization context set on the client, or with a different + organization context set + () + +- Fixed incorrect responses being returned from the `GET + /api/quality/reports` endpoint when the `parent_id` parameter + refers to a report inaccessible by the current user + () + +### Security + +- Fixed a bug allowing a user to determine whether an inaccessible + quality report is located in an organization they are not a member of + or elsewhere + () + + +## \[2.68.0\] - 2026-06-10 + +### Added + +- It's now possible to change the minimum user role needed to create + organizations by setting the `CVAT_ORGANIZATIONS_MIN_ROLE_TO_CREATE` + environment variable + () + +- Added `attempt` and `request_duration` fields to webhook deliveries + () + +- Added an audio annotation workspace with waveform controls, interval regions, and audio task creation + () + +### Changed + +- \[CLI\] Simplified native function agent task cache limiting to keep the last 10 task caches + after task annotation stopped persisting shared chunk caches. + () + +- \[CLI\] Native function agents now download and process task image chunks incrementally + for task annotation requests, reducing temporary disk usage and allowing progress + updates to start sooner. + () + +### Fixed + +- Preserve annotation source in GT jobs during edit/undo/redo so annotations in GT jobs no longer displayed as GT. + () + +- Restricted organization workers assigned to resources from exporting datasets, annotations, or backups + unless they own the resource or relevant parent resource + () + +- Bug which retained organization name in the list even when its deleted () + +- Fixed stale unsaved-change prompts after saving audio interval annotations + () + + +## \[2.67.0\] - 2026-06-02 + +### Added + +- \[Server API\] The project/task/job `preview` endpoints accept + the `Prefer: handling=empty` header (RFC 7240). When set, entities without a + media-derived preview (e.g. point cloud tasks) return `204 No Content` + instead of the default placeholder. + () + +### Fixed + +- \[SDK\] Fixed a PyTorch `UserWarning` about non-writable tensors when converting + polygon masks to tensors in `ExtractInstanceMasks`. + () + +- Memory growth during schema generation + () + +### Security + +- Fixed an XSS vulnerability in annotation guide asset handling + () + + +## \[2.66.0\] - 2026-05-26 + +### Added + +- \[Server API\] Interval annotations: new annotation type with `start`/`stop` boundaries + () +- \[Server API\] Backup export and import for audio tasks + () +- \[Server API\] Generic TSV dataset format for importing and exporting interval annotations + () + +- Added client IP address tracking to event logs + () + +- Added the ability to move objects between layers when z-order sorting is enabled in the object sidebar + () + +- Added the ability to move an object to a specific layer from the object item menu + () + +- Added the ability to move and merge existing layers by drag and drop + () + +- Added layer compaction for minimizing annotation z-order layers and re-enumerating them + () + +- Added z-order filtering for annotations + () + +- Added layer display support in canvas object details + () + +- Chunk download retries in case of network errors + () + +- \[CLI\] Added `backup` and `create-from-backup` commands for projects + () + +- \[CLI\] Added `export-dataset` and `import-dataset` commands for projects in `cvat-cli` + () + +- \[SDK\] Added an `ExtractInstanceMasks` PyTorch target transform for torchvision + instance segmentation models. + () + +- Added CI to trigger build of agent images in cvat-models repo, moved code from ai-models/ to cvat-models repo + () + +### Changed + +- \[SDK\] `ExtractBoundingBoxes` now returns an empty `boxes` tensor with shape + `[0, 4]` instead of `[0]`. + () + +### Fixed + +- Return a validation error when swapping label attribute names instead of failing with an integrity error + () + +- Return a validation error when renaming a label attribute to an existing attribute name + instead of failing with an integrity error + () + +- Fixed severe truncation of label names in the annotation page Labels sidebar + () + +- Fixed a crash when backing up a task with no manifest that's been migrated + to backing cloud storage + () + + +## \[2.65.0\] - 2026-05-19 + +### Added + +- Support filtering by skeleton sub-label and sub-label attributes in the Filters modal + () + +- \[Server API\] The `media_type` field for tasks and jobs + () + +- \[Server API\] An option to create audio-based tasks + () + +- An option to import annotations without removing the existing ones + in task and job annotation uploads. + () + +- \[Server API\] New filter parameters: `read_only` on `/api/access_tokens`; `user_id` and + `accepted` on `/api/invitations`; `org_id` on `/api/requests`. The `accepted` field is now + also returned by `/api/invitations`, and `operation.org_id` by `/api/requests` + () + +- Webhook deliveries are now retried automatically on 5xx, connection errors, + and timeouts, with backoff 5s → 5min → 30min → 3h → 24h × 4 + () + +- \[Server API\] Ground truth jobs can now be created in audio tasks + () + +- \[Server API\] Webhook events `create:export` and `create:backup` that + fire when a dataset export or a project/task backup finishes (success + or failure), so subscribers no longer need to poll `/api/requests` for + the outcome + () + +- Username updates are now available from the profile page + () + +- Added support for deleting label attributes from the label editor and REST API + () + +- Added support for adding, editing, and deleting skeleton element attributes after task or project creation + () + +- Added raw label editor warnings when removing saved attributes, including skeleton element attributes + () + +- Added cancellation support for export requests that are already in progress + () + +- A shortcut to show or hide the bitmap layer on 2D annotation view. + () + +### Changed + +- \[Server API\] Annotations now use `0` as the default value for groups instead of `null`. + It worked this way already, but wasn't reflected in the server API. No behavior or logic changes. + () + +- Change case of 'Checkbox' var names in label editor to lower case + () + +-\[Server API\] The `dimension` field of tasks without data will now be empty instead of `2d` + () + +- \[Server API\] Tasks without data will not report their chunk types or chunk size + () + +- \[Server API\] The minimum accepted value of `image_quality` in + `/api/tasks//data` is now 1 (was 0); `0` was never a usable JPEG quality + () + +- \[Server API\] Enum-like fields (e.g. `status`, `state`, `role`, `type`, `provider_type`) are + no longer matched by the `?search=` parameter on list endpoints; use the corresponding + exact-match filter instead + () + +- Updated Django to 5.2.x + () + +- The server Docker image is now based on Ubuntu 24.04 + () + +- Restricted updates of existing label attributes to fields that do not invalidate annotations + () + +- \[Server API\] Made the requirements for inputs representing paths in + cloud storage or attached file share more strict in several endpoints; + empty, `..` and `.` components are no longer accepted and neither are + leading slashes + () + +### Deprecated + +- \[Server API\] The use of `null` for the `group` field in annotations. Use 0 instead. + () + +- \[Server API\] `image_quality` in `/api/tasks[/]` and `/api/{tasks,jobs}//data/meta` + endpoints for 3D tasks + () + +### Fixed + +- Backend chunk job failures now preserve more meaningful exception details + () +- S3 cloud-storage status probes now fail fast on unreachable or misconfigured + endpoints + () + +- Helm deployments now support multiple releases in one namespace by rendering + OPA, Redis, and Kvrocks wiring with release-scoped service and secret names + () + +- Updated `psycopg2-binary` to avoid local macOS test startup failures caused + by older bundled libpq versions, including `SCRAM authentication requires + libpq version 10 or above` when connecting to PostgreSQL + () + +- Task creation with `remote_files` now returns a readable validation error + when a URL is unreachable or uses an unsupported scheme, instead of a 500 + with a raw traceback + () + +- Fixed polygon/polyline auto simplification switch state is shared + () + +- \[Server API\] Improved performance of the organization, membership, and invitation endpoints + () + +- Show server health-check failures and timeout details when CVAT cannot reach required services + () + +- Clarified 3D cuboid size labels in the object details sidebar to use Length, Width, and Height terminology. + () + +- Fixed task backup export for tasks with media stored in cloud storage + when image database rows are returned out of frame order + () + +### Security + +- Fixed multiple path traversal vulnerabilities + () + + +## \[2.64.0\] - 2026-04-29 + +### Added + +- Polygon and polyline simplification to reduce the number of points. Accessible via button + in object menu (or shortcut) for single object and via annotation actions for multiple objects + () + +- The server can now be configured to store all new eligible tasks on a + particular backing cloud storage + () + +### Removed + +- The unused `list` chunk type. This change is not expected to affect anyone. + () + +### Fixed + +- Fixed chunk retrieval for tasks with point cloud files in the `.bin` format + located in cloud storage + () + +- Fixed creating tasks from `.bin` files in cloud storage that are less + than 16384 bytes long + () + +- Fixed PCDLoader incorrectly reading intensity values in some point cloud files. + () + +- Fixed incorrect reported per-task time in the `movetasktobackingcs` and + `movetaskfrombackingcs` commands + () + +### Security + +- Fixed a cross-site scripting vulnerability in code related to annotation + guides + () + + +## \[2.63.0\] - 2026-04-23 + +### Added + +- CI for agent images + fixed CVE 2025-69720(68121) + () + +- \[CLI\] The `function create-native` command now allows creating functions + with public visibility + () + +- The `movetasktobackingcs` and `movetasktobackingcs` commands can now load + a list of tasks to migrate from a file + () + +- The `movetasktobackingcs` and `movetasktobackingcs` commands now print + statistics about the transfer + () + +### Changed + +- When importing track annotations from a dataset, + the last visible shape of every interval will now include + 2 keyframes - the last visible shape and the outside shape. + If the annotations were originally created in CVAT, the "keyframe" property + can be slightly different from the original annotations after importing. + () + +- CVAT now verifies that the filters defined in Rego policy files + call `add_organization_filter` when the corresponding object belongs + to an organization + () + +- The `movetasktobackingcs` and `movetasktobackingcs` no longer exit with a + failure status when the given task already has the expected backing CS + () + +- Updated Kvrocks to 2.15.0 + () + +### Removed + +- Log output to the `/home/django/logs/supervisord.log.*` files has been disabled, leaving only stdout output + () + +### Fixed + +- Added missing hover tooltips for annotation job page left toolbar controls + () + +- Fixed issue text scaling when zooming into an annotation via double-click + () + +- Imported tracks can be interpolated incorrectly + () + +- Added background to skeleton point state item elements for better visibility and readability on the canvas + () + +- Annotation `score` is not preserved in backups + () +- Invalid date of `annotations.json` in backups + () + +- Snap to point was not working with rotated bounding boxes + () + +- Snap to contour is not working with rotated bounding boxes + () + +- New passwords are now limited to 8 to 256 characters across registration, + password change, and password reset flows + () + +- Tasks without manifests can now use backing cloud storage + () + +- First drawn point was not snapping in `snap to point` feature + () + +- Prevented half-created tasks from being moved to backing cloud storage + () + + +## \[2.62.0\] - 2026-04-02 + +### Added + +- Show success notification after saving annotation guide () + +- Compose for transformers + Helm support for function name env var + () + +### Changed + +- The format for filters in Rego policy files has changed; if you have + added custom policy files, you may need to update them + () + +- 'Remove annotations' confirmation message now more user-friendly. +- 'Delete' button renamed to 'Remove' to match the message + () + +- Bounding box is used as a default prompt for SAM models instead of points + () + +- Snap to contour and snap to point features are now accessible on the controls sidebar + () + +### Fixed + +- Width misalignment between top bars and content lists (tasks, jobs, projects, + cloud storages, webhooks pages) by applying consistent scrollbar gutter spacing. + () + +- Content size on resource pages for smaller screens (<1000px) to provide better responsive layout + () + +- Exported interpolated shapes in 3D cuboid tracks can have invalid rotation + () + +- Fixed chunk generation in tasks with backing cloud storage where the + manifest contains incomplete paths + () + + +## \[2.61.0\] - 2026-03-20 + +### Added + +- "Snap to point" feature for polygon and polyline editing. While enabled, + points being moved or drawn will automatically snap to nearby points from other shapes + () + +- Join tool now supports polygon shapes, allowing to merge multiple + polygons into a single unified polygon. + () + +### Changed + +- Minor visual improvements on the controls sidebar + () + +### Fixed + +- Helm modifiable permissionFix paths and command + () + + +## \[2.60.0\] - 2026-03-17 + +### Added + +- Docker + compose for YOLO + () + +- Interactors can now return multiple shapes instead of a single one + () + +- Interactors can now return a confidence attribute. If present, interaction + results can be filtered in the UI () + +- Added UI controls to remove point and rectangle prompts when using Interactors or OpenCV tools + () + +- Server may add text prompts to interactors as `text_prompts` field in `POST: /api/lambda/functions/` + () + +### Changed + +- Unified the expected interactor interface to align with the `/annotations` API and AI detector outputs + () + +- Updated the IoG serverless function to support the new interactor interface + () + +### Fixed + +- Reduced memory usage by replacing JavaScript arrays with typed arrays + () + +- Fixed memory leaks related to unreleased object URLs in `cvat-canvas` + () + +- \[SDK\] Fixed a crash in `TasksRepo.create_from_backup`, + `ProjectsRepo.create_from_backup`, `Task.upload_data` that could occur + if a recoverable error occurred during chunk uploading + () + + +## \[2.59.1\] - 2026-03-09 + +### Fixed + +- Missing escaping for string fields in `.csv` export + () + + +## \[2.59.0\] - 2026-03-06 + +### Added + +- Docker compose to run SAM2 agent + () + +- Added support for using cloud storage as backing storage for local tasks + () + +- Feature to download Projects, Tasks, Jobs list as `.csv` table + () + +### Changed + +- Changed the display of parent and replica jobs to a flat list. + For convenience, jobs are now marked with “parent” and “replica” tags. + By default, replica jobs are hidden using a filter, but they can be shown if needed. + () + +- Consistent behaviour for Tab/Shift+Tab on 2D and 3D workspace + () + +- Filters on a task page and on jobs page now create a new browser history entry, + enabling Back/Forward navigation between filter states. + () + +- Models page is now always visible on UI, regardless of whether the serverless module is installed + () + +### Deprecated + +- \[Server API\] `GET api/jobs` and `GET api/jobs/{id}/` responses: + - `consensus_replicas` - deprecated in favor of the new `replicas_count` field, + () + +- Property `MODELS` from `/api/server/plugins` now unused on client and soon will be removed from the response + () + +### Removed + +- Field `consensus_replicas`, used for consensus-based tasks, removed from task backups. + Backups created before this change can still be imported, but newer backups will require + a recent CVAT version. + () + +- Deprecated properties `GIT_INTEGRATION` and `PREDICT` from `/api/server/plugins` + () + +### Fixed + +- The `POST /api/task//data` endpoint now responds with a correct + `Location` header when invoked without a trailing slash + () + +- Ground truth annotations are not locked in review mode of regular jobs + () + + +## \[2.58.0\] - 2026-02-23 + +### Added + +- Added a new annotation filter to find rotated bounding boxes and ellipses. + () + +### Fixed + +- Memory leaks on 3D canvas because of unreleased GL resources + () + + +## \[2.57.0\] - 2026-02-17 + +### Added + +- Ability to change zOrder of an object one by one, using corresponding buttons in object menu or shortcuts + () + +### Fixed + +- Creation of extra "create" and "update" events on children summary field + updates and inclusion of invalid data in the events for these fields + () + +- Fixed a crash on fetching an empty related image chunk + of a task in cloud storage + () + +- Fixed creating a task from an attached file share in the case where + at least one image and one directory are specified + () + + +## \[2.56.1\] - 2026-02-03 + +### Fixed + +- Consensus score calculation always returning 1.0 regardless of actual agreement + () + + +## \[2.56.0\] - 2026-02-02 + +### Added + +- Adds the option to sort annotations by label + () + +- Score visualization in UI with a virtual "Votes" attribute calculated as `score × replica_jobs` + () +- A user now may navigate between different shapes with shortcuts (Tab/Shift+Tab by default) in Standard, Review modes + () +- Review mode now supports editing objects. Users can unlock and edit individual annotations as needed + () +- Double-clicking an object in the sidebar now centers it on the canvas and expands its details + () + +- Added a keyboard shortcut to switch object appearance using the “Color by” setting on the annotation page + () + +- Individual annotations for CVAT and analytics ingress in Helm chart + () + +### Changed + +- Increased the maximum number of assets per guide from 30 to 150 + () + +- Consensus merge function now preserves all shapes with their scores, regardless of quorum threshold + () + +### Removed + +- Consensus quorum setting has been removed. All merged annotations are now kept with their consensus scores, + allowing users to filter results based on score thresholds instead + () + +### Fixed + +- Missing validation for the maximum number of annotation guide assets + () + +- In the raw label editor, double quote characters inside SVG strings are no + longer displayed as "\"" + () + +- Annotation `source` was not updated on label change + () + +- Incorrect camera position on 3d scene when position of point clouds + shifted from center of coordinates () + +- Possible 500 error in TUS uploading via SDK/CLI in development or customized deployments + () + + +## \[2.55.0\] - 2026-01-19 + +### Added + +- An option to download quality report as a confusion matrix CSV file + () + +### Fixed + +- Added reading HTTP request data in chunks instead of loading entire body into memory for TUS PATCH requests + () + +- Return a 404 for HEAD requests after a file upload is completed via TUS, instead of a 500 Internal Server Error + () + +- When creating a task, manifest files uploaded alongside videos are no longer ignored + () + +- Improved annotation import performance for Ultralytics YOLO Classification + () + +- Fixed a AssertionError that occurred during the download of 3D annotations. + () + +- Removed redundant `PATCH: /api/jobs//meta` requests sent from annotation view + () + +- \[CLI\] Fixed agents waiting for inappropriate amounts between retrying + failed AR acquire requests + () + +- \[CLI\] Made agents handle failure of queue-related HTTP requests + in a more robust manner + () + +### Security + +- Fixed XSS vulnerabilities related to skeleton SVG images + () + +- Upgrade OPA to new version (1.12.2) + () + +- Users with staff status can no longer change their permissions + () + + +## \[2.54.0\] - 2025-12-24 + +### Added + +- \[CLI\] Added an `Author-email` field to the package metadata + () + +### Changed + +- \[CLI\] Replaced the deprecated `Home-page` field and license classifier + with `Project-URL` and `License-Expression` fields, respectively + () + +### Fixed + +- Сonsecutive slicing of shapes without press Escape key + () + +- Inability to access some tasks with videos with bad keyframes + () + +- Related image display in tasks from the attached file share + () + + +## \[2.53.0\] - 2025-12-18 + +### Changed + +- \[Server API\] TUS upload endpoints no longer accept requests with no + `Content-Length` header + () + +- \[SDK\] Package metadata no longer uses the deprecated `Home-page` and + `License` fields + () + +### Fixed + +- Image scaling was not applied when image filter is enabled + () + +- Fixed TUS resumable upload validation to properly reject chunks that would exceed the declared file size + () + +- \[SDK\] Removed the redundant setuptools runtime dependency + () + +### Security + +- Fixed a directory traversal vulnerability in the `/api/server/share` + endpoint + () + + +## \[2.52.0\] - 2025-12-15 + +### Added + +- Video chapters now can be displayed in the frame player + and can be used for frame navigation during annotation. + () + +- "Return to Previous Page" button on Task, Project, Job, and Cloud Storage Not Found pages + () + +- Double click on a shape in 2D or 3D workspace now fits the shape into scene + () + +- Parameter "Control points size" now have effect for points on 3D canvas + () + +### Changed + +- It is now possible to back up tasks created from a mounted file share that + use static chunk storage + () + +- Updated Nuclio to 1.15.9 + () + +- Reduced RAM usage on track export + () + +- Better zoom-in, zoom-out algorithm on side views of 3D canvas + () + +- Last zoom value on side cameras of 3D canvas memoized per object and restored when object reselected + () + +- Settings AAM Zoom Margin is now more general and responsible for paddings around focused objects + () + +- Improved algorithm for default zoom on side views of 3D canvas + () + +### Removed + +- SiamMask and some OpenVINO-based functions + () + +### Fixed + +- Backups of tasks created from a mounted file share no longer fail to import. + Note that backups of such tasks created by previous versions of CVAT still cannot be imported + () + +- Heavyweight backups created from tasks using cloud storage that have + images as frames and non-default start frame, stop frame or frame step + settings no longer fail to import. Note that the fix is for backup + creation; as such, CVAT will still not be able to import backups of + such tasks created by previous versions + () + +- \[CLI\] Fixed a truncated error message that could be printed when running + an agent for a remote function missing a (sub)label from the loaded AA function + () + +- Fixed creation of tasks from images in cloud storage without a manifest + that use static chunks and a custom frame range + () + +- Low visibility of object details over canvas if background or image is dark + () + +- Weird camera behaviour when layout of 3D canvas gets resized + () + + +## \[2.51.0\] - 2025-12-01 + +### Added + +- Documentation for using Backblaze B2 as an S3-compatible cloud storage option in CVAT + () + +### Changed + +- Relaxed video manifest creation to make use of keyframes even if seek lands earlier + () + +### Removed + +- Python 3.9 support (due to Python 3.9 EOL) + () + +### Fixed + +- Fixed OpenAPI schema for `retrieve_data` endpoints: marked `type` parameter as required for both tasks and jobs API + () + +- Calculation of statistics in the job is failed when there is a track without keyframes + () + +- Update the `updated_date` field of the Task when PATCHing `/api/tasks//data/meta` + () + +- Incorrect retry handling of `429 TooManyRequests` error in case of data uploading via TUS protocol + () + +- Error message is not displayed if not possible to fetch data for 3D canvas + () + + +## \[2.50.0\] - 2025-11-26 + +### Added + +- \[Helm\] Kvrocks PVC configuration via annotations + () + +- Added kvrocks PVC VolumeAttributeClass support + () + +- Added VolumeAttributesClass creation to public chart + () + +### Changed + +- Change expiration date format view in 'Security' -> 'Create API Token' + from default ISO to DD/MM/YYYY so that it matches the dates in the token table + () + +- Files located in the `data/tasks/` directory are no longer included + in task backups, nor extracted from such backups when restoring. Recent + versions of CVAT (since v2.6.2) no longer create or use such files + () + +- Updated Traefik to v3.6.x + () + +### Fixed + +- Excessive `GET /api/users` requests on task page for each assigned job + () + +- Actions menu can be opened twice on different resource cards: Projects, Jobs, Cloud storages, etc. + () + +- Quality conflicts can now be displayed in the review mode of consensus replicas + () + +- Fixed cloud storage status. Unavailable storages now return NOT_FOUND status instead of 400 Bad Request + () + + +## \[2.49.0\] - 2025-11-06 + +### Added + +- Helm charts are now available on Docker Hub, at + () + +### Changed + +- Admins will no longer see access tokens of other users on the token management page + () + +### Removed + +- \[Server API\] Removed `GOOGLE_DRIVE` from the list of accepted cloud + storage provider types; it has never been implemented + () + +- \[Server API\] Only own access tokens will be returned in the `GET /api/auth/access_tokens` + responses for everyone, including admins + () +- \[Server API\] The `owner` filters are removed from the `GET /api/auth/access_tokens` endpoint + () + +- \[Server API\] The redundant `storage` parameter of the `POST /api/tasks//data` + endpoint has been removed; the storage location is determined based on + other parameters + () + +### Fixed + +- Improved memory use in project dataset exports + () + +- Aligned the names of cloud storage services in the UI with their official + names + () + +- Improved performance of access token editing page in the admin panel + () + +- Incorrect chunk creation for some video files after FFmpeg update + () + +### Security + +- Fixed a vulnerability that let users write to the attached network share + () + + +## \[2.48.1\] - 2025-10-29 + +### Removed + +- It is no longer possible to upgrade directly from CVAT releases prior + to v2.0.0 + () + +### Fixed + +- UI crush on failed `GET /api/server/annotation/formats` request + () + + +## \[2.48.0\] - 2025-10-27 + +### Added + +- \[CLI\] `CVAT_ACCESS_TOKEN` environment variable can now be used for authentication with an API token + () +- \[SDK\] `Client.login()` and `make_client()` can now be called with an API token + () +- \[SDK\] `make_client()` can now be called with a server URL that contains a port component + () + +- \[Server API\] Support for API access tokens + () + +- \[Server\] A configuration option to set maximum job limit per task + () + +### Fixed + +- Tracks does not leak to other jobs on task export + () + +- Incorrect cloud storage value in tasks within a project after transferring between organizations + () + +- Inefficient memory usage when counting number of objects in tracks + when updating job annotations or analytics report computing + () + + +## \[2.47.0\] - 2025-10-14 + +### Added + +- Made disk usage health check threshold configurable and updated documentation + () + +### Changed + +- FFmpeg updated to 8.0 + () + +- \[SDK\] Enabled retrying on some server error statuses by default + () + +### Fixed + +- Improved performance of task creation from cloud without manifest + () + + +## \[2.46.1\] - 2025-10-09 + +### Security + +- Bump Redis version to 7.2.11 to address CVE-2025-49844 + () + + +## \[2.46.0\] - 2025-10-08 + +### Added + +- Support for related images in 2d and 3d tasks bound to cloud storages + () +- Support for 3d tasks with non-archived files bound to cloud storages + () +- \[Dataset manifest tool\] now can handle 3d datasets in all 4 supported file layouts + () + +### Changed + +- Enabled validation of the frame `width` and `height` fields in manifests + (now required both for 2d and 3d dataset manifests) + () +- Dataset manifests now can include the `original_name` meta field with the server file name + () + +- Tasks can now be created without specifying the labels in advance + () + +- \[SDK\] Removed the `tuspy` dependency + () + +- Bump helm chart version, using images from our public repo now + () + +- The `CVAT_ALLOW_STATIC_CACHE` server configuration parameter + will no longer affect the existing tasks. It will only affect new tasks, + as stated in the description. Existing tasks will use chunks as + configured in the task. + () + +### Deprecated + +- Excessive filtering for media files containing "related_images" in the path during task creation. + Only the actual related images wrt. the input media layout will be filtered out in the future. + () + +### Removed + +- Removed non-functional API URL signing support + () + +### Fixed + +- Related image detection for 2d and 3d media in all 5 supported layouts + () +- Improved documentation about supported task file layouts with related images + () +- Improved error messages for invalid media in task creation + () + +- Re-enables helm chart compatibility with K8s pre-release versions + () + +- Model card clipping on windows with small height + () + +- UI error `Cannot read properties of undefined (reading 'points')` occurring in various scenarios + () + +- Fixed incorrect class names in some user messages + () + +- Updating organization description does not work on the organization page + () + +- Bulk delete for cloud storages was deleting only one instance + () + +- Fixed backend failing to start when analytics are disabled + () + +- A possible error when trying to merge tag annotations via consensus + () + +- Worked around an issue that may cause an error message to be replaced + with "No module named 'pkg_resources'" + () + + +## \[2.45.0\] - 2025-09-17 + +### Added + +- Basic user profile page that allows to change personal info, change password + () + +- \[Helm\] Set fsGroup for the Kvrocks pod to GID of the user the process runs as + () + +### Fixed + +- Fixed spacing in task creation status messages + () + +- Unaccepted organization invitations could not be deleted + () + +- \[SDK\] Fixed the `torchvision_instance_segmentation` AA function + returning invalid polygons in cases when the underlying model detects a very + small object + () + +- Reduced RAM usage during task export + () + + +## \[2.44.2\] - 2025-09-08 + +### Fixed + +- User may create a skeleton with invalid structure in configurator, leads to UI crash + () + +- Crash on job export if the job contains tags or is not the first job in the task + () + + +## \[2.44.1\] - 2025-09-02 + +### Fixed + +- Attribute/label input fields are not disabled when the object is locked + () + + +## \[2.44.0\] - 2025-09-01 + +### Added + +- Introduced bulk actions to perform operations on multiple selected resources at once + - Multi-select resources using click with Ctrl, Shift or using Select all button at the top bar + - Supported resources: Tasks, Jobs, Projects, Requests, Organization members, Webhooks, Cloud Storages + - Supported operations: Export, Backup, Delete, Download, Change: Assignee, State, Stage, Role + () + +- \[SDK\] Auto-annotation functions are now able to output tags + () + +- Now it is possible to move projects and tasks between organizations + () + +- Improved validation errors for invalid json filter queries + () + +- \[SDK, CLI\] Support for exporting with server-generated filename + () + +- Organization transfer now supported as a bulk action + () + +- Ability to replace cloud storage for tasks. + () + +- Lightweight backup option in Export backup dialog (excludes media for cloud-storage tasks). + () + +### Changed + +- Updated Yarn version from 1.22.22 to 4.9.2 + () + +- \[SDK\] simplified sending of custom requests with `ApiClient`, added documentation + and improved related APIs. Documented using foreign libraries for sending requests. + () + +- \[CLI\] `task backup` and `task export-dataset` now download files with the server-generated + filenames by default + () +- \[CLI\] `task backup` and `task export-dataset` now always export files locally, + regardless of the default export location on the server + () + +- The cvat/server Docker image is now configured with a numeric UID + rather than a username + () + +- The frontend container no longer runs as root + () + +- \[Helm\] The Clickhouse, PostgreSQL and Redis containers now use + the images from the Bitnami Legacy repository by default + () + +- \[Compose, Helm\] The Vector container now runs as a non-root user + () + +- \[Helm\] The Kvrocks container now has overridden UID/GID, making + it compatible with the `runAsNonRoot` security context setting + +- Made the error message when a particular image cannot be saved + to a compressed chunk more useful + () + +- When registering a user, the server will now reject overly long email, + first name and last name fields, instead of truncating them + () + +### Removed + +- Removed deprecated `seed` parameter in job creation in favor of `random_seed` + () + +### Fixed + +- Invalid GT job frame numbers in backups of video tasks with custom start/stop frame or frame step + () + +- Server error in GT job creation if the `random_per_job` frame selection method + was used with the `seed` parameter. + () + +- \[Helm\] A useless `/models` directory is no longer created in the main + data volume + () + + +## \[2.43.0\] - 2025-08-07 + +### Added + +- Django command to remove user with all resources `python manage.py deleteuser ` + () + +### Changed + +- Better validation of fields specified in raw labels editor + () + +- Optimized preview requests for Projects, Tasks, Jobs, etc. — now sent sequentially to reduce load on the server + () + +### Fixed + +- Issue dialogs appear outside the visible area when the issue is located near the right or bottom edges of the frame + () + +- Job meta could include `deleted_frames` outside the job + () + + +## \[2.42.0\] - 2025-07-29 + +### Added + +- \[SDK, CLI\] Added an auto-annotation function interface for tracker + functions, and agent support for it + () + +- \[SDK\] `TaskDataset` now supports tasks with video chunks when created + with `MediaDownloadPolicy.PRELOAD_ALL`; this also means that agents can + process interactive detection requests on such tasks + () + +- Support for cgroup v2 to calculate the number of threads when downloading images from cloud storages + () + +### Fixed + +- Bump Python runtime version for Segment Anything interactor Nuclio function from 3.8 to 3.10 + () + +- User could not log in if an email used for the invitation had a different case + than the one used during manual registration + () + +- Fixed downloading images only in 1 thread when preparing chunks in case of using cloud storage + () + +### Security + +- Added missing email verification check when Basic HTTP authentication is used + and server is configured to require email verification (`ACCOUNT_EMAIL_VERIFICATION` == `mandatory`) + () + + +## \[2.41.0\] - 2025-07-09 + +### Added + +- Page size selector for different resource pages + () + +- Selector that allows inline editing of the following fields from card views: `assignee`, `state`, and `stage`. + () + +### Changed + +- Improved email templates for email confirmation and organization invitation + () + +- \[CLI\] Reduced log clutter in the `function run-agent` command + () + +- The `PATCH` and `PUT` methods on the `/api/(tasks|jobs)//annotations` + paths now verify that annotation IDs are present/absent, depending on the + action + () + +- Changed the default Django cache backend from LocMem to Redis + () + +- Unified design of actions menu on organization page to match style of other action menus + () + +### Deprecated + +- \[Server API\] Token authentication + () + +### Fixed + +- Fixing COCO keypoints export for case when some keypoints are absent + () + +- Incorrect logo in email template for email confirmation + () + +- Fixed LDAP as an issue was formed with the wrong arguments being apart of the function definition + () + +- \[Server API\] Actualized outdated API schema for token and session authentication + () + +- Low performance in `GET /api/jobs(tasks)//annotations` + when a target resource have many tracks with attributes, especially mutable + () + +- Shortcuts cannot be properly configured in tag annotation mode + () + +- Setting "Automatically go to the next frame" does not apply when the first tag is added + on tag annotation workspace + () + + +## \[2.40.1\] - 2025-07-07 + +### Fixed + +- Low performance of DELETE `/api/tasks/` and GET `/api/jobs(tasks)//annotations` + Because of inefficient database queries () + + +## \[2.40.0\] - 2025-06-25 + +### Added + +- Serverless tracker functions may now accept shapes other than rectangles + () + +- \[CLI\] A more helpful error message is now raised if a loaded + auto-annotation function has no `spec` attribute + () + +- `CVAT_CACHE_ITEM_MAX_SIZE` option that limits size of data chunk at CVAT level. + Generating data that exceeds the size will result in an exception. + () + +- CVAT server tracks `last_activity_date` of a user, the field is updated once a day + () + +- Filtration by username to Grafana dashboards + () + +### Changed + +- Updated zooming algorithm, it works much smoother with touchpads and a little bit smoother for mice + () + +- Kvrocks: configured auto compaction at scheduled time. + () + +- Nuclio tracker functions are no longer passed the previous frame's shapes + when continuing the tracking + () + +- Endpoints accepting annotations as input now check that shapes have + point and element counts that are appropriate for the shape type + () + +### Fixed + +- Reduced excessive DB use in dataset export + () + +- Page size selector on organization page was not working + () + +- Incorrect width of project field on webhook setup page + () + +- Relevant task quality reports now can be reused in project quality reports + () + +- Fixing 3d export for projects + () + +### Security + +- Added missing file name validation when initiating an import process + from a file uploaded via the TUS protocol + () + + +## \[2.39.0\] - 2025-06-05 + +### Added + +- \[SDK\] Added `decode_mask`, a utility function that creates a bitmap + based on the `points` array in a mask shape + () + +- \[SDK\] `encode_mask` may now be called without specifying a bounding box + () + +### Changed + +- \[TUS\] After finishing file upload using the TUS protocol, the next request that initiates + the import process must include the TUS `file_id` instead of the original file name. + It can be obtained from the `Upload-Filename` response header. + () + +- Frame input field now dynamically adjusts its width according to the maximum frame number in the job + () + +### Removed + +- \[TUS\] `Upload-Filename` header from server responses when handling append chunk requests + () + +### Fixed + +- \[TUS\] TUS metadata files store only declared fields + () + +- \[Helm\] Fixed configuration of Grafana Clickhouse data source, + which led to the impossibility to connect to the Clickhouse database + () + +- Improved performance of GET `api/lambda/requests` requests + () + + +## \[2.38.0\] - 2025-05-27 + +### Added + +- Set `file` as default source of annotations imported from files + () + +- CVAT measures the size of data, uploaded by users, like images, videos and different guide assets. + Newly created resources will be measured automatically. + For existing resources, please, run `python manage.py initcontentsize`. + () + +### Changed + +- Cache files with exported events now are stored in `/data/cache/export/` instead of + `/data/tmp/`. These files are periodically deleted by the + `cleanup_export_cache_directory` cron job + () + +### Deprecated + +- The `GET /api/events` endpoint is deprecated in favor of the `POST /api/events/export`, + `GET /api/requests/rq_id`, and `GET result_url`, where `result_url` is obtained from + background request details + () +- The `POST /api/quality/reports/rq_id=rq_id` is deprecated in favor of + `GET /api/requests/rq_id` + () + +### Removed + +- The `POST /api/consensus/merges?rq_id=rq_id` endpoint no longer supports + process status checking + () +- The `GET /api/projects/id/dataset?action=import_status` endpoint no longer + supports process status checking + () +- The `POST /api/projects/backup?rq_id=rq_id` endpoint no longer supports + process status checking + () +- The `POST /api/tasks/backup?rq_id=rq_id` endpoint no longer supports + process status checking + () +- The `PUT /api/tasks/id/annotations?rq_id=rq_id&format=format` endpoint + no longer supports process status checking + () +- The `PUT /api/jobs/id/annotations?rq_id=rq_id&format=format` endpoint + no longer supports process status checking + () +- \[SDK\] `DatasetWriteRequest`, `BackupWriteRequest`, `TaskAnnotationsWriteRequest`, + `JobAnnotationsUpdateRequest`, `TaskAnnotationsUpdateRequest` classes were removed + () + +### Fixed + +- YOLO formats can now be imported from archives where dataset is located in a folder + () + +- Quality setting `Check orientation` was not updated after save + () + +- Only first 10 tasks were shown on project quality page + () + +- When restoring a task or a project from backup, 'owner' field for assets has empty value + () + +- Ultralytics YOLO format could not import annotations + if no image info was provided in the dataset + () + +### Security + +- Fixed disclosure of certain resource names and IDs via the browsable API + () + + +## \[2.37.0\] - 2025-05-15 + +### Added + +- Annotation quality checks for projects + () + +- \[CLI\] Agents are now able to handle interactive detection requests + () + +- \[SDK\] `BackgroundRequestException` which is now raised instead of `ApiException` + when a background request (e.g., exporting a dataset or creating a task) fails + () + +- Datumaro format now supports ellipses + () + +- Frame search by filename + () + +### Changed + +- Requests that initiate background processes (e.g. exporting datasets) now return a request ID + too when a 409 status is returned + () + +### Deprecated + +- \[Server API\] `GET api/quality/reports` and `GET api/quality/reports/{id}/` responses: + - `frame_count` - deprecated in favor of the new `validation_frames` field, + - `frame_share` - deprecated in favor of the new `validation_frame_share` field + () + +### Fixed + +- Optimized `GET api/quality/reports/`, `GET api/quality/conflicts/` requests, + permission checks in `api/quality/*` endpoints + () + +- Do not require COCO annotations to have fields which are not necessary for import + () + +- `redis.exceptions.ResponseError: wrong number of arguments for 'watch' command` exception that + could be raised inside a worker when enqueuing jobs that depend on a running job + () + +- Fixed issue on track interpolation: 'tuple' object has no attribute 'copy' + () + +- \[SDK\] Fixed outdated note about attributes in the docstring of + `DetectionFunction.detect` + () + +- When no shortcuts were assigned, the tooltip displayed empty brackets + () + + +## \[2.36.0\] - 2025-05-08 + +### Added + +- UI button to export raw resource events on analytics page + () + +- Input size controls for cuboids in 3D workspace + () + +### Changed + +- Export of events using the server endpoint `GET /api/events` ignores `send:exception` scope. + () +- Updated default value of `from` query parameter in GET `/api/events`. + Now it exports all events of the target resource if `from` and `to` are not specified. + () +- Export table as CSV feature now considers applied filtration on the table + () + +- \[CLI\] The default value for `--server-host` is now `http://localhost` + () + +### Deprecated + +- \[SDK, CLI\] Automatic server URL scheme detection is deprecated. + Add `https://` or `http://` to the host explicitly to avoid future breakage + () + +### Fixed + +- Fixed helm-chart to use selectorLabels template for matchLabels #9358 + () + +- 500 status code returned when an API method is not allowed + () + +- Optimized the `api/jobs/` server endpoint + () +- Optimized DB requests for server permission checks + () + +- \[CLI\] Commands with invalid arguments or `--help` no longer ask for the + server password + () + +- Improved performance of `GET /api/tasks`, `GET /api/quality/conflicts` + and `GET /api/cloudstorages` requests + () + +- Improved performance of `GET /api/webhooks` requests + () + +- Tracking with the AI model was not starting automatically after being re-enabled + () + + +## \[2.35.0\] - 2025-04-29 + +### Changed + +- Streaming import for YOLO and COCO formats + () + +- The `POST /api/lambda/functions/` endpoint now returns the results + in the same format as the `GET /api/tasks//annotations` endpoint + when the function is of the `detector` kind + () + +### Fixed + +- Numeric attribute values returned by Nuclio functions are now checked + for being in the acceptable range when running whole-task auto-annotation + () + +- With per-frame auto-annotation, numeric attribute range validation now + works correctly when the minimum value is not a multiple of the step + () + +- Reduced memory consumption for annotation export to CVAT formats + () + +- UI crashes when paste cuboids with hold Ctrl key + () + +- Fixed service name of utils worker in `docker-compose.external_db.yml` + () + +- Slow performance in exports that require CVAT RLE to COCO RLE conversion + () + + +## \[2.34.0\] - 2025-04-17 + +### Added + +- Cuboid orientation arrows for 3D canvas + () + +### Fixed + +- Fixed rotation of shapes existing in CVAT before #9289 + () + + +## \[2.33.0\] - 2025-04-15 + +### Added + +- \[CLI\] Agents can now receive real-time notifications about new annotation + requests from the server + () + +- Collecting User-Agent info in events + () + +### Fixed + +- Reduced memory consumption for annotation import to tasks + () + +- Reduced memory consumption for annotation import to jobs + () + +- Links in the actions menu now behave as regular links, allowing middle-click to open them in a new tab. + () + +- Recorded working time may be less than actual when dealing with complex masks or polygons + () + +- Rotated rectangles and ellipses unstably reset after resizing + () + +- Redis migration `002_update_meta_in_export_related_jobs` could fail + due to stale RQ job keys in the deferred job registry + () + +- Duplicate buttons in skeleton point config modal + () + +- Fixed inference with the YOLOv7 model on grayscale images + () + +- Reduced memory consumption for dataset import to project + () + +- Rough rotation of cuboids in 3D workspace + () + +- Dataset structure validation now runs when uploading task/job annotations in CVAT format + () + +- Incorrect link on job import success notification + () + + +## \[2.32.0\] - 2025-03-24 + +### Added + +- Added parameter `conv_mask_to_poly` support for importing annotations in projects,tasks and jobs + () + +- \[SDK\] Auto-annotation functions that output skeletons can now be used + via agents + () + +- Search bar and filtering components on the organization page () + +- Error notification if something is wrong in quality/consensus settings + () + +- \[Helm\] Added a new value, `cvat.backend.extensionEnv`, to support + supercharts adding environment variables to backend containers + () + +- Timestamps to Uvicorn stdout logs + () + +- Added robots.txt file to manage crawling traffic + () + +### Changed + +- \[SDK\] `DetectionFunctionSpec` now requires that the type of keypoint + sublabels is set to `points`. Accordingly, `keypoint_spec` now sets + this type by default + () + +- Optimized memory usage on export with YOLO and COCO formats for tasks + () + +- Optimized memory usage for project export in YOLO and COCO formats + () + +- Updated Traefik to v3.3.x + () + +- \[Compose\] Traefik access log is now limited to the same fields as in + Helm-based deployments + () + +### Deprecated + +- Utilizing `GET /api/projects/id/dataset?action=import_status` API endpoint + to check the status of the import process. Instead, the `GET /api/requests/rq_id` + requests API should be used () + +### Removed + +- `GET /api/projects/id/dataset` API endpoint no longer handles dataset export process + () +- `GET /api/projects/id/annotations` API endpoint no longer handles annotations export process + () +- `GET /api/projects/id/backup` API endpoint no longer handles project export process + () +- `GET /api/tasks/id/dataset` API endpoint no longer handles dataset export process + () +- `GET /api/tasks/id/annotations?format=` API endpoint no longer handles annotations export process + () +- `GET /api/tasks/id/backup` API endpoint no longer handles task export process + () +- `GET /api/jobs/id/dataset` API endpoint no longer handles dataset export process + () +- `GET /api/jobs/id/annotations?format=` API endpoint no longer handles annotations export process + () + +- Existing implementation of analytics reports + () + +### Fixed + +- Removed extra sliders on quality control page + () + +- Server returns a 404 status code with details instead of a 500 + when cloud storage preview defined by a manifest cannot be downloaded + () + +- Broken styles on Consensus management page + () + +- \[Helm\] Fixed frontend deployment template issue that caused rendering to fail if + additional volumes and volume mounts were defined. + () + +- Incorrect behavior of standard browser back button on the quality page + () + +- Fixed a 500 status code that could occur when listing requests by GET /api/requests + () + +- Memory usage optimization on backup import + () + + +## \[2.31.0\] - 2025-03-03 + +### Added + +- \[SDK\] Auto-annotation detection functions can now output shape/keypoint attributes + () + +- \[SDK\] Added a utility module for working with label attributes, + `cvat_sdk.attributes` + () + +- Simple merging for consensus-enabled tasks + () + +- A setting to display rectangles and ellipses dimensions and rotation + () + +### Changed + +- Hidden points in skeletons now also contribute to the skeleton similarity + in quality computations and in consensus merging + () + +- SDK `task.upload_data()` can accept resources of the `Path` type + when `resource_type` is `REMOTE` or `SHARE` + () + +### Deprecated + +- Utilizing `PUT /api/tasks|jobs/id/annotations?rq_id=rq_id` API endpoint + to check the status of the import process + () + +### Fixed + +- 500 status code returned by API endpoints that support TUS OPTIONS requests + () + +- Possible race condition that could occur when importing annotations + () + +- Issue label scaling on image filter application + () + +- Invalid display of images in simple GT jobs + () + +- Related images in a simple GT jobs are displayed incorrectly + () + + +## \[2.30.0\] - 2025-02-14 + +### Added + +- Gamma filter settings are now automatically saved and restored upon reload + () + +- Ability to customize `api/sever/about` endpoint via settings including logo and sign-in page subtitle + () + +### Changed + +- Client settings are now saved automatically + () + +### Fixed + +- \[SDK\] `skeleton_label_spec` now correctly forwards `kwargs` to + `PatchedLabelRequest` + () + +- Error: Cannot read properties of undefined (reading 'width') that occurs when changing frames in a video-based GT job + () + + +## \[2.29.0\] - 2025-02-10 + +### Added + +- Tasks created from cloud storage can be backed up now + () + +- \[CLI\] `function create-native` now sends the function's declared label types + to the server + () + +### Changed + +- When invoking Nuclio functions, labels of type `any` can now be mapped to + labels of all types except `skeleton` + () + +### Fixed + +- Fixed invalid server-side track interpolation in tasks with deleted frames + () + + +## \[2.28.0\] - 2025-02-06 + +### Added + +- Support for managing Redis migrations + () + +### Changed + +- Updated limitation for minimal object size from 9px area to 1px in dimensions + () + +### Fixed + +- Invalid chunks and backups after honeypot updates in tasks with cloud storage data + () + +- In some cases effect of drag/resize may be reset implicitly for a user + () + + +## \[2.27.0\] - 2025-02-04 + +### Added + +- Saving drawn shape on submit in `single shape` mode + () + +- An option to create tasks with consensus jobs + () + +- \[SDK\] The shapes output by auto-annotation functions are now checked + for compatibility with the function's and the task's label specs + () + +- A `threshold` parameter to UI detector runner + () + +### Changed + +- `DetectorFunctionSpec` will now raise `BadFunctionError` if it detects + any violations of the documented constraints on the labels + () + +### Fixed + +- Improved performance and memory utilization for quality reports in tasks with ellipses and masks + () + +- \[Compose\] An outdated version of Traefik is no longer used in + deployments with HTTPS enabled + () + + +## \[2.26.1\] - 2025-01-29 + +### Added + +- A button to copy a filename of the image into the clipboard + () + +### Changed + +- Changed location of events cache dir + () + +### Removed + +- \[Helm\] Removed `disableDistinctCachePerService` settings + () + +### Fixed + +- The backend now rejects invalid label types + () + +- \[Helm\] Impossible to download exported annotations + () + + +## \[2.26.0\] - 2025-01-27 + +### Added + +- Setting `TMP_FILE_OR_DIR_RETENTION_DAYS`, which defines maximum retention period + of a file or dir in temporary directory + () +- Cron job to remove outdated files and directories from CVAT tmp directory + () + +- Ability to set Django's secret key using an environment variable + () + +### Changed + +- Export cache cleaning moved to a separate cron job + () + +- Improved UX of quality management page: better table layout, file name search, ability to download table as `.csv` + () + +- Enhanced MIL tracker. Optimized memory usage. Now it is runnable on many frames, and applicable to drawn rectangles. + () + +- The UI only displays one version for the whole client component, + which is now aligned with the server version + () + +### Fixed + +- Fixed webhook worker not restarting after losing Redis connection + () + +- Fixed incorrect results being returned from lambda functions when all + detected shapes have labels that aren't mapped + () + +- Optimized importing from cloud storage + () + +- A job cannot be opened if to remove an image with the latest keyframe of a track + () + +- A track will be interpolated incorrectly if to delete an image containing the object keyframe + () + +- Error: Cannot read properties of undefined (reading 'startPoints') when dragging an object + () + +- Extra shortcuts enabled from brush tools on views where not necessary + () + +- \[Helm\] Fixed Nuclio dashboard crashes when running in a cluster + that doesn't use Docker + () + +- \[SDK\] `cvat_sdk.auto_annotation.functions.torchvision_detection` and + `torchvision_instance_segmentation` no longer declare meaningless "N/A" labels + () + +### Security + +- Protected tracker functions against deserializing untrusted input + () + + +## \[2.25.0\] - 2025-01-09 + +### Added + +- \[CLI\] Added commands for working with native functions + () + +- Ultralytics YOLO formats now support tracks + () + +### Changed + +- YOLOv8 formats renamed to Ultralytics YOLO formats + () + +- The `match_empty_frames` quality setting is changed to `empty_is_annotated`. + The updated option includes any empty frames in the final metrics instead of only + matching empty frames. This makes metrics such as Precision much more representative and useful. + () + +### Fixed + +- Changing rotation after export/import in Ultralytics YOLO Oriented Boxes format + () + +- Export to yolo formats if both Train and default dataset are present + () + +- Issue with deleting frames + () + + +## \[2.24.0\] - 2024-12-20 + +### Added + +- \[CLI\] Added new commands: `project create`, `project delete`, `project ls` + () + +- \[SDK\] You can now use `client.projects.remove_by_ids` to remove multiple + projects + () + +- Support for boolean parameters in annotations actions + () + +### Changed + +- Improved uniformity of validation frames distribution in honeypot tasks and + random honeypot rerolls + () + +- \[CLI\] Switched to a new subcommand hierarchy; now CLI subcommands + have the form `cvat-cli ` + () + +- \[CLI\] The output of the `task create`, `task create-from-backup` and + `project create` commands is now just the created resource ID, + making it machine-readable + () + +- /api/events can now be used to receive events from several sources + () + +### Deprecated + +- \[CLI\] All existing CLI commands of the form `cvat-cli ` + are now deprecated. Use `cvat-cli task ` instead + () + +### Removed + +- Automatic calculation of quality reports in tasks + () + +### Fixed + +- Uploading a skeleton template in configurator does not work + () + +- Installation of YOLOv7 on GPU + () + +- \[Server API\] Significantly improved performance of honeypot changes in tasks + () +- \[Server API\] `PATCH tasks/id/validation_layout` responses now include correct + `disabled_frames` and handle simultaneous updates of + `disabled_frames` and honeypot frames correctly + () + +- Fixed handling of tracks keyframes from deleted frames on export + () + +- Exporting datasets could start significantly later than expected, both for 1 + and several users in the same project/task/job () +- Scheduled RQ jobs could not be restarted due to incorrect RQ job status + updating and handling () + + +## \[2.23.1\] - 2024-12-09 + +### Changed + +- \[CLI\] Log messages are now printed on stderr rather than stdout + () + +### Fixed + +- Optimized memory consumption and reduced the number of database queries + when importing annotations to a task with a lot of jobs and images + () + +- Incorrect display of validation frames on the task quality management page + () + +- Player may navigate to removed frames when playing + () + +- User may navigate forward with a keyboard when a modal opened + () + +- fit:canvas event is not generated if to fit it from the controls sidebar + () + +- Color of 'Create object URL' button for a not saved on the server object + () + +- Failed request for a chunk inside a job after it was recently modified by updating `validation_layout` + () + +- Memory consumption during preparation of image chunks + () + +- Possible endless lock acquisition for chunk preparation job + () + +- Fixed issue: Cannot read properties of undefined (reading 'getUpdated') + () + + +## \[2.23.0\] - 2024-11-29 + +### Added + +- Support for direct .json file import in Datumaro format + () + +- \[SDK, CLI\] Added a `conf_threshold` parameter to + `cvat_sdk.auto_annotation.annotate_task`, which is passed as-is to the AA + function object via the context. The CLI equivalent is `auto-annotate + --conf-threshold`. This makes it easier to write and use AA functions that + support object filtering based on confidence levels + () + +- \[SDK\] Built-in auto-annotation functions now support object filtering by + confidence level + () + +- New events (create|update|delete):(membership|webhook) and (create|delete):invitation + () + +- \[SDK\] Added new auto-annotation helpers (`mask`, `polygon`, `encode_mask`) + to support AA functions that return masks or polygons + () + +- \[SDK\] Added a new built-in auto-annotation function, + `torchvision_instance_segmentation` + () + +- \[SDK, CLI\] Added a new auto-annotation parameter, `conv_mask_to_poly` + (`--conv-mask-to-poly` in the CLI) + () + +- A user may undo or redo changes, made by an annotations actions using general approach (e.g. Ctrl+Z, Ctrl+Y) + () + +- Basically, annotations actions now support any kinds of objects (shapes, tracks, tags) + () + +- A user may run annotations actions on a certain object (added corresponding object menu item) + () + +- A shortcut to open annotations actions modal for a currently selected object + () + +- A default role if IAM_TYPE='LDAP' and if the user is not a member of any group in 'DJANGO_AUTH_LDAP_GROUPS' () + +- The `POST /api/lambda/requests` endpoint now has a `conv_mask_to_poly` + parameter with the same semantics as the old `convMaskToPoly` parameter + () + +- \[SDK\] Model instances can now be pickled + () + +### Changed + +- Chunks are now prepared in a separate worker process + () + +- \[Helm\] Traefik sticky sessions for the backend service are disabled + () + +- Payload for events (create|update|delete):(shapes|tags|tracks) does not include frame and attributes anymore + () + +### Deprecated + +- The `convMaskToPoly` parameter of the `POST /api/lambda/requests` endpoint + is deprecated; use `conv_mask_to_poly` instead + () + +### Removed + +- It it no longer possible to run lambda functions on compressed images; + original images will always be used + () + +### Fixed + +- Export without images in Datumaro format should include image info + () + +- Inconsistent zOrder behavior on job open + () + +- Ground truth annotations can be shown in standard mode + () + +- Keybinds in UI allow drawing disabled shape types + () + +- Style issues on the Quality page when browser zoom is applied + () +- Flickering of masks in review mode, even when no conflicts are highlighted + () + +- Fixed security header duplication in HTTP responses from the backend + () + +- The error occurs when trying to copy/paste a mask on a video after opening the job + () + +- Attributes do not get copied when copy/paste a mask + () + + +## \[2.22.0\] - 2024-11-11 + +### Added + +- Feature to hide a mask during editing () + +- A quality setting to compare point groups without using bbox + () + +- A quality check option to consider empty frames matching + () + +### Changed + +- Reduced memory usage of the utils container + () + +### Removed + +- Removed unused business group + () + +### Fixed + +- Propagation creates copies on non-existing frames in a ground truth job + () + +- Exporting projects with tasks containing honeypots. Honeypots are no longer exported. + () + +- Error after creating GT job on Create job page with frame selection method `random_per_job` + () + +- Fixed issue 'Cannot read properties of undefined (reading 'push')' + () + +- Re-newed import/export request failed immediately if the previous failed + () + +- Fixed automatic zooming in attribute annotation mode for masks + () + +- Export dataset in CVAT format misses frames in tasks with non-default frame step + () + +- Incorrect progress representation on `Requests` page + () + + +## \[2.21.3\] - 2024-10-31 + +### Changed + +- CLI no longer prints the stack trace in case of HTTP errors + () + +### Removed + +- Dropped support for Python 3.8 since its EOL was on 2024-10-07 + () + +### Fixed + +- Requests page crush with `Cannot read property 'target' of undefined` error + () + +- Tags in ground truth job were displayed as `tag (GT)` + () + +- Tags in ground truth job couldn't be deleted via `x` button + () + +- Exception 'Canvas is busy' when change frame during drag/resize a track + () + +- A shape gets shifted if auto save triggered during dragging + () + + +## \[2.21.2\] - 2024-10-24 + +### Added + +- Access to /analytics can now be granted + () + +### Fixed + +- Expired sessions are now cleared from the database daily + () + +- Fixed export/import errors for tracks with duplicated shapes. + Fixed a bug which caused shape duplication on track import. + () + +- Fix Grafana container restart policy + () + +- Fixed some interface tooltips having 'undefined' shortcuts + () + +- Memory consumption during preparation of image chunks + () + +- Fixed a bug where an export RQ job being retried may break scheduling + of new jobs + () + +- UI now allows the user to start automatic annotation again + if the previous request fails + () + + +## \[2.21.1\] - 2024-10-18 + +### Added + +- Keyboard shortcuts for **brush**, **eraser**, **polygon** and **polygon remove** tools on masks drawing toolbox + () + +### Fixed + +- Ground truth tracks are displayed not only on GT frames in review mode + () + +- Incorrect navigation by keyframes when annotation job ends earlier than track in a ground truth job + () +- Tracks from a ground truth job displayed on wrong frames in review mode when frame step is not equal to 1 + () + +- Task creation with cloud storage data and GT_POOL validation mode + () + +- Incorrect quality reports and immediate feedback with non default start frame or frame step + () + +- av context closing issue when using AUTO thread_type + () + + +## \[2.21.0\] - 2024-10-10 + +### Added + +- New task mode: Honeypots (GT pool) + () +- New task creation options for quality control: Honeypots (GT pool), GT job + () +- New GT job frame selection method: `random_per_job`, + which guarantees each job will have GT overlap + () +- \[Server API\] POST `/jobs/`: new frame selection parameters, + which accept percentages, instead of absolute values + () +- \[Server API\] GET `/api/tasks/{id}/` got a new `validation_mode` field, + reflecting the current validation configuration (immutable) + () +- \[Server API\] POST `/api/tasks/{id}/data` got a new `validation_params` field, + which allows to enable `GT` and `GT_POOL` validation for a task on its creation + () + +- Added custom certificates documentation + () + +- Support for YOLOv8 Classification format + () + +- \[Server API\] An option to change real frames for honeypot frames in tasks with honeypots + () +- \[Server API\] New endpoints for validation configuration management in tasks and jobs + `/api/tasks/{id}/validation_layout`, `/api/jobs/{id}/validation_layout` + () + +- \[Helm\] Readiness and liveness probes + () + +### Changed + +- \[Server API\] POST `/jobs/` `.frames` field now expects relative frame numbers + instead of absolute (source data) ones + () + +- \[Server API\] Now chunks in tasks can be changed. + There are new API elements to check chunk relevancy, if they are cached: + `/api/tasks/{id}/data/meta` got a new field `chunks_updated_date`, + `/api/tasks/{id}/data/?type=chunk` got 2 new headers: `X-Updated-Date`, `X-Checksum` + () + +- Made the `PATCH` endpoints for projects, tasks, jobs and memberships check + the input more strictly + (): + + - unknown fields are rejected; + - updating a field now requires the same level of permissions regardless of + whether the new value is the same as the old value. + +- \[Server API\] Quality report computation is now allowed to regular users + () + +### Fixed + +- Invalid chunks for GT jobs when `?number` is used in the request and task frame step > 1 + () +- Invalid output of frames for specific GT frame requests with `api/jobs/{id}/data/?type=frame` + () + + +## \[2.20.0\] - 2024-10-01 + +### Added + +- A server setting to enable or disable storage of permanent media chunks on the server filesystem + () +- \[Server API\] `GET /api/jobs/{id}/data/?type=chunk&index=x` parameter combination. + The new `index` parameter allows to retrieve job chunks using 0-based index in each job, + instead of the `number` parameter, which used task chunk ids. + () + +### Changed + +- Job assignees will not receive frames from adjacent jobs in chunks + () + +### Deprecated + +- \[Server API\] `GET /api/jobs/{id}/data/?type=chunk&number=x` parameter combination + () + +### Removed + +- Removed the non-functional `task_subsets` parameter from the project create + and update endpoints + () + +### Fixed + +- Various memory leaks in video reading on the server + () + + +## \[2.19.1\] - 2024-09-26 + +### Security + +- Fixed a security issue that occurred in PATCH requests to projects|tasks|jobs|memberships + () + + +## \[2.19.0\] - 2024-09-20 + +### Added + +- Quality management tab on `quality control` allows to enabling/disabling GT frames + () + +### Changed + +- Moved quality control from `analytics` page to `quality control` page + () + +### Removed + +- Quality report no longer available in CVAT community version + () + +### Fixed + +- Fixing a problem when project export does not export skeleton tracks + () + +### Security + +- Fixed an XSS vulnerability in request-related endpoints + () + +- Fixed an XSS vulnerability in the quality report data endpoint + () + + +## \[2.18.0\] - 2024-09-10 + +### Added + +- New quality settings `Target metric`, `Target metric threshold`, `Max validations per job` + () + +- Ability to specify location when exporting datasets and backups using SDK + () + +- Shortcuts in user interface now may be customized depends on a user requirements + () + +- Added analytics events for function calls + () + +### Changed + +- `Mean annotation quality` card on quality page now displays a value depending on `Target metric` setting + () + +- When cancelling a request, a user is no longer required to have + permissions to perform the original action + () + +- Lambda function endpoints now return 500 instead of 404 + if a function's metadata is invalid + () + +- An unknown lambda function type is now treated as invalid metadata + and the function is no longer included in the list endpoint output + () + +### Removed + +- Legacy component to setup shortcuts to switch a label + () + +### Fixed + +- An issue that occurred when exporting the same dataset or backup twice in a row using SDK + () +- An issue that occurred when exporting a dataset or backup using SDK + when the default project or task location refers to cloud storage + () + +- Export crashed on skeleton track with missing shapes + () + +- One lambda function with invalid metadata will no longer + break function listing + () + +### Security + +- Fixed a missing authorization vulnerability in webhook delivery endpoints + () + + +## \[2.17.0\] - 2024-08-27 + +### Added + +- Added support for YOLOv8 formats + () + +- Last assignee update date in quality reports, new options in quality settings + () + +### Changed + +- User sessions now expire after two weeks of inactivity + () + +- A user changing their password will now invalidate all of their sessions + except for the current one + () + +### Deprecated + +- Client events `upload:annotations`, `lock:object`, `change:attribute`, `change:label` + () + +### Removed + +- Client event `restore:job` () + +- Removed the `/auth/login-with-token` page + () + +### Fixed + +- Go back button behavior on analytics page + () + +- Logging out of one session will no longer log the user out of all their + other sessions + () + +- Prevent export process from restarting when downloading a result file, + that resulted in downloading a file with new request ID + () +- Race condition occurred while handling parallel export requests + () +- Requests filtering using format and target filters + () + +- Sometimes it is not possible to switch workspace because active control broken after + trying to create a tag with a shortcut + () + + +## \[2.16.3\] - 2024-08-13 + +### Added + +- Labels mapper on UI now supports attributes for skeleton points + () + +- Segment Anything now supports bounding box input + () + +### Changed + +- Player navigation not blocked anymore if a frame is being loaded from the server + () + +- Accelerated implementation of IntelligentScissors from OpenCV + () + +### Fixed + +- Issue tool was not reset after creating new issue + () + +- Fixed issue with slices handling in `LazyList` which caused problems with exporting masks + in `CVAT for images 1.1` format. + () + + +## \[2.16.2\] - 2024-08-06 + +### Changed + +- Following the link in notification no longer reloads the page + () + +### Fixed + +- Copy/paste annotation guide with assets did not work, showing the message + **Asset is already related to another guide** + () + +- Undo can't be done when a shape is rotated + () + +- Exporting a skeleton track in a format defined for shapes raises error + `operands could not be broadcast together with shapes (X, ) (Y, )` + () + +- Delete label modal window does not have cancellation button + () + +- Export and export cache clean rq job retries' hangs + () + +- The automatic annotation process failed for tasks from cloud data + () + +- Request card was not disabled properly after downloading + () + +- Annotations in a ground truth jobs marked as GT annotations after modifying + () + +- API call to run automatic annotations fails on a model with attributes + when mapping not provided in the request + () + +- Fixed a label collision issue where labels with similar prefixes + and numeric suffixes could conflict, causing error on export. + () + + +## \[2.16.1\] - 2024-07-18 + +### Added + +- Datumaro format now supports skeletons + () + +### Changed + +- Quality analytics page will now report job assignees from quality reports + instead of current job assignees + () + +- When exporting projects in COCO format, images in different subsets are now stored in different subfolders + () + +- On task export, put images to folders depending on subset + () + +### Fixed + +- User interface crashed if there are active creating task requests on a project page + () + +- Permission error: organization owner cannot export dataset and backup + () + + +## \[2.16.0\] - 2024-07-15 + +### Added + +- Set of features to track background activities: importing/exporting datasets, annotations or backups, creating tasks. + Now you may find these processes on Requests page, it allows a user to understand current status of these activities + and enhances user experience, not losing progress when the browser tab is closed + () + +- User now may update a job state from the corresponding task page + () + +- The server will now record and report last assignee update time + () + +### Changed + +- "Finish the job" button on annotation view now only sets state to 'completed'. + The job stage keeps unchanged + () + +- Log files for individual backend processes are now stored in ephemeral + storage of each backend container rather than in the `cvat_logs` volume + () + +- Do not reset opacity level each time frame switched if there are masks on the frame + () + +### Removed + +- Renew the job button in annotation menu was removed + () + +### Fixed + +- A possible crash in quality computation for tasks with skeletons and normal labels + () + +- Quality report button and timestamp alignments on quality page + () + +- Fixed display of working time in Grafana management dashboard + () + +- Fixed unexpected deletion of log files of other processes that led to OSError: + \[Errno 116\] Stale file handle error on NFS volumes + () + +- Attribute values with ":" may be displayed incorrectly on canvas + () + +- Fixed broken server Docker image build + () + +- DOMException: Failed to execute 'atob' on 'Window': The string to be decoded is not correctly encoded + () + + +## \[2.15.0\] - 2024-07-02 + +### Added + +- `Propagate shapes` action to create copies of visible shapes on multiple frames forward or backward + () + +- \[Helm\] Ability to use an external ClickHouse instance + () + +### Changed + +- Improved performance for mask import and export + () + +### Fixed + +- Failing dataset export cleanup attempts for exports before #7864 + () + +- Exception 'this.el.node.getScreenCTM() is null' occurring in Firefox when + a user resizes window during skeleton dragging/resizing + () + +- Exception 'Edge's nodeFrom M or nodeTo N do not to refer to any node' + occurring when a user resizes window during skeleton dragging/resizing + () + +- Slightly broken layout when running attributed face detection model + () + +- Exception 'this.el.node.getScreenCTM() is null' when cancel drawing shape for any tracker + () + +- The switcher to block an active tool on annotation header is not highlighted properly + () + +- Points shape color wasn't changed on changing label + () + +- Incorrect counting of tracked shapes when computing analytics report + () + +- Ordering of `frame intersection` column on task quality page + () + +- The property "outside" not propagated correctly on skeleton elements + () + + +## \[2.14.4\] - 2024-06-20 + +### Added + +- Polyline editing may be finished using corresponding shortcut + () + +### Changed + +- Single shape annotation mode allows to modify/delete objects + () + +### Fixed + +- Invalid server cache cleanup for backups and events (after #7864) + () + +- Filters by created date, updated date do not work on different pages (e.g. list of tasks or jobs) + () + + +## \[2.14.3\] - 2024-06-13 + +### Changed + +- Increased server healthcheck timeout 5 -> 15 seconds + () + +### Fixed + +- Cannot read properties of null (reading 'draw') happens when use shortcut N in a task where first label has type "tag" + () + +- When use route `/auth/login-with-token/` without `next` query parameter + the page reloads infinitely + () + +- Fixed kvrocks port naming for istio + () + +- Exception: State cannot be updated during editing, need to finish current editing first + () + +### Security + +- Mitigated a CSRF vulnerability in backup and export-related endpoints + () + +- Fixed an SSRF vulnerability with custom cloud storage endpoints + () + + +## \[2.14.2\] - 2024-06-07 + +### Fixed + +- Queued jobs are not considered in deferring logic + () + +- Significant memory leak related to the frames, which did not memory after become unused + () + + +## \[2.14.1\] - 2024-06-05 + +### Added + +- Improved message of DatasetNotFoundError + () + +### Changed + +- Upgraded React and Antd dependencies, it leads to stylistic changes in the user interface + () + +- CVAT now stores users' working time in events of a dedicated type + () + +### Fixed + +- The 500 / "The result file does not exist in export cache" error + on dataset export request + () + +- Fix missing serviceName field in kvrocks (issue #7741) + () + +- UI crash on hovering conflict related to hidden objects + () + +- Login when the domain of a user email contains capital symbols and a user was created after being invited to an org + () + +- Exception **"Cannot set properties of undefined (setting 'serverID')"** occurs when attempting + to save a job after removing the first keyframe of a track () + +- Spent working time for a user may not be counted in analytics + () + +- A classifier model can not be used on annotation view (unknown object shape error) + () + +- Optimized memory usage by not keeping all downloaded images/part of images in memory while creating a manifest file + () +- Optimized the number of requests to CS providers by downloading only images from a specified range + (`use_cache==False`) () +- Task creation with random sorting and cloud data + () + + +## \[2.14.0\] - 2024-05-21 + +### Added + +- Added security headers enforcing strict `Referrer-Policy` for cross origins and disabling MIME type sniffing via `X-Content-Type-Options`. + () + +- \[Helm\] Ability to specify ServiceAccount for backend pods + () + +### Changed + +- Working time rounding to a minimal value of 1 hour is not applied to the annotation speed metric any more + () + +- Total annotation speed metric renamed to Average annotation speed + () + +- Ground truth jobs are not considered when computing analytics report for a task/project + () + +### Fixed + +- Fixed calculation of annotation speed metrics for analytics reports + () + +- \[Helm\] Prevented spurious 200 OK responses from API endpoints + before the backend is ready + () + +- Analytic reports incorrect count of objects for a skeleton track/shape + () + +- Analytic reports incorrect number of objects for a track (always less by 1) + () + +- REST API allowed to create several attributes with the same name within one label + () + +- Job's/task's status are not updated when job's state updated to completed and stage is already acceptance + () + +- Exception: Cannot read properties of undefined (reading 'onBlockUpdated') + () + +- One more found way to create an empty mask + () + +- Slice function may not work in Google Chrome < 110 + () + +- Selecting a skeleton by cursor does not work correctly when there are some hidden points + () + + +## \[2.13.0\] - 2024-05-09 + +### Added + +- Quality Report calculation will now also include annotation of type Tag. + () + +- Added feature to show tags of GT and manual job in separate row. Tags of GT job have '(GT)' in their name. + () + +### Changed + +- Analytics reports calculation may be initiated manually instead of automatic scheduling + () + +- Update the Nuclio version and related packages/libraries + () + +- Remove keyframe button is disabled when there is only one keyframe element + () + +### Removed + +- The `mask_rcnn` function has been removed because it was using python3.6. + In new version of Nuclio python3.6 is no longer supported. Nuclio officially recommends using python3.9. + Running `mask_rcnn` on python3.9 causes errors within the function and package conflicts. () + +### Fixed + +- Analytics report calculation fails with timeout because of redundant number of requests to ClickHouse + () + +- Incorrect duration of `change:frame` event + () + +- Infinite loading cloud storage update page when a lot of cloud storages are available for a user + () + +- Opening update CS page sends infinite requests when CS id does not exist + () + +- Uploading files with TUS immediately failed when one of the requests failed + () + +- Longer analytics report calculation because of inefficient requests to analytics db + () + +- Cannot read properties of undefined (reading 'addClass') + () + +- Fixed exception 'Could not read property length of undefined' when copy/paste a skeleton point + () + +- Task creation from a video file without keyframes allowing for random iteration + () + +- Cannot read property 'annotations' of null when uploading annotations into a job + () + +- Vertical polyline of two points is difficult to select + () + +- Tracked attribute values are lost when moving a task to a project + () + +### Security + +- Disable the nginx server signature by default to make it slightly harder for attackers to find known vulnerabilities. + () + + +## \[2.12.1\] - 2024-04-26 + +### Fixed + +- Formats with the custom `track_id` attribute should import `outside` track shapes properly + (e.g. `COCO`, `COCO Keypoints`, `Datumaro`, `PASCAL VOC`) + () + +- Inefficient resources fetching in admin panel leading to 504 Gateway Timeout + () + +- Optimized memory usage when retrieving annotations by disabling internal Django QuerySet caching + () + +- Annotations are not shown on the `0` frame sometimes + () + +- Extra requests in PolicyEnforcer when at least one policy is rejected, others are not checked + () + +- Project's `updated_date` was not updated after changing annotations in jobs + () + + +## \[2.12.0\] - 2024-04-15 + +### Added + +- Number of objects on the frame is shown on the right sidebar + () + +- Shortcut to switch "pinned" property (P) + () + +- Support for `.rar`, `.tar`, `.gz`, `.bz2`, `.cpio`, `.7z` archives + () + +### Changed + +- Updated links to the documentation website to point to the new domain, + `docs.cvat.ai` + () + +- Job and task `download_frames` now accepts custom extension for images + () + +### Fixed + +- Creating tasks with special characters in uploaded filename + () + +- `Find next frame with issues` ignored `hide resolved issues` setting + () + +- Objects menu is invisible for GT objects in GT job + () + +- Missing RegisterSerializerEx `email_verification_required` and `key` parameters now are included in the server schema + () + +- Standardize the alignment of empty-list components + () + +- Labels in WiderFace dataset example + () +- Export without images in Datumaro format - + no empty "media" and "point_cloud" fields should be present + () + +- Fixed the inability to rename label attributes after creating them. + () + +- When user starts editing a mask, it becomes smoother (not pixelated) + () + + +## \[2.11.3\] - 2024-04-02 + +### Added + +- Tooltips for long names on cards (projects, tasks, cloud storages, and models) + () + +### Removed + +- The `POST /api/tasks/{id}/data` endpoint no longer accepts several + parameters that didn't have any useful function: `size`, + `compressed_chunk_type`, `original_chunk_type` + () + +### Fixed + +- Duplicated notifications for automatic annotation + () + +- Made quality report update job scheduling more efficient + () + +- Incorrect file name usage when importing annotations from a cloud storage + () + +- Using single shape annotation mode with multiple labels + () + +- Part of sidebar not visible in attribute annotation mode when there are a lot of attribute values + () + +- Changed interpolation behavior in `annotation.py`, now correctly keep the last frame +- Insert last frame if it is key to the track, fixes data corruption when tracks crossing more than 1 jobs + () + +- Label constructor validation of empty label names + () + +- Incorrect alignment of empty job list component + () + +- Remove underlying pixels feature is not applied immediately + () + +- Corrected the formula for per-class accuracy in quality reports; + the old formula is now exposed as the `jaccard_index` key + () + +- Sending `/events` request from logged-out user () + +- Fixed accuracy being displayed incorrectly on the task analytics page + () + +- Fixed an invalid default overlap size being selected for video tasks + with small segments + () + +- Fixed redundant jobs being created for tasks with non-zero overlap + in certain cases + () + +- Accumulation of confusion matrix across all jobs in a task when creating a quality report + () + +- 90 deg-rotated video was added with "Prefer Zip Chunks" disabled + was warped, fixed using the static cropImage function. + () + + +## \[2.11.2\] - 2024-03-11 + +### Changed + +- Sped up resource updates when there are no matching webhooks + () + +### Fixed + +- Job and task `updated_date` are no longer bumped twice when updating + annotations + () + +- Sending `PATCH /jobs/{id}/data/meta` on each job save even if nothing changed in meta data + () +- Sending `GET /jobs/{id}/data/meta` twice on each job load + () + +- Made analytics report update job scheduling more efficient + () + +- Fixed being unable to connect to in-mem Redis + when the password includes URL-unsafe characters + () + +- Segment anything decoder is loaded anytime when CVAT is opened, but might be not required + () + + +## \[2.11.1\] - 2024-03-05 + +### Added + +- Single shape annotation mode allowing to easily annotate scenarios where a user + only needs to draw one object on one image () + +### Fixed + +- Fixed a problem with Korean/Chinese characters in attribute annotation mode + () + +- Fixed incorrect working time calculation in the case where an event + occurred during another event + () + +- Fixed working time not being calculated for the first event in each batch + sent from the UI + () + +- Submit button is enabled while creating a ground truth job + () + + +## \[2.11.0\] - 2024-02-23 + +### Added + +- Added `dataset:export` and `dataset:import` events that are logged when + the user initiates an export or import of a project, task or job + () + +### Changed + +- Now menus in the web interface are triggered by click, not by hover as before + () + +### Removed + +- Removed support for the TFRecord dataset format + () + +### Fixed + +- On quality page for a task, only the first page with jobs has quality report metrics + () + +- Side effects of data changes, such as the sending of webhooks, + are no longer triggered until after the changes have been committed + to the database + (, + ) + + +## \[2.10.3\] - 2024-02-09 + +### Changed + +- The "message" field of the payload of send:exception events + no longer includes a trailing linebreak + () + +- Annotation guide is opened automatically if not seen yet when the job is "new annotation" + () +- Annotation guide will be opened automatically if this is specified in a link `/tasks//jobs/?openGuide` + () + +- Reduced number of server requests, made by clients + () + +- Server exception rest_framework.exceptions.NotAuthenticated is not logged by analytics anymore + () + +### Fixed + +- Prevented zombie processes from accumulating in the Kvrocks container + () + +- Fix Redis exceptions crashing the `/api/server/health/` endpoint + () + +- Unhandled exception "Cannot read properties of null (reading 'plot')" + () + +- Unhandled exception "Cannot read properties of undefined (reading 'toLowerCase')" + () + + +## \[2.10.2\] - 2024-01-26 + +### Changed + +- Enhanced errors messaging for better perception by users + () + +### Fixed + +- Empty masks might be created with `polygon-minus` tool () +- Empty masks might be created as a result of removing underlying pixels () + +- Fixed excessive memory usage + when exporting a project with multiple video tasks + () + +- OpenCV tracker MIL works one frame behind + () + + +## \[2.10.1\] - 2024-01-18 + +### Changed + +- KeyDB used as data cache replaced by Kvrocks + () + +### Fixed + +- 504 Timeout error when exporting resources to cloud storage + () +- Enqueuing deferred jobs when their dependencies have been started -> cancelled -> restarted -> finished + () + +- UI failed when open context menu for a skeleton element on a frame with a conflict + () +- Issue can not be created for a skeleton element in review mode + () + + +## \[2.10.0\] - 2024-01-10 + +### Changed + +- When the `ORG_INVITATION_CONFIRM` setting is enabled, organization invitations for existing users are no + longer accepted automatically. Instead, the invitee can now review the invitation and choose to accept or decline it. + () + +- \[Compose, Helm\] Updated Clickhouse to version 23.11.* + () + +- Job queues are now stored in a dedicated Redis instance + () + +### Removed + +- PermissionDenied error thrown before OPA call in case if user is not a member of organization + () + +### Fixed + +- Can not input Chinese correctly in text attributes on objects sidebar + () + +- Restored Compose file compatibility with Docker Compose 2.17.0 and earlier + () + +- Attaching GCS and AWS S3 buckets with dots in name + () + +- Annotation actions are applied to the objects from a ground truth job + () +- Ground truth objects removed together with annotation objects when press "Remove annotations" in menu + () +- Frame search by a filter is affected by ground truth annotations + () + +- Creating duplicating annotations when nginx throws 504 timeout status (workaround) + () + +- `TIFF` images are saved as `JPEG` images with `.tif` extension in original chunks + () +- EXIF rotated TIFF images are handled incorrectly + () + +- RQ Scheduler launch, broken in PR 7245 + () + +- UI crashes if user highlights conflict related to annotations hidden by a filter + () +- Annotations conflicts are not highlighted properly on the first frame of a job + () + +- Error message `Edge's nodeFrom ${dataNodeFrom} or nodeTo ${dataNodeTo} do not to refer to any node` + when upload a file with some abscent skeleton nodes () +- Wrong context menu position in skeleton configurator (Firefox only) + () +- Fixed console error `(Error: attribute width: A negative value is not valid` + appearing when skeleton with all outside elements is created () + +- Updating cloud storage attached to CVAT using Azure connection string + () + + +## \[2.9.2\] - 2023-12-11 + +### Added + +- Introduced CVAT actions. Actions allow performing different + predefined scenarios on annotations automatically (e.g. shape converters) + () + +- The UI will now retry requests that were rejected due to rate limiting + () + +### Changed + +- Update nvidia/cuda image version from 11.7.0 to 11.7.1 in transt serverless function. + () + +- \[Helm\] Allow pre-release versions in kubernetes requirement to include AWS EKS versions () + +- GPU versions of serverless functions now use the `latest-gpu` Docker tag + rather than `latest` + () + +- \[Compose, Helm\] Downgraded KeyDB to 6.3.2 + () + +### Fixed + +- The GPU version of the YOLOv7 serverless function not actually using the GPU + () + +- It is now possible to create Ground Truth jobs containing all frames in the task + () +- Incorrect Ground Truth chunks saving + () + +- Reset source/target storage if related cloud storage has been deleted + () + +- Prevent possible cyclic dependencies when enqueuing a rq job when ONE_RUNNING_JOB_IN_QUEUE_PER_USER is used + () +- Enqueue deferred jobs when their dependencies are moved to the failed job registry due to AbandonedJobError + () + +- Reduce the number of requests to the server for task details + () + +- Shape settings **opacity** and **selected opacity** reset on each frame change + () + +- Server error in list quality settings API, when called in an org + () + +- Incorrect handling of the hidden points in skeletons in quality comparisons + () + +- \[Helm\] Fixed installing Traefik Middleware even if Traefik is disabled in the values () + +- Error code 500 when send `change:frame` event without `duration`. + () + +- Added workaround for corrupted cached chunks + (, ) + + +## \[2.9.1\] - 2023-11-23 + +This release has changes only in the Enterprise version. + + +## \[2.9.0\] - 2023-11-23 + +### Added + +- CVAT now supports serverless Nuclio functions that return skeleton annotations. + We've added a keypoint detector that supports skeletons for the following classes: + body, head, foot, and hands. Deployment command: `./deploy_cpu.sh pytorch/mmpose/hrnet32/nuclio/` + () + +- Implemented a feature that allows slicing one polygon/mask shape into two parts + () + +- Implemented a feature that allows joining several masks into a single one + () + +- \[Helm\] Introduced values that apply to all backend deployments/jobs + () + +### Changed + +- The "use cache" option on the server is now ignored when creating a + task with cloud storage data () + +- The Docker Compose file and Helm chart have been updated to enable Traefik + access logs by default and change the log format to JSON + () + +- \[Helm\] The PersistentVolumeClaim for the volume used to hold application + data is now retained after uninstall + () + +- \[Helm\] All backend-related deployments now + use `cvat-app` as the value for the `app` label + () + +- \[Helm\] The minimum compatible Kubernetes version + is now 1.19.0 () + +- \[Helm\] The CVAT hostname can now be configured with `ingress.hostname` option + () + +- \[Helm\] The `ingress.tls` configuration has been reworked. + () + +- \[Helm\] The Traefik subchart updated to 25.0.0 (appVersion v2.10.5) + () + +- \[Docker Compose\] Traefik updated to v2.10.\* + () + +### Removed + +- Support for V1 cloudstorages/id/content endpoint + () + +- \[Helm\] `ingress.hosts` has been removed, use `ingress.hostname` instead. + () + +### Fixed + +- Fixed a data race condition during GT job creation + () + +- Resolved an issue where the job state could not be changed + multiple times without reloading the annotation view + () + +- Corrected an issue where compressed chunks did not + utilize the Exif rotation tag + () + +- Minor styling issues on empty models page + () + +- Fixed minor issue when brush marker is appended to a final mask + () + + +## \[2.8.2\] - 2023-11-06 + +### Fixed + +- OpenCV runtime initialization + () + + +## \[2.8.1\] - 2023-11-03 + +### Added + +- Support for default bucket prefix + () +- Search for cloud storage and share files + () + +- Ability to limit one user to one task at a time + () + +- Support for using an external database in a Docker Compose-based deployment + () + +### Changed + +- Migrated to rq 1.15.1 + () + +- Compressed sequental `change:frame` events into one + () + +- Create a local session for AWS S3 client instead of using the default global one + () + +- Improved performance of chunk preparation when creating tasks + () + +### Fixed + +- Race condition in a task data upload request, which may lead to problems with task creation in some specific cases, + such as multiple identical data requests at the same time + () + +- Bug with viewing dependent RQ jobs for downloading resources from + cloud storage when file path contains sub-directories. + This is relevant for admins that can view detailed information about RQ queues. + () + +- OpenCV.js memory leak with TrackerMIL + () + +- Can't deploy detectron serverless function + () + +- A mask becomes visible even if hidden after changing opacity level + () + +- There is no switcher to personal workspace if an organization request failed + () + + + +## \[2.8.0\] - 2023-10-23 + +### Added + +- A new feature allowing users to invite others to the organization via email. + () + +- \[SDK\] In the SDK, a parameter has been introduced to `TaskDataset` + which enables the option to disable annotation loading + () + +- A test has been incorporated for retrieving bucket content in + cases where the bucket includes manually created directories. + () + +### Changed + +- The maximum length of the secret access key has been + increased to 64 characters. + () + +- The client will no longer load all organizations upon start + () + +- The default value for Zookeeper from the + Clickhouse subchart has been set to disabled. + () + +### Removed + +- The endpoints `/api/projects`, `/api/tasks`, and `/api/jobs` + will no longer return information regarding the count of labels. + This information was complicating SQL queries, + making them hard to optimize. + Instead, use `/api/labels?task_id=tid` or `/api/labels?project_id=pid`. + () + +### Fixed + +- Issues causing potential double-sized file writes during task + data uploading have been addressed. + () + +- Issues encountered when retrieving CS content from GCS + buckets containing manually created directories have been resolved. + () + +- \[SDK\] In the SDK, `cvat_sdk.auto_annotation.annotate_task` + has been optimized to avoid unnecessary fetching of + existing annotations. + () + +- The project/task/job update time is now correctly + modified upon label updates. + () + + + +## \[2.7.6\] - 2023-10-13 + +### Changed + +- Enabled nginx proxy buffering + () + +- Helm: set memory request for keydb + () + +- Supervisord (): + - added `autorestart=true` option for all workers + - unified program names to use dashes as delimiter instead of mixed '\_' and '-' + - minor improvements to supervisor configurations + +### Removed + +- Removed gitter link from about modal + () + +### Fixed + +- Persist image filters across jobs + () + +- Splitting skeleton tracks on jobs + () + +- Uploading skeleton tracks in COCO Keypoints format + () + +- Fixed Siammask tracker error on grayscale images + () + +- Fixed memory leak on client side when event listener was not removed together with its context + () + +- Fixed crash related to issue tries to mount to not existing parent + () + +- Added 'notranslate' markers to avoid issues caused by extension translators + () + +- Getting CS content when S3 bucket contains manually created directories + () + +- Optimized huge memory consumption when working with masks in the interface + () + +### Security + +- Security upgrade opencv-python-headless from 4.5.5.62 to 4.8.1.78 + () + +- Added X-Frame-Options: deny + () + + + +## \[2.7.5\] - 2023-10-09 + +### Added + +- Temporary workaround to fix corrupted zip file + () + + + +## \[2.7.4\] - 2023-10-06 + +### Added + +- The latest comment displayed in issues sidebar () + +### Fixed + +- It was not possible to copy issue comment from issue dialog () + +### Security + +- Update Grafana from 9.3.6 to 10.1.2 + +## \[2.7.3\] - 2023-10-02 + +### Added + +- New , form-based Issue templates for Github repository + +### Removed + +- Functionality for synchronizing a task with a Git repository + () + +### Fixed + +- PCD files with nan values could not be opened on 3D workspace + () +- Fixed direct navigation to neighbour chunk on 3D workspace + () +- Intencity level from .bin lidar data ignored when converting .bin -> .pcd + () +- Incorrectly determined video frame count when the video contains an MP4 edit list + () +- Internal server error when retrieving data from CS and cache=True + () + +### Security + +- Security upgrade Pillow from 9.3.0 to 10.0.1 + () +- Security update cryptography from 41.0.3 to 41.0.4 + () + +## \[2.7.2\] - 2023-09-25 + +### Changed + +- Do not reload annotation view when renew the job or update job state () +- Now images from cloud buckets are loaded in parallel when preparing a chunk () + +### Fixed + +- Downloading additional data from cloud storage if use_cache=true and job_file_mapping are specified + () +- Leaving an organization () +- Order of images in annotation file when dumping project in CVAT format () +- Validation on Cloud Storage form / error message on create task form () + +## \[2.7.1\] - 2023-09-15 + +### Fixed + +- Include cloud storage manifest file to selected files if manifest was used as data source () +- Keep sequence of files when directories were specified in server_files () + +## \[2.7.0\] - 2023-09-10 + +### Added + +- Admin actions for easy activation/deactivation of users () + +### Fixed + +- Invalid input validation in for `cloud_storage_id` () +- Incorrect task progress report for 3rdparty users () + +### Security + +- Security upgrade gitpython from 3.1.33 to 3.1.35 () +- Security upgrade numpy from 1.22.0 to 1.22.4 () + +## \[2.6.2\] - 2023-09-06 + +### Added + +- Gamma correction filter () +- Introduced the feature to hide or show objects in review mode () + +### Changed + +- \[Helm\] Database migrations are now executed as a separate job, + rather than in the server pod, to mitigate the risk of data + corruption when using multiple server replicas + () +- Clicking multiple times on icons in the left + sidebar now toggles the corresponding popovers open and closed + () +- Transitioned to using KeyDB with FLASH for data + chunk caching, replacing diskcache () + +### Removed + +- Removed outdated use of hostnames when accessing Git, OpenCV, or analytics via the UI () +- Removed the Feedback/Share component () + +### Fixed + +- Resolved the issue of the canvas zooming while scrolling + through the comments list in an issue () +- Addressed the bug that allowed for multiple issue + creations upon initial submission () +- Fixed the issue of running deep learning models on + non-JPEG compressed TIFF images () +- Adjusted padding on the tasks, projects, and models pages () +- Corrected hotkey handlers to avoid overriding default behavior when modal windows are open + () +- Resolved the need to move the mouse to activate + brush or eraser effects; a single click is now sufficient () +- Fixed a memory leak issue in the logging system () +- Addressed a race condition that occurred during the initial creation of `secret_key.py` + () +- Eliminated duplicate log entries generated by the CVAT server + () + +## \[2.6.1\] - 2023-08-25 + +### Added + +- More information about task progress on tasks page () +- Prefetching next chunk when user navigates by frames manually () + +### Changed + +- Bumped nuclio version to 1.11.24 and removed `/tmp` mounting in the nuclio container to adhere the update. +- Response code for empty cloud storage preview 204 -> 404 () +- Organization now opened immediately after it is created () +- More responsive automatic annotation progress bar () +- Improved message when invite more users to an organization () + +### Fixed + +- Exporting project when its tasks has not data () +- Removing job assignee () +- UI fail when select a mask or a skeleton with center-aligned text () +- Fixed switching from organization to sandbox while getting a resource () +- You do not have permissions when user is cancelling automatic annotation () +- Automatic annotation progress bar is invisible if the app initialized on the task page + () +- Extra status check requests for automatic annotation () +- \[SDK\]: `FileExistsError` exception raised on Windows when a dataset is loaded from cache + () + +### Security + +- Remote Code Execution (RCE) [SNYK-PYTHON-GITPYTHON-5840584](https://snyk.io/vuln/SNYK-PYTHON-GITPYTHON-5840584) + +## \[2.6.0\] - 2023-08-11 + +### Added + +- \[SDK\] Introduced the `DeferredTqdmProgressReporter` class, + which avoids the glitchy output seen with the `TqdmProgressReporter` under certain circumstances + () +- \[SDK, CLI\] Added the `cvat_sdk.auto_annotation` + module, providing functionality to automatically annotate tasks + by executing a user-provided function on the local machine. + A corresponding CLI command (`auto-annotate`) is also available. + Some predefined functions using torchvision are also available. + (, + ) +- Included an indication for cached frames in the interface + () + +### Changed + +- Raised the default guide assets limitations to 30 assets, + with a maximum size of 10MB each + () +- \[SDK\] Custom `ProgressReporter` implementations should now override `start2` instead of `start` + The old implementation is still supported. + () +- Improved memory optimization and code in the decoding module () + +### Removed + +- Removed the YOLOv5 serverless function + () + +### Fixed + +- Corrected an issue where the prebuilt FFmpeg bundled in PyAV + was being used instead of the custom build. +- Fixed the filename for labels in the CamVid format () + +## \[2.5.2\] - 2023-07-27 + +### Added + +- We've added support for multi-line text attributes () +- You can now set a default attribute value for SELECT, RADIO types on UI + () +- \[SDK\] `cvat_sdk.datasets`, is now available, providing a framework-agnostic alternative to `cvat_sdk.pytorch` + () +- We've introduced analytics for Jobs, Tasks, and Project () + +### Changed + +- \[Helm\] In Helm, we've added a configurable default storage option to the chart () + +### Removed + +- \[Helm\] In Helm, we've eliminated the obligatory use of hardcoded traefik ingress () + +### Fixed + +- Fixed an issue with calculating the number of objects on the annotation view when frames are deleted + () +- \[SDK\] In SDK, we've fixed the issue with creating attributes with blank default values + () +- \[SDK\] We've corrected a problem in SDK where it was altering input data in models () +- Fixed exporting of hash for shapes and tags in a specific corner case () +- Resolved the issue where 3D jobs couldn't be opened in validation mode () +- Fixed SAM plugin (403 code for workers in organizations) () +- Fixed the issue where initial frame from query parameter was not opening specific frame in a job + () +- Corrected the issue with the removal of the first keyframe () +- Fixed the display of project previews on small screens and updated stylelint & rules () +- Implemented server-side validation for attribute specifications + () +- \[API\] Fixed API issue related to file downloading failures for filenames with special characters () +- \[Helm\] In Helm, we've resolved an issue with multiple caches + in the same RWX volume, which was preventing db migration from starting () + +## \[2.5.1\] - 2023-07-19 + +### Fixed + +- Memory leak related to unclosed av container () + +## \[2.5.0] - 2023-07-05 + +### Added + +- Now CVAT supports project/task markdown description with additional assets + (png, jpeg, gif, webp images and pdf files) () +- Ground Truth jobs and quality analytics for tasks () + +### Fixed + +- The problem with manifest file in tasks restored from backup () +- The problem with task mode in a task restored from backup () +- Visible 'To background' button in review mode () +- Added missed auto_add argument to Issue model () +- \[API\] Performance of several API endpoints () +- \[API\] Invalid schema for the owner field in several endpoints () +- Some internal errors occurring during lambda function invocations + could be mistakenly reported as invalid requests + () +- \[SDK\] Loading tasks that have been cached with the PyTorch adapter + () +- The problem with importing annotations if dataset has extra dots in filenames + () + +### Security + +- More comprehensive SSRF mitigations were implemented. + Previously, on task creation it was prohibited to specify remote data URLs + with hosts that resolved to IP addresses in the private ranges. + Now, redirects to such URLs are also prohibited. + In addition, this restriction is now also applied to webhook URLs. + System administrators can allow or deny custom IP address ranges + with the `SMOKESCREEN_OPTS` environment variable. + (). + +## \[2.4.9] - 2023-06-22 + +### Fixed + +- Error related to calling serverless functions on some image formats () + +## \[2.4.8] - 2023-06-22 + +### Fixed + +- Getting original chunks for items in specific cases () + +## \[2.4.7] - 2023-06-16 + +### Added + +- \[API\] API Now supports the creation and removal of Ground Truth jobs. () +- \[API\] We've introduced task quality estimation endpoints. () +- \[CLI\] An option to select the organization. () + +### Fixed + +- Issues with running serverless models for EXIF-rotated images. () +- File uploading issues when using https configuration. () +- Dataset export error with `outside` property of tracks. () +- Broken logging in the TransT serverless function. () + +## \[2.4.6] - 2023-06-09 + +### Added + +- \[Server API\] An option to supply custom file ordering for task data uploads () +- New option `semi-auto` is available as annotations source () + +### Changed + +- Allowed to use dataset manifest for the `predefined` sorting method for task data () + +### Changed + +- Replaced Apache mod_wsgi with Uvicorn ASGI server for backend use() + +### Fixed + +- Incorrect location of temporary file during job annotation import.() +- Deletion of uploaded file along with annotations/backups when an RQ job + has been initiated, but no subsequent status check requests have been made.() +- Deletion of uploaded files, including annotations and backups, + after they have been uploaded to the server using the TUS protocol but before an RQ job has been initiated. () +- Simultaneous creation of tasks or projects with identical names from backups by multiple users.() +- \[API\] The `predefined` sorting method for task data uploads () +- Allowed slashes in export filenames. () + +## \[2.4.5] - 2023-06-02 + +### Added + +- Integrated support for sharepoint and cloud storage files, along with + directories to be omitted during task creation (server) () +- Enabled task creation with directories from cloud storage or sharepoint () +- Enhanced task creation to support any data type supported by the server + by default, from cloud storage without the necessity for the `use_cache` option () +- Added capability for task creation with data from cloud storage without the `use_cache` option () + +### Changed + +- User can now access resource links from any organization or sandbox, granted it's available to them () +- Cloud storage manifest files have been made optional () +- Updated Django to the 4.2.x version () +- Renamed certain Nuclio functions to adhere to a common naming convention. For instance, + `onnx-yolov7` -> `onnx-wongkinyiu-yolov7`, `ultralytics-yolov5` -> `pth-ultralytics-yolov5` + () + +### Deprecated + +- Deprecated the endpoint `/cloudstorages/{id}/content` () + +### Fixed + +- Fixed the issue of skeletons dumping on created tasks/projects () +- Resolved an issue related to saving annotations for skeleton tracks () + +## \[2.4.4] - 2023-05-18 + +### Added + +- Introduced a new configuration option for controlling the invocation of Nuclio functions. + () + +### Changed + +- Relocated SAM masks decoder to frontend operation. + () +- Switched `person-reidentification-retail-0300` and `faster_rcnn_inception_v2_coco` Nuclio functions + with `person-reidentification-retail-0277` and `faster_rcnn_inception_resnet_v2_atrous_coco` respectively. + () +- Upgraded OpenVINO-based Nuclio functions to utilize the OpenVINO 2022.3 runtime. + () + +### Fixed + +- Resolved issues with tracking multiple objects (30 and more) using the TransT tracker. + () +- Addressed azure.core.exceptions.ResourceExistsError: The specified blob already exists. + () +- Corrected image scaling issues when transitioning between images of different resolutions. + () +- Fixed inaccurate reporting of completed job counts. + () +- Allowed OpenVINO-based Nuclio functions to be deployed to Kubernetes. + () +- Improved skeleton size checks after drawing. + () +- Fixed HRNet CPU serverless function. + () +- Prevented sending of empty list of events. + () + +## \[2.4.3] - 2023-04-24 + +### Changed + +- Docker images no longer include Ubuntu package sources or FFmpeg/OpenH264 sources + () +- TUS chunk size changed from 100 MB to 2 MB + () + +## \[2.4.2] - 2023-04-14 + +### Added + +- Support for Azure Blob Storage connection string authentication() +- Segment Anything interactor for CPU/GPU () + +### Changed + +- The capability to transfer a task from one project to another project has been disabled () +- The bounding rectangle in the skeleton annotation is visible solely when the skeleton is active () +- Base backend image upgraded from ubuntu:20.04 to ubuntu:22.04 () + +### Deprecated + +- TDB + +### Removed + +- Cloud storage `unique_together` limitation () +- Support for redundant request media types in the API + () +- Static URLs and direct SDK support for the tus chunk endpoints. + Clients must use the `Location` header from the response to the `Upload-Length` request, + as per the tus creation protocol + () + +### Fixed + +- An invalid project/org handling in webhooks () +- Warning `key` is undefined on project page () +- An invalid mask detected when performing automatic annotation on a task () +- The 'Reset zoom' option now retains the user's preferences upon reloading CVAT () +- Cloud storage content listing when the manifest name contains special characters + () +- Width and height in CVAT dataset format mask annotations () +- Empty list of export formats for a project without tasks () +- Downgraded NumPy used by HRNet because `np.int` is no longer available () +- Empty previews responsive to page resize () +- Nuclio function invocations when deployed via the Helm chart + () +- Export of a job from a task with multiple jobs () +- Points missing when exporting tracked skeleton () +- Escaping in the `filter` parameter in generated URLs + () +- Rotation property lost during saving a mutable attribute () +- Optimized /api/jobs request () +- Server micro version support check in SDK/CLI () +- \[SDK\] Compatibility with upcoming urllib 2.1.0 + () +- Fix TUS file uploading if multiple apache processes are used () +- The issue related to webhook events not being sent has been resolved () + +### Security + +- Updated Redis (in the Compose file) to 7.0.x, and redis-py to 4.5.4 + () + +## \[2.4.1] - 2023-04-05 + +### Fixed + +- Optimized annotation fetching up to 10 times () +- Incorrect calculation of working time in analytics () + +## \[2.4.0] - 2023-03-16 + +### Added + +- \[SDK\] An arg to wait for data processing in the task data uploading function + () +- Filename pattern to simplify uploading cloud storage data for a task (, ) +- \[SDK\] Configuration setting to change the dataset cache directory + () +- \[SDK\] Class to represent a project as a PyTorch dataset + () +- Grid view and multiple context images supported () +- Interpolation is now supported for 3D cuboids. +- Tracks can be exported/imported to/from Datumaro and Sly Pointcloud formats () +- Support for custom file to job splits in tasks (server API & SDK only) + () +- \[SDK\] A PyTorch adapter setting to disable cache updates + () +- YOLO v7 serverless feature added using ONNX backend () +- Cypress test for social account authentication () +- Dummy github and google authentication servers () +- \[Server API\] Simple filters for object collection endpoints + () +- Analytics based on Clickhouse, Vector and Grafana instead of the ELK stack () +- \[SDK\] High-level API for working with organizations + () +- Use correct service name in LDAP authentication documentation () + +### Changed + +- The Docker Compose files now use the Compose Specification version + of the format. This version is supported by Docker Compose 1.27.0+ + (). +- \[SDK\] The `resource_type` args now have the default value of `local` in task creation functions. + The corresponding arguments are keyword-only now. + () +- \[Server API\] Added missing pagination or pagination parameters in + `/jobs/{id}/commits`, `/organizations` + () +- Windows Installation Instructions adjusted to work around +- The contour detection function for semantic segmentation () +- Delete newline character when generating a webhook signature () +- DL models UI () +- \[Server API\], \[SDK\] Arbitrary-sized collections in endpoints: + `/api/projects/{id}.tasks`, `/api/tasks/{id}.segments`, `/api/jobs/{id}.issues`, + `/api/issues/{id}.comments`, `/api/projects | tasks | jobs/{id}.labels` + () +- Hide analytics link from non-admin users () +- Hide notifications on login/logout/register () +- CVAT and CVAT SDK now use a custom `User-Agent` header in HTTP requests + () + +### Deprecated + +- TBD + +### Removed + +- \[Server API\] Endpoints with collections are removed in favor of their full variants + `/project/{id}/tasks`, `/tasks/{id}/jobs`, `/jobs/{id}/issues`, `/issues/{id}/comments`. + Corresponding fields are added or changed to provide a link to the child collection + in `/projects/{id}`, `/tasks/{id}`, `/jobs/{id}`, `/issues/{id}` + () +- Limit on the maximum number of manifest files that can be added for cloud storage () + +### Fixed + +- Helm: Empty password for Redis () +- Resolved HRNet serverless function runtime error on images with an alpha channel () +- Addressed ignored preview & chunk cache settings () +- Fixed exporting annotations to Azure container () +- Corrected the type of the credentials parameter of `make_client` in the Python SDK +- Reduced noisy information in ortho views for 3D canvas () +- Cleared disk space after project removal (, ) +- Locked submit button when file is not selected during dataset import () +- \[Server API\]Various errors in the generated schema () +- Resolved browser freezing when requesting a job with NaN id () +- Fixed SiamMask and TransT serverless functions () +- Addressed creation of a project or task with the same labels () +- \[Server API\] Fixed ability to rename label to an existing name () +- Resolved issue of resetting attributes when moving a task to a project () +- Fixed error in dataset export when parsing skeleton sublabels containing spaces () +- Added missing `CVAT_BASE_URL` in docker-compose.yml () +- Create cloud storage button size and models pagination () + +### Security + +- Fixed vulnerability with social authentication () + +## \[2.3.0] - 2022-12-22 + +### Added + +- SDK section in documentation () +- Option to enable or disable host certificate checking in CLI () +- REST API tests with skeletons () +- Host schema auto-detection in SDK () +- Server compatibility checks in SDK () +- Objects sorting option in the sidebar, by z-order. Additional visualization when sorting is applied + () +- Added YOLOv5 serverless function with NVIDIA GPU support () +- Mask tools now supported (brush, eraser, polygon-plus, + polygon-minus, returning masks from online detectors & interactors) + () +- Added Webhooks () +- Authentication with social accounts: Google & GitHub + (, + , + ) +- REST API tests for exporting job datasets & annotations and validating their structure () +- Backward propagation on UI () +- Keyboard shortcut to delete a frame (Alt + Del) () +- PyTorch dataset adapter layer in the SDK + () +- Method for debugging the server deployed with Docker () + +### Changed + +- `api/docs`, `api/swagger`, `api/schema`, `server/about` endpoints now allow unauthorized access + (, ) +- 3D canvas now can be dragged in IDLE mode () +- Datumaro version is upgraded to 0.3 (dev) () +- Allowed trailing slashes in the SDK host address () +- Adjusted initial camera position, enabled 'Reset zoom' option for 3D canvas () +- Enabled authentication via email () +- Unified error handling with the cloud storage () +- In the SDK, functions taking paths as strings now also accept path-like objects + () + +### Removed + +- The `--https` option of CLI () + +### Fixed + +- Significantly optimized access to DB for api/jobs, api/tasks, and api/projects. +- Removed a possibly duplicated encodeURI() calls in `server-proxy.ts` to prevent doubly encoding + non-ascii paths while adding files from "Connected file share" (issue #4428) +- Removed unnecessary volumes defined in docker-compose.serverless.yml + () +- Added support for Image files that use the PIL.Image.mode 'I;16' +- Project import/export with skeletons (, + ) +- Shape color is not changed on canvas after changing a label () +- Unstable e2e restore tests () +- IOG and f-BRS serverless function () +- Invisible label item in label constructor when label color background is white, + or close to it () +- Fixed cvat-core ESlint problems () +- Fixed task creation with non-local files via the SDK/CLI + () +- HRNET serverless function () +- Invalid export of segmentation masks when the `background` label gets nonzero id () +- A trailing slash in hostname doesn't allow SDK to send some requests + () +- Double modal export/backup a task/project () +- Fixed bug of computing Job's unsolved/resolved issues numbers () +- Dataset export for job () +- Angle is not propagated when use `propagate` feature () +- Could not fetch task in a corner case () +- Restoring CVAT in case of React-rendering fail () +- Deleted frames become restored if a user deletes frames from another job of the same task + () +- Wrong issue position when create a quick issue on a rotated shape () +- Extra rerenders of different pages with each click () +- Skeleton points exported out of order in the COCO Keypoints format + () +- PASCAL VOC 1.1 can't import dataset () +- Changing an object causes current z layer to be set to the maximum () +- Job assignee can not resolve an issue () +- Create manifest with cvat/server docker container command () +- Cannot assign a resource to a user who has an organization () +- Logs and annotations are not saved when logout from a job page () +- Added "type" field for all the labels, allows to reduce number of controls on annotation view () +- Occluded not applied on canvas instantly for a skeleton elements () +- Oriented bounding boxes broken with COCO format ss() +- Can't dump annotations with objects type is track from several jobs () +- Fixed upload resumption in production environments + () +- Fixed job exporting () +- Visibility and ignored information fail to be loaded (MOT dataset format) () +- Added force logout on CVAT app start if token is missing () +- Drawing issues on 3D canvas () +- Missed token with using social account authentication () +- Redundant writing of skeleton annotations (CVAT for images) () +- The same object on 3D scene or `null` selected each click (PERFORMANCE) () +- An exception when run export for an empty task () +- Fixed FBRS serverless function runtime error on images with alpha channel () +- Attaching manifest with custom name () +- Uploading non-zip annotation files () +- Loss of rotation in CVAT format () +- A permission problem with interactive model launches for workers in orgs () +- Fix chart not being upgradable () +- Broken helm chart - if using custom release name () +- Missing source tag in project annotations () +- Creating a task with a Git repository via the SDK + () +- Queries via the low-level API using the `multipart/form-data` Content-Type with string fields + () +- Skeletons cannot be added to a task or project () + +### Security + +- `Project.import_dataset` not waiting for completion correctly + () + +## \[2.2.0] - 2022-09-12 + +### Added + +- Added ability to delete frames from a job based on () +- Support of attributes returned by serverless functions based on () +- Project/task backups uploading via chunk uploads +- Fixed UX bug when jobs pagination is reset after changing a job +- Progressbars in CLI for file uploading and downloading +- `utils/cli` changed to `cvat-cli` package +- Support custom file name for backup +- Possibility to display tags on frame +- Support source and target storages (server part) +- Tests for import/export annotation, dataset, backup from/to cloud storage +- Added Python SDK package (`cvat-sdk`) () +- Previews for jobs +- Documentation for LDAP authentication () +- OpenCV.js caching and autoload () +- Publishing dev version of CVAT docker images () +- Support of Human Pose Estimation, Facial Landmarks (and similar) use-cases, new shape type: +- Skeleton (), () +- Added helm chart support for serverless functions and analytics () +- Added confirmation when remove a track () +- [COCO Keypoints](https://cocodataset.org/#keypoints-2020) format support (, + ) +- Support for Oracle OCI Buckets () +- `cvat-sdk` and `cvat-cli` packages on PyPI () +- UI part for source and target storages () +- Backup import/export modals () +- Annotations import modal () + +### Changed + +- Bumped nuclio version to 1.8.14 +- Simplified running REST API tests. Extended CI-nightly workflow +- REST API tests are partially moved to Python SDK (`users`, `projects`, `tasks`, `issues`) +- cvat-ui: Improve UI/UX on label, create task and create project forms () +- Removed link to OpenVINO documentation () +- Clarified meaning of chunking for videos + +### Fixed + +- Task creation progressbar bug +- Removed Python dependency `open3d` which brought different issues to the building process +- Analytics not accessible when https is enabled +- Dataset import in an organization +- Updated minimist npm package to v1.2.6 +- Request Status Code 500 "StopIteration" when exporting dataset +- Generated OpenAPI schema for several endpoints +- Annotation window might have top offset if try to move a locked object +- Image search in cloud storage () +- Reset password functionality () +- Creating task with cloud storage data () +- Show empty tasks () +- Fixed project filtration () +- Maximum callstack exceed when create task with 100000+ files from cloud storage () +- Fixed invocation of serverless functions () +- Removing label attributes () +- Notification with a required manifest file () + +## \[2.1.0] - 2022-04-08 + +### Added + +- Task annotations importing via chunk uploads () +- Advanced filtration and sorting for a list of tasks/projects/cloudstorages () +- Project dataset importing via chunk uploads () +- Support paginated list for job commits () + +### Changed + +- Added missing geos dependency into Dockerfile () +- Improved helm chart readme () +- Added helm chart support for CVAT 2.X and made ingress compatible with Kubernetes >=1.22 () + +### Fixed + +- Permission error occurred when accessing the JobCommits () +- job assignee can remove or update any issue created by the task owner () +- Bug: Incorrect point deletion with keyboard shortcut () +- some AI Tools were not sending responses properly () +- Unable to upload annotations () +- Fix build dependencies for Siammask () +- Bug: Exif orientation information handled incorrectly () +- Fixed build of retinanet function image () +- Dataset import for Datumaro, KITTI and VGGFace2 formats () +- Bug: Import dataset of Imagenet format fail () + +## \[2.0.0] - 2022-03-04 + +### Added + +- Handle attributes coming from nuclio detectors () +- Add additional environment variables for Nuclio configuration () +- Add KITTI segmentation and detection format () +- Add LFW format () +- Add Cityscapes format () +- Add Open Images V6 format () +- Rotated bounding boxes () +- Player option: Smooth image when zoom-in, enabled by default () +- Google Cloud Storage support in UI () +- Add project tasks pagination () +- Add remove issue button () +- Data sorting option () +- Options to change font size & position of text labels on the canvas () +- Add "tag" return type for automatic annotation in Nuclio () +- Helm chart: Make user-data-permission-fix optional () +- Advanced identity access management system, using open policy agent () +- Organizations to create "shared space" for different groups of users () +- Dataset importing to a project () +- User is able to customize information that text labels show () +- Support for uploading manifest with any name () +- Added information about OpenVINO toolkit to login page () +- Support for working with ellipses () +- Add several flags to task creation CLI () +- Add YOLOv5 serverless function for automatic annotation () +- Add possibility to change git repository and git export format from already created task () +- Basic page with jobs list, basic filtration to this list () +- Added OpenCV.js TrackerMIL as tracking tool () +- Ability to continue working from the latest frame where an annotator was before () +- `GET /api/jobs//commits` was implemented () +- Advanced filtration and sorting for a list of jobs () + +### Changed + +- Users don't have access to a task object anymore if they are assigned only on some jobs of the task () +- Different resources (tasks, projects) are not visible anymore for all CVAT instance users by default () +- API versioning scheme: using accept header versioning instead of namespace versioning () +- Replaced 'django_sendfile' with 'django_sendfile2' () +- Use drf-spectacular instead of drf-yasg for swagger documentation () +- Update development-environment manual to work under MacOS, supported Mac with Apple Silicon () + +### Deprecated + +- Job field "status" is not used in UI anymore, but it has not been removed from the database yet () + +### Removed + +- Review rating, reviewer field from the job instance (use assignee field together with stage field instead) () +- Training django app () +- v1 api version support () + +### Fixed + +- Fixed Interaction handler keyboard handlers () +- Points of invisible shapes are visible in autobordering () +- Order of the label attributes in the object item details() +- Order of labels in tasks and projects () +- Fixed task creating with large files via webpage () +- Added information to export CVAT_HOST when performing local installation for accessing over network () +- Fixed possible color collisions in the generated colormap () +- Original pdf file is deleted when using share () +- Order in an annotation file() +- Fixed task data upload progressbar () +- Email in org invitations is case sensitive () +- Caching for tasks and jobs can lead to an exception if its assignee user is removed () +- Added intelligent function when paste labels to another task () +- Uncaught TypeError: this.el.node.getScreenCTM() is null in Firefox () +- Bug: canvas is busy when start playing, start resizing a shape and do not release the mouse cursor () +- Bug: could not receive frame N. TypeError: Cannot read properties of undefined (reading "filename") () +- Cannot choose a dataset format for a linked repository if a task type is annotation () +- Fixed tus upload error over https () +- Issues disappear when rescale a browser () +- Auth token key is not returned when registering without email verification () +- Error in create project from backup for standard 3D annotation () +- Annotations search does not work correctly in some corner cases (when use complex properties with width, height) () +- Kibana requests are not proxied due to django-revproxy incompatibility with Django >3.2.x () +- Content type for getting frame with tasks/{id}/data/ endpoint () +- Bug: Permission error occurred when accessing the comments of a specific issue () + +### Security + +- Updated ELK to 6.8.23 which uses log4j 2.17.1 () +- Added validation for URLs which used as remote data source () + +## \[1.7.0] - 2021-11-15 + +### Added + +- cvat-ui: support cloud storages () +- interactor: add HRNet interactive segmentation serverless function () +- Added GPU implementation for SiamMask, reworked tracking approach () +- Progress bar for manifest creating () +- IAM: Open Policy Agent integration () +- Add a tutorial on attaching cloud storage AWS-S3 () + and Azure Blob Container () +- The feature to remove annotations in a specified range of frames () +- Project backup/restore () + +### Changed + +- UI tracking has been reworked () +- Updated Django till 3.2.7 (automatic AppConfig discovery) +- Manifest generation: Reduce creating time () +- Migration from NPM 6 to NPM 7 () +- Update Datumaro dependency to 0.2.0 () + +### Fixed + +- Fixed JSON transform issues in network requests () +- Display a more user-friendly exception message () +- Exception `DataCloneError: The object could not be cloned` () +- Fixed extension comparison in task frames CLI () +- Incorrect work when copy job list with "Copy" button () +- Iterating over manifest () +- Manifest removing () +- Fixed project updated date () +- Fixed dextr deployment () +- Migration of `dataset_repo` application () +- Helm settings for external psql database were unused by backend () +- Updated WSL setup for development () +- Helm chart config () + +### Security + +- Fix security issues on the documentation website unsafe use of target blank + and potential clickjacking on legacy browsers () + +## \[1.6.0] - 2021-09-17 + +### Added + +- Added ability to import data from share with cli without copying the data () +- Notification if the browser does not support necessary API +- Added ability to export project as a dataset () + and project with 3D tasks () +- Additional inline tips in interactors with demo gifs () +- Added intelligent scissors blocking feature () +- Support cloud storage status () +- Support cloud storage preview () +- cvat-core: support cloud storages () + +### Changed + +- Non-blocking UI when using interactors () +- "Selected opacity" slider now defines opacity level for shapes being drawnSelected opacity () +- Cloud storage creating and updating () +- Way of working with cloud storage content () + +### Removed + +- Support TEMP_KEY_SECRET_KEY_TOKEN_SET for AWS S3 cloud storage () + +### Fixed + +- Fixed multiple tasks moving () +- Fixed task creating CLI parameter () +- Fixed import for MOTS format () + +## \[1.5.0] - 2021-08-02 + +### Added + +- Support of context images for 2D image tasks () +- Support of cloud storage without copying data into CVAT: server part () +- Filter `is_active` for user list () +- Ability to export/import tasks () +- Add a tutorial for semi-automatic/automatic annotation () +- Explicit "Done" button when drawing any polyshapes () +- Histogram equalization with OpenCV javascript () +- Client-side polyshapes approximation when using semi-automatic interactors & scissors () +- Support of Google Cloud Storage for cloud storage () + +### Changed + +- Updated manifest format, added meta with related images () +- Update of COCO format documentation () +- Updated Webpack Dev Server config to add proxy () +- Update to Django 3.1.12 () +- Updated visibility for removable points in AI tools () +- Updated UI handling for IOG serverless function () +- Changed Nginx proxy to Traefik in `docker-compose.yml` () +- Simplify the process of deploying CVAT with HTTPS () + +### Fixed + +- Project page requests took a long time and did many DB queries () +- Fixed Python 3.6 support () +- Incorrect attribute import in tracks () +- Issue "is not a constructor" when create object, save, undo, save, redo save () +- Fix CLI create an infinite loop if git repository responds with failure () +- Bug with sidebar & fullscreen () +- 504 Gateway Time-out on `data/meta` requests () +- TypeError: Cannot read property 'clientX' of undefined when draw cuboids with hotkeys () +- Duplication of the cuboids when redraw them () +- Some code issues in Deep Extreme Cut handler code () +- UI fails when inactive user is assigned to a task/job () +- Calculate precise progress of decoding a video file () +- Falsely successful `cvat_ui` image build in case of OOM error that leads to the default nginx welcome page + () +- Fixed issue when save filtered object in AAM () +- Context image disappears after undo/redo () +- Using combined data sources (directory and image) when create a task () +- Creating task with labels in project () +- Move task and autoannotation modals were invisible from project page () + +## \[1.4.0] - 2021-05-18 + +### Added + +- Documentation on mask annotation () +- Hotkeys to switch a label of existing object or to change default label (for objects created with N) () +- A script to convert some kinds of DICOM files to regular images () +- Helm chart prototype () +- Initial implementation of moving tasks between projects () + +### Changed + +- Place of migration logger initialization () + +### Removed + +- Kubernetes templates from () due to helm charts () + +### Fixed + +- Export of instance masks with holes () +- Changing a label on canvas does not work when 'Show object details' enabled () +- Make sure frame unzip web worker correctly terminates after unzipping all images in a requested chunk () +- Reset password link was unavailable before login () +- Manifest: migration () +- Fixed cropping polygon in some corner cases () + +## \[1.3.0] - 3/31/2021 + +### Added + +- CLI: Add support for saving annotations in a git repository when creating a task. +- CVAT-3D: support lidar data on the server side () +- GPU support for Mask-RCNN and improvement in its deployment time () +- CVAT-3D: Load all frames corresponding to the job instance + () +- Intelligent scissors with OpenCV javascript () +- CVAT-3D: Visualize 3D point cloud spaces in 3D View, Top View Side View and Front View () +- [Inside Outside Guidance](https://github.com/shiyinzhang/Inside-Outside-Guidance) serverless + function for interactive segmentation +- Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and + [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub () +- Project task subsets () +- Kubernetes templates and guide for their deployment () +- [WiderFace](http://shuoyang1213.me/WIDERFACE/) format support () +- [VGGFace2](https://github.com/ox-vgg/vgg_face2) format support () +- [Backup/Restore guide](cvat/apps/documentation/backup_guide.md) () +- Label deletion from tasks and projects () +- CVAT-3D: Implemented initial cuboid placement in 3D View and select cuboid in Top, Side and Front views + () +- [Market-1501](https://www.aitribune.com/dataset/2018051063) format support () +- Ability of upload manifest for dataset with images () +- Annotations filters UI using react-awesome-query-builder () +- Storing settings in local storage to keep them between browser sessions () +- [ICDAR](https://rrc.cvc.uab.es/?ch=2) format support () +- Added switcher to maintain polygon crop behavior ( +- Filters and sorting options for job list, added tooltip for tasks filters () + +### Changed + +- CLI - task list now returns a list of current tasks. () +- Updated HTTPS install README section (cleanup and described more robust deploy) +- Logstash is improved for using with configurable elasticsearch outputs () +- Bumped nuclio version to 1.5.16 () +- All methods for interactive segmentation accept negative points as well +- Persistent queue added to logstash () +- Improved maintenance of popups visibility () +- Image visualizations settings on canvas for faster access () +- Better scale management of left panel when screen is too small () +- Improved error messages for annotation import () +- Using manifest support instead video meta information and dummy chunks () + +### Fixed + +- More robust execution of nuclio GPU functions by limiting the GPU memory consumption per worker () +- Kibana startup initialization () +- The cursor jumps to the end of the line when renaming a task () +- SSLCertVerificationError when remote source is used () +- Fixed filters select overflow () +- Fixed tasks in project auto annotation () +- Cuboids are missed in annotations statistics () +- The list of files attached to the task is not displayed () +- A couple of css-related issues (top bar disappear, wrong arrow position on collapse elements) () +- Issue with point region doesn't work in Firefox () +- Fixed cuboid perspective change () +- Annotation page popups (ai tools, drawing) reset state after detecting, tracking, drawing () +- Polygon editing using trailing point () +- Updated the path to python for DL models inside automatic annotation documentation () +- Fixed of receiving function variable () +- Shortcuts with CAPSLOCK enabled and with non-US languages activated () +- Prevented creating several issues for the same object () +- Fixed label editor name field validator () +- An error about track shapes outside of the task frames during export () +- Fixed project search field updating () +- Fixed export error when invalid polygons are present in overlapping frames () +- Fixed image quality option for tasks created from images () +- Incorrect text on the warning when specifying an incorrect link to the issue tracker () +- Updating label attributes when label contains number attributes () +- Crop a polygon if its points are outside the bounds of the image () + +## \[1.2.0] - 2021-01-08 + +### Fixed + +- Memory consumption for the task creation process () +- Frame preloading () +- Project cannot be removed from the project page () + +## \[1.2.0-beta] - 2020-12-15 + +### Added + +- GPU support and improved documentation for auto annotation () +- Manual review pipeline: issues/comments/workspace () +- Basic projects implementation () +- Documentation on how to mount cloud starage(AWS S3 bucket, Azure container, Google Drive) as FUSE () +- Ability to work with share files without copying inside () +- Tooltips in label selectors () +- Page redirect after login using `next` query parameter () +- [ImageNet](http://www.image-net.org) format support () +- [CamVid](http://mi.eng.cam.ac.uk/research/projects/VideoRec/CamVid/) format support () + +### Changed + +- PATCH requests from cvat-core submit only changed fields () +- deploy.sh in serverless folder is separated into deploy_cpu.sh and deploy_gpu.sh () +- Bumped nuclio version to 1.5.8 +- Migrated to Antd 4.9 () + +### Fixed + +- Fixed FastRCNN inference bug for images with 4 channels i.e. png () +- Django templates for email and user guide () +- Saving relative paths in dummy chunks instead of absolute () +- Objects with a specific label cannot be displayed if at least one tag with the label exist () +- Wrong attribute can be removed in labels editor () +- UI fails with the error "Cannot read property 'label' of undefined" () +- Exception: "Value must be a user instance" () +- Reset zoom option doesn't work in tag annotation mode () +- Canvas is busy error () +- Projects view layout fix () +- Fixed the tasks view (infinite loading) when it is impossible to get a preview of the task () +- Empty frames navigation () +- TypeError: Cannot read property 'toString' of undefined () +- Extra shapes are drawn after Esc, or G pressed while drawing a region in grouping () +- Reset state (reviews, issues) after logout or changing a job () +- TypeError: Cannot read property 'id' of undefined when updating a task () + +## \[1.2.0-alpha] - 2020-11-09 + +### Added + +- Ability to login into CVAT-UI with token from api/v1/auth/login () +- Added layout grids toggling ('ctrl + alt + Enter') +- Added password reset functionality () +- Ability to work with data on the fly () +- Annotation in process outline color wheel () +- On the fly annotation using DL detectors () +- Displaying automatic annotation progress on a task view () +- Automatic tracking of bounding boxes using serverless functions () +- \[Datumaro] CLI command for dataset equality comparison () +- \[Datumaro] Merging of datasets with different labels () +- Add FBRS interactive segmentation serverless function () +- Ability to change default behaviour of previous/next buttons of a player. + It supports regular navigation, searching a frame according to annotations + filters and searching the nearest frame without any annotations () +- MacOS users notes in CONTRIBUTING.md +- Ability to prepare meta information manually () +- Ability to upload prepared meta information along with a video when creating a task () +- Optional chaining plugin for cvat-canvas and cvat-ui () +- MOTS png mask format support () +- Ability to correct upload video with a rotation record in the metadata () +- User search field for assignee fields () +- Support of mxf videos () + +### Changed + +- UI models (like DEXTR) were redesigned to be more interactive () +- Used Ubuntu:20.04 as a base image for CVAT Dockerfile () +- Right colors of label tags in label mapping when a user runs automatic detection () +- Nuclio became an optional component of CVAT () +- A key to remove a point from a polyshape (Ctrl => Alt) () +- Updated `docker-compose` file version from `2.3` to `3.3`() +- Added auto inference of url schema from host in CLI, if provided () +- Track frames in skips between annotation is presented in MOT and MOTS formats are marked `outside` () +- UI packages installation with `npm ci` instead of `npm install` () + +### Removed + +- Removed Z-Order flag from task creation process + +### Fixed + +- Fixed multiple errors which arises when polygon is of length 5 or less () +- Fixed task creation from PDF () +- Fixed CVAT format import for frame stepped tasks () +- Fixed the reading problem with large PDFs () +- Fixed unnecessary pyhash dependency () +- Fixed Data is not getting cleared, even after deleting the Task from Django Admin App() +- Fixed blinking message: "Some tasks have not been showed because they do not have any data" () +- Fixed case when a task with 0 jobs is shown as "Completed" in UI () +- Fixed use case when UI throws exception: Cannot read property 'objectType' of undefined #2053 () +- Fixed use case when logs could be saved twice or more times #2202 () +- Fixed issues from #2112 () +- Git application name (renamed to dataset_repo) () +- A problem in exporting of tracks, where tracks could be truncated () +- Fixed CVAT startup process if the user has `umask 077` in .bashrc file () +- Exception: Cannot read property "each" of undefined after drawing a single point () +- Cannot read property 'label' of undefined (Fixed?) () +- Excluded track frames marked `outside` in `CVAT for Images` export () +- 'List of tasks' Kibana visualization () +- An error on exporting not `jpg` or `png` images in TF Detection API format () + +## \[1.1.0] - 2020-08-31 + +### Added + +- Siammask tracker as DL serverless function () +- \[Datumaro] Added model info and source info commands () +- \[Datumaro] Dataset statistics () +- Ability to change label color in tasks and predefined labels () +- \[Datumaro] Multi-dataset merge () +- Ability to configure email verification for new users () +- Link to django admin page from UI () +- Notification message when users use wrong browser () + +### Changed + +- Shape coordinates are rounded to 2 digits in dumped annotations () +- COCO format does not produce polygon points for bbox annotations () + +### Fixed + +- Issue loading openvino models for semi-automatic and automatic annotation () +- Basic functions of CVAT works without activated nuclio dashboard +- Fixed a case in which exported masks could have wrong color order () +- Fixed error with creating task with labels with the same name () +- Django RQ dashboard view () +- Object's details menu settings () + +## \[1.1.0-beta] - 2020-08-03 + +### Added + +- DL models as serverless functions () +- Source type support for tags, shapes and tracks () +- Source type support for CVAT Dumper/Loader () +- Intelligent polygon editing () +- Support creating multiple jobs for each task through python cli () +- python cli over https () +- Error message when plugins weren't able to initialize instead of infinite loading () +- Ability to change user password () + +### Changed + +- Smaller object details () +- `COCO` format does not convert bboxes to polygons on export () +- It is impossible to submit a DL model in OpenVINO format using UI. + Now you can deploy new models on the server using serverless functions + () +- Files and folders under share path are now alphabetically sorted + +### Removed + +- Removed OpenVINO and CUDA components because they are not necessary anymore () +- Removed the old UI code () + +### Fixed + +- Some objects aren't shown on canvas sometimes. For example after propagation on of objects is invisible () +- CVAT doesn't offer to restore state after an error () +- Cannot read property 'shapeType' of undefined because of zOrder related issues () +- Cannot read property 'pinned' of undefined because of zOrder related issues () +- Do not iterate over hidden objects in aam (which are invisible because of zOrder) () +- Cursor position is reset after changing a text field () +- Hidden points and cuboids can be selected to be grouped () +- `outside` annotations should not be in exported images () +- `CVAT for video format` import error with interpolation () +- `Image compression` definition mismatch () +- Points are duplicated during polygon interpolation sometimes () +- When redraw a shape with activated autobordering, previous points are visible () +- No mapping between side object element and context menu in some attributes () +- Interpolated shapes exported as `keyframe = True` () +- Stylelint filetype scans () +- Fixed toolip closing issue () +- Clearing frame cache when close a task () +- Increase rate of throttling policy for unauthenticated users () + +## \[1.1.0-alpha] - 2020-06-30 + +### Added + +- Throttling policy for unauthenticated users () +- Added default label color table for mask export () +- Added environment variables for Redis and Postgres hosts for Kubernetes deployment support () +- Added visual identification for unavailable formats () +- Shortcut to change color of an activated shape in new UI (Enter) () +- Shortcut to switch split mode () +- Built-in search for labels when create an object or change a label () +- Better validation of labels and attributes in raw viewer () +- ClamAV antivirus integration () +- Added canvas background color selector () +- SCSS files linting with Stylelint tool () +- Supported import and export or single boxes in MOT format () +- \[Datumaro] Added `stats` command, which shows some dataset statistics + like image mean and std () +- Add option to upload annotations upon task creation on CLI +- Polygon and polylines interpolation () +- Ability to redraw shape from scratch (Shift + N) for an activated shape () +- Highlights for the first point of a polygon/polyline and direction () +- Ability to change orientation for poylgons/polylines in context menu () +- Ability to set the first point for polygons in points context menu () +- Added new tag annotation workspace () +- Appearance block in attribute annotation mode () +- Keyframe navigations and some switchers in attribute annotation mode () +- \[Datumaro] Added `convert` command to convert datasets directly () +- \[Datumaro] Added an option to specify image extension when exporting datasets () +- \[Datumaro] Added image copying when exporting datasets, if possible () + +### Changed + +- Removed information about e-mail from the basic user information () +- Update https install manual. Makes it easier and more robust. + Includes automatic renewing of lets encrypt certificates. +- Settings page move to the modal. () +- Implemented import and export of annotations with relative image paths () +- Using only single click to start editing or remove a point () +- Added support for attributes in VOC XML format () +- Added annotation attributes in COCO format () +- Colorized object items in the side panel () +- \[Datumaro] Annotation-less files are not generated anymore in COCO format, unless tasks explicitly requested () + +### Fixed + +- Problem with exported frame stepped image task () +- Fixed dataset filter item representation for imageless dataset items () +- Fixed interpreter crash when trying to import `tensorflow` with no AVX instructions available () +- Kibana wrong working time calculation with new annotation UI use () +- Wrong rexex for account name validation () +- Wrong description on register view for the username field () +- Wrong resolution for resizing a shape () +- React warning because of not unique keys in labels viewer () +- Fixed issue tracker () +- Fixed canvas fit after sidebar open/close event () +- A couple of exceptions in AAM related with early object activation () +- Propagation from the latest frame () +- Number attribute value validation (didn't work well with floats) () +- Logout doesn't work () +- Annotations aren't updated after reopening a task () +- Labels aren't updated after reopening a task () +- Canvas isn't fitted after collapsing side panel in attribute annotation mode () +- Error when interpolating polygons () + +### Security + +- SQL injection in Django `CVE-2020-9402` () + +## \[1.0.0] - 2020-05-29 + +### Added + +- cvat-ui: cookie policy drawer for login page () +- `datumaro_project` export format () +- Ability to configure user agreements for the user registration form () +- Cuboid interpolation and cuboid drawing from rectangles () +- Ability to configure custom pageViewHit, which can be useful for web analytics integration () +- Ability to configure access to the analytics page based on roles () + +### Changed + +- Downloaded file name in annotations export became more informative () +- Added auto trimming for trailing whitespaces style enforcement () +- REST API: updated `GET /task//annotations`: parameters are `format`, `filename` + (now optional), `action` (optional) () +- REST API: removed `dataset/formats`, changed format of `annotation/formats` () +- Exported annotations are stored for N hours instead of indefinitely () +- Formats: CVAT format now accepts ZIP and XML () +- Formats: COCO format now accepts ZIP and JSON () +- Formats: most of formats renamed, no extension in title () +- Formats: definitions are changed, are not stored in DB anymore () +- cvat-core: session.annotations.put() now returns ids of added objects () +- Images without annotations now also included in dataset/annotations export () + +### Removed + +- `annotation` application is replaced with `dataset_manager` () +- `_DATUMARO_INIT_LOGLEVEL` env. variable is removed in favor of regular `--loglevel` cli parameter () + +### Fixed + +- Categories for empty projects with no sources are taken from own dataset () +- Added directory removal on error during `extract` command () +- Added debug error message on incorrect XPath () +- Exporting frame stepped task + (, ) +- Fixed broken command line interface for `cvat` export format in Datumaro () +- Updated Rest API document, Swagger document serving instruction issue () +- Fixed cuboid occluded view () +- Non-informative lock icon () +- Sidebar in AAM has no hide/show button () +- Task/Job buttons has no "Open in new tab" option () +- Delete point context menu option has no shortcut hint () +- Fixed issue with unnecessary tag activation in cvat-canvas () +- Fixed an issue with large number of instances in instance mask () +- Fixed full COCO dataset import error with conflicting labels in keypoints and detection () +- Fixed COCO keypoints skeleton parsing and saving () +- `tf.placeholder() is not compatible with eager execution` exception for auto_segmentation () +- Canvas cannot be moved with move functionality on left mouse key () +- Deep extreme cut request is sent when draw any shape with Make AI polygon option enabled () +- Fixed an error when exporting a task with cuboids to any format except CVAT () +- Synchronization with remote git repo () +- A problem with mask to polygons conversion when polygons are too small () +- Unable to upload video with uneven size () +- Fixed an issue with `z_order` having no effect on segmentations () + +### Security + +- Permission group whitelist check for analytics view () + +## \[1.0.0-beta.2] - 2020-04-30 + +### Added + +- Re-Identification algorithm to merging bounding boxes automatically to the new UI () +- Methods `import` and `export` to import/export raw annotations for Job and Task in `cvat-core` () +- Versioning of client packages (`cvat-core`, `cvat-canvas`, `cvat-ui`). Initial versions are set to 1.0.0 () +- Cuboids feature was migrated from old UI to new one. () + +### Removed + +- Annotation conversion utils, currently supported natively via Datumaro framework + () + +### Fixed + +- Auto annotation, TF annotation and Auto segmentation apps () +- Import works with truncated images now: "OSError:broken data stream" on corrupt images + () +- Hide functionality (H) doesn't work () +- The highlighted attribute doesn't correspond to the chosen attribute in AAM () +- Inconvenient image shaking while drawing a polygon (hold Alt key during drawing/editing/grouping to drag an image) () +- Filter property "shape" doesn't work and extra operator in description () +- Block of text information doesn't disappear after deactivating for locked shapes () +- Annotation uploading fails in annotation view () +- UI freezes after canceling pasting with escape () +- Duplicating keypoints in COCO export () +- CVAT new UI: add arrows on a mouse cursor () +- Delete point bug (in new UI) () +- Fix apache startup after PC restart () +- Open task button doesn't work () + +## \[1.0.0-beta.1] - 2020-04-15 + +### Added + +- Special behaviour for attribute value `__undefined__` (invisibility, no shortcuts to be set in AAM) +- Dialog window with some helpful information about using filters +- Ability to display a bitmap in the new UI +- Button to reset colors settings (brightness, saturation, contrast) in the new UI +- Option to display shape text always +- Dedicated message with clarifications when share is unmounted () +- Ability to create one tracked point () +- Ability to draw/edit polygons and polylines with automatic bordering feature + () +- Tutorial: instructions for CVAT over HTTPS +- Deep extreme cut (semi-automatic segmentation) to the new UI () + +### Changed + +- Increase preview size of a task till 256, 256 on the server +- Public ssh-keys are displayed in a dedicated window instead of console when create a task with a repository +- React UI is the primary UI + +### Fixed + +- Cleaned up memory in Auto Annotation to enable long running tasks on videos +- New shape is added when press `esc` when drawing instead of cancellation +- Dextr segmentation doesn't work. +- `FileNotFoundError` during dump after moving format files +- CVAT doesn't append outside shapes when merge polyshapes in old UI +- Layout sometimes shows double scroll bars on create task, dashboard and settings pages +- UI fails after trying to change frame during resizing, dragging, editing +- Hidden points (or outsided) are visible after changing a frame +- Merge is allowed for points, but clicks on points conflict with frame dragging logic +- Removed objects are visible for search +- Add missed task_id and job_id fields into exception logs for the new UI () +- UI fails when annotations saving occurs during drag/resize/edit () +- Multiple savings when hold Ctrl+S (a lot of the same copies of events were sent with the same working time) + () +- UI doesn't have any reaction when git repos synchronization failed () +- Bug when annotations cannot be saved after (delete - save - undo - save) () +- VOC format exports Upper case labels correctly in lower case () +- Fixed polygon exporting bug in COCO dataset () +- Task creation from remote files () +- Job cannot be opened in some cases when the previous job was failed during opening + () +- Deactivated shape is still highlighted on the canvas () +- AttributeError: 'tuple' object has no attribute 'read' in ReID algorithm () +- Wrong semi-automatic segmentation near edges of an image () +- Git repos paths () +- Uploading annotations for tasks with multiple jobs () + +## \[1.0.0-alpha] - 2020-03-31 + +### Added + +- Data streaming using chunks () +- New UI: showing file names in UI () +- New UI: delete a point from context menu () + +### Fixed + +- Git app cannot clone a repository () +- New UI: preview position in task details () +- AWS deployment () + +## \[0.6.1] - 2020-03-21 + +### Changed + +- VOC task export now does not use official label map by default, but takes one + from the source task to avoid primary-class and class part name + clashing ([#1275](https://github.com/opencv/cvat/issues/1275)) + +### Fixed + +- File names in LabelMe format export are no longer truncated ([#1259](https://github.com/opencv/cvat/issues/1259)) +- `occluded` and `z_order` annotation attributes are now correctly passed to Datumaro ([#1271](https://github.com/opencv/cvat/pull/1271)) +- Annotation-less tasks now can be exported as empty datasets in COCO ([#1277](https://github.com/opencv/cvat/issues/1277)) +- Frame name matching for video annotations import - + allowed `frame_XXXXXX[.ext]` format ([#1274](https://github.com/opencv/cvat/pull/1274)) + +### Security + +- Bump acorn from 6.3.0 to 6.4.1 in /cvat-ui ([#1270](https://github.com/opencv/cvat/pull/1270)) + +## \[0.6.0] - 2020-03-15 + +### Added + +- Server only support for projects. Extend REST API v1 (/api/v1/projects\*) +- Ability to get basic information about users without admin permissions ([#750](https://github.com/opencv/cvat/issues/750)) +- Changed REST API: removed PUT and added DELETE methods for /api/v1/users/ID +- Mask-RCNN Auto Annotation Script in OpenVINO format +- Yolo Auto Annotation Script +- Auto segmentation using Mask_RCNN component (Keras+Tensorflow Mask R-CNN Segmentation) +- REST API to export an annotation task (images + annotations) + [Datumaro](https://github.com/opencv/cvat/tree/develop/datumaro) - + a framework to build, analyze, debug and visualize datasets +- Text Detection Auto Annotation Script in OpenVINO format for version 4 +- Added in OpenVINO Semantic Segmentation for roads +- Ability to visualize labels when using Auto Annotation runner +- MOT CSV format support ([#830](https://github.com/opencv/cvat/pull/830)) +- LabelMe format support ([#844](https://github.com/opencv/cvat/pull/844)) +- Segmentation MASK format import (as polygons) ([#1163](https://github.com/opencv/cvat/pull/1163)) +- Git repositories can be specified with IPv4 address ([#827](https://github.com/opencv/cvat/pull/827)) + +### Changed + +- page_size parameter for all REST API methods +- React & Redux & Antd based dashboard +- Yolov3 interpretation script fix and changes to mapping.json +- YOLO format support ([#1151](https://github.com/opencv/cvat/pull/1151)) +- Added support for OpenVINO 2020 + +### Fixed + +- Exception in Git plugin [#826](https://github.com/opencv/cvat/issues/826) +- Label ids in TFrecord format now start from 1 [#866](https://github.com/opencv/cvat/issues/866) +- Mask problem in COCO JSON style [#718](https://github.com/opencv/cvat/issues/718) +- Datasets (or tasks) can be joined and split to subsets with Datumaro [#791](https://github.com/opencv/cvat/issues/791) +- Output labels for VOC format can be specified with Datumaro [#942](https://github.com/opencv/cvat/issues/942) +- Annotations can be filtered before dumping with Datumaro [#994](https://github.com/opencv/cvat/issues/994) + +## \[0.5.2] - 2019-12-15 + +### Fixed + +- Frozen version of scikit-image==0.15 in requirements.txt because next releases don't support Python 3.5 + +## \[0.5.1] - 2019-10-17 + +### Added + +- Integration with Zenodo.org (DOI) + +## \[0.5.0] - 2019-09-12 + +### Added + +- A converter to YOLO format +- Installation guide +- Linear interpolation for a single point +- Video frame filter +- Running functional tests for REST API during a build +- Admins are no longer limited to a subset of python commands in the auto annotation application +- Remote data source (list of URLs to create an annotation task) +- Auto annotation using Faster R-CNN with Inception v2 (utils/open_model_zoo) +- Auto annotation using Pixel Link mobilenet v2 - text detection (utils/open_model_zoo) +- Ability to create a custom extractors for unsupported media types +- Added in PDF extractor +- Added in a command line model manager tester +- Ability to dump/load annotations in several formats from UI (CVAT, Pascal VOC, YOLO, MS COCO, png mask, TFRecord) +- Auth for REST API (api/v1/auth/): login, logout, register, ... +- Preview for the new CVAT UI (dashboard only) is available: +- Added command line tool for performing common task operations (/utils/cli/) + +### Changed + +- Outside and keyframe buttons in the side panel for all interpolation shapes (they were only for boxes before) +- Improved error messages on the client side (#511) + +### Removed + +- "Flip images" has been removed. UI now contains rotation features. + +### Fixed + +- Incorrect width of shapes borders in some cases +- Annotation parser for tracks with a start frame less than the first segment frame +- Interpolation on the server near outside frames +- Dump for case when task name has a slash +- Auto annotation fail for multijob tasks +- Installation of CVAT with OpenVINO on the Windows platform +- Background color was always black in utils/mask/converter.py +- Exception in attribute annotation mode when a label are switched to a value without any attributes +- Handling of wrong labelamp json file in auto annotation () +- No default attributes in dumped annotation () +- Required field "Frame Filter" on admin page during a task modifying (#666) +- Dump annotation errors for a task with several segments (#610, #500) +- Invalid label parsing during a task creating (#628) +- Button "Open Task" in the annotation view +- Creating a video task with 0 overlap + +### Security + +- Upgraded Django, djangorestframework, and other packages + +## \[0.4.2] - 2019-06-03 + +### Fixed + +- Fixed interaction with the server share in the auto annotation plugin + +## \[0.4.1] - 2019-05-14 + +### Fixed + +- JavaScript syntax incompatibility with Google Chrome versions less than 72 + +## \[0.4.0] - 2019-05-04 + +### Added + +- OpenVINO auto annotation: it is possible to upload a custom model and annotate images automatically. +- Ability to rotate images/video in the client part (Ctrl+R, Shift+Ctrl+R shortcuts) (#305) +- The ReID application for automatic bounding box merging has been added (#299) +- Keyboard shortcuts to switch next/previous default shape type (box, polygon etc) (Alt + <, Alt + >) (#316) +- Converter for VOC now supports interpolation tracks +- REST API (/api/v1/\*, /api/docs) +- Semi-automatic semantic segmentation with the + [Deep Extreme Cut](http://www.vision.ee.ethz.ch/~cvlsegmentation/dextr/) work + +### Changed + +- Propagation setup has been moved from settings to bottom player panel +- Additional events like "Debug Info" or "Fit Image" have been added for analytics +- Optional using LFS for git annotation storages (#314) + +### Deprecated + +- "Flip images" flag in the create task dialog will be removed. + Rotation functionality in client part have been added instead. + +### Fixed + +- Django 2.1.5 (security fix, [CVE-2019-3498](https://nvd.nist.gov/vuln/detail/CVE-2019-3498)) +- Several scenarios which cause code 400 after undo/redo/save have been fixed (#315) + +## \[0.3.0] - 2018-12-29 + +### Added + +- Ability to copy Object URL and Frame URL via object context menu and player context menu respectively. +- Ability to change opacity for selected shape with help "Selected Fill Opacity" slider. +- Ability to remove polyshapes points by double click. +- Ability to draw/change polyshapes (except for points) by slip method. Just press ENTER and moving a cursor. +- Ability to switch lock/hide properties via label UI element (in right menu) for all objects with same label. +- Shortcuts for outside/keyframe properties +- Support of Intel OpenVINO for accelerated model inference +- Tensorflow annotation now works without CUDA. It can use CPU only. OpenVINO and CUDA are supported optionally. +- Incremental saving of annotations. +- Tutorial for using polygons (screencast) +- Silk profiler to improve development process +- Admin panel can be used to edit labels and attributes for annotation tasks +- Analytics component to manage a data annotation team, monitor exceptions, collect client and server logs +- Changeable job and task statuses (annotation, validation, completed). + A job status can be changed manually, a task status is computed automatically based on job statuses (#153) +- Backlink to a task from its job annotation view (#156) +- Buttons lock/hide for labels. They work for all objects with the same label on a current frame (#116) + +### Changed + +- Polyshape editing method has been improved. You can redraw part of shape instead of points cloning. +- Unified shortcut (Esc) for close any mode instead of different shortcuts (Alt+N, Alt+G, Alt+M etc.). +- Dump file contains information about data source (e.g. video name, archive name, ...) +- Update requests library due to [CVE-2018-18074](https://nvd.nist.gov/vuln/detail/CVE-2018-18074) +- Per task/job permissions to create/access/change/delete tasks and annotations +- Documentation was improved +- Timeout for creating tasks was increased (from 1h to 4h) (#136) +- Drawing has become more convenience. Now it is possible to draw outside an image. + Shapes will be automatically truncated after drawing process (#202) + +### Fixed + +- Performance bottleneck has been fixed during you create new objects (draw, copy, merge etc). +- Label UI elements aren't updated after changelabel. +- Attribute annotation mode can use invalid shape position after resize or move shapes. +- Labels order is preserved now (#242) +- Uploading large XML files (#123) +- Django vulnerability (#121) +- Grammatical cleanup of README.md (#107) +- Dashboard loading has been accelerated (#156) +- Text drawing outside of a frame in some cases (#202) + +## \[0.2.0] - 2018-09-28 + +### Added + +- New annotation shapes: polygons, polylines, points +- Undo/redo feature +- Grid to estimate size of objects +- Context menu for shapes +- A converter to PASCAL VOC format +- A converter to MS COCO format +- A converter to mask format +- License header for most of all files +- .gitattribute to avoid problems with bash scripts inside a container +- CHANGELOG.md itself +- Drawing size of a bounding box during resize +- Color by instance, group, label +- Group objects +- Object propagation on next frames +- Full screen view + +### Changed + +- Documentation, screencasts, the primary screenshot +- Content-type for save_job request is application/json + +### Fixed + +- Player navigation if the browser's window is scrolled +- Filter doesn't support dash (-) +- Several memory leaks +- Inconsistent extensions between filenames in an annotation file and real filenames + +## \[0.1.2] - 2018-08-07 + +### Added + +- 7z archive support when creating a task +- .vscode/launch.json file for developing with VS code + +### Fixed + +- #14: docker-compose down command as written in the readme does not remove volumes +- #15: all checkboxes in temporary attributes are checked when reopening job after saving the job +- #18: extend CONTRIBUTING.md +- #19: using the same attribute for label twice -> stuck + +### Changed + +- More strict verification for labels with attributes + +## \[0.1.1] - 2018-07-6 + +### Added + +- Links on a screenshot, documentation, screencasts into README.md +- CONTRIBUTORS.md + +### Fixed + +- GitHub documentation + +## \[0.1.0] - 2018-06-29 + +### Added + +- Initial version diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..aa063aa --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,38 @@ +# This CITATION.cff file was generated with cffinit. +# Visit https://bit.ly/cffinit to generate yours today! + +cff-version: 1.2.0 +title: Computer Vision Annotation Tool (CVAT) +message: >- + If you use this software, please cite it using the + metadata from this file. +type: software +authors: + - email: support+github@cvat.ai + name: CVAT.ai Corporation +identifiers: + - type: doi + value: 10.5281/zenodo.4009388 +repository-code: 'https://github.com/cvat-ai/cvat' +url: 'https://cvat.ai/' +abstract: >- + Annotate better with CVAT, the industry-leading + data engine for machine learning. Used and trusted + by teams at any scale, for data of any scale. +keywords: + - image-labeling-tool + - computer-vision-annotation + - labeling-tool + - image-labeling + - semantic-segmentation + - annotation-tool + - object-detection + - image-classification + - video-annotation + - computer-vision + - deep-learning + - annotation +license: MIT +version: 2.25.0 +date-released: '2023-11-06' + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..179e091 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,209 @@ +ARG BASE_IMAGE=ubuntu:24.04 + +FROM ${BASE_IMAGE} AS build-image-base + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ + cargo-1.85 \ + curl \ + g++ \ + gcc \ + git \ + libhdf5-dev \ + libldap2-dev \ + libmp3lame-dev \ + libsasl2-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libxmlsec1-openssl \ + make \ + nasm \ + pkg-config \ + python3-dev \ + python3-pip \ + && update-alternatives \ + --install /usr/bin/rustc rustc /usr/bin/rustc-1.85 185 \ + --slave /usr/bin/cargo cargo /usr/bin/cargo-1.85 \ + && rm -rf /var/lib/apt/lists/* + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 + +# We build OpenH264, FFmpeg and PyAV in a separate build stage, +# because this way Docker can do it in parallel to all the other packages. +FROM build-image-base AS build-image-av + +# Compile Openh264 and FFmpeg +ARG PREFIX=/opt/ffmpeg +ARG PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig + +ENV FFMPEG_VERSION=8.0 \ + OPENH264_VERSION=2.6.0 + +WORKDIR /tmp/openh264 +RUN curl -sL https://github.com/cisco/openh264/archive/v${OPENH264_VERSION}.tar.gz --output - | \ + tar -zx --strip-components=1 && \ + make -j5 && make install-shared PREFIX=${PREFIX} && make clean + +WORKDIR /tmp/ffmpeg +RUN curl -sL https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz --output - | \ + tar -zx --strip-components=1 && \ + ./configure --disable-nonfree --disable-gpl --enable-libopenh264 --enable-libmp3lame \ + --enable-shared --disable-static --disable-doc --disable-programs --prefix="${PREFIX}" && \ + make -j5 && make install && make clean + +COPY utils/dataset_manifest/requirements.txt /tmp/utils/dataset_manifest/requirements.txt + +# Since we're using pip-compile-multi, each dependency can only be listed in +# one requirements file. In the case of PyAV, that should be +# `dataset_manifest/requirements.txt`. Make sure it's actually there, +# and then remove everything else. +RUN grep -q '^av==' /tmp/utils/dataset_manifest/requirements.txt +RUN sed -i '/^av==/!d' /tmp/utils/dataset_manifest/requirements.txt + +RUN --mount=type=cache,target=/root/.cache/pip/http-v2 \ + python3 -m pip wheel --no-binary=av \ + -r /tmp/utils/dataset_manifest/requirements.txt \ + -w /tmp/wheelhouse + +# This stage builds wheels for all dependencies (except PyAV) +FROM build-image-base AS build-image + +COPY cvat/requirements/ /tmp/cvat/requirements/ +COPY utils/dataset_manifest/requirements.txt /tmp/utils/dataset_manifest/requirements.txt + +# Exclude av from the requirements file +RUN sed -i '/^av==/d' /tmp/utils/dataset_manifest/requirements.txt + +ARG CVAT_CONFIGURATION="production" + +# https://github.com/SAML-Toolkits/python3-saml#note +# Building from source is recommended for lxml and xmlsec to avoid libxml2 version conflicts +RUN --mount=type=cache,target=/root/.cache/pip/http-v2 \ + DATUMARO_HEADLESS=1 python3 -m pip wheel --no-deps --no-binary lxml,xmlsec \ + -r /tmp/cvat/requirements/${CVAT_CONFIGURATION}.txt \ + -w /tmp/wheelhouse + +FROM golang:1.26.4 AS build-smokescreen + +RUN git clone --filter=blob:none --no-checkout https://github.com/stripe/smokescreen.git +RUN cd smokescreen && git checkout eb1ac09 && go build -o /tmp/smokescreen + +FROM ${BASE_IMAGE} + +ARG http_proxy +ARG https_proxy +ARG no_proxy +ARG socks_proxy +ARG TZ="Etc/UTC" + +ENV TERM=xterm \ + http_proxy=${http_proxy} \ + https_proxy=${https_proxy} \ + no_proxy=${no_proxy} \ + socks_proxy=${socks_proxy} \ + LANG='C.UTF-8' \ + LC_ALL='C.UTF-8' \ + TZ=${TZ} + +ARG USER="django" +ARG CVAT_CONFIGURATION="production" +ENV DJANGO_SETTINGS_MODULE="cvat.settings.${CVAT_CONFIGURATION}" + +# Install necessary apt packages +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ + adduser \ + bzip2 \ + ca-certificates \ + curl \ + git \ + libgl1 \ + libgomp1 \ + libldap2 \ + libmp3lame0 \ + libpython3.12t64 \ + libsasl2-2 \ + libxml2 \ + libxmlsec1 \ + libxmlsec1-openssl \ + nginx \ + p7zip-full \ + poppler-utils \ + python3 \ + python3-venv \ + supervisor \ + tzdata \ + unrar \ + wait-for-it \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata && \ + rm -rf /var/lib/apt/lists/* + +# Install smokescreen +COPY --from=build-smokescreen /tmp/smokescreen /usr/local/bin/smokescreen + +# Add a non-root user +ENV USER=${USER} +ENV HOME /home/${USER} +RUN deluser --remove-home ubuntu && \ + adduser --uid=1000 --shell /bin/bash --disabled-password --gecos "" ${USER} + +ARG CLAM_AV="no" +RUN if [ "$CLAM_AV" = "yes" ]; then \ + apt-get update && \ + apt-get --no-install-recommends install -yq \ + clamav \ + libclamunrar && \ + sed -i 's/ReceiveTimeout 30/ReceiveTimeout 300/g' /etc/clamav/freshclam.conf && \ + freshclam && \ + chown -R ${USER}:${USER} /var/lib/clamav && \ + rm -rf /var/lib/apt/lists/*; \ + fi + +# Install wheels from the build image +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin:${PATH}" +ARG PIP_DISABLE_PIP_VERSION_CHECK=1 + +RUN --mount=type=bind,from=build-image,source=/tmp/wheelhouse,target=/mnt/wheelhouse \ + --mount=type=bind,from=build-image-av,source=/tmp/wheelhouse,target=/mnt/wheelhouse-av \ + python -m pip install --no-index /mnt/wheelhouse/*.whl /mnt/wheelhouse-av/*.whl + +ENV NUMPROCS=1 +COPY --from=build-image-av /opt/ffmpeg/lib /usr/lib + +# These variables are required for supervisord substitutions in files +# This library allows remote python debugging with VS Code +ARG CVAT_DEBUG_ENABLED +RUN if [ "${CVAT_DEBUG_ENABLED}" = 'yes' ]; then \ + python3 -m pip install --no-cache-dir debugpy; \ + fi + +# Removing pip due to security reasons. See: https://scout.docker.com/vulnerabilities/id/CVE-2018-20225 +# The vulnerability is dubious and we don't use pip at runtime, but some vulnerability scanners mark it as a high vulnerability, +# and it was decided to remove pip from the final image +RUN python -m pip uninstall -y pip + +# Install and initialize CVAT, copy all necessary files +COPY cvat/nginx.conf /etc/nginx/nginx.conf +COPY --chown=${USER} supervisord/ ${HOME}/supervisord +COPY --chown=${USER} backend_entrypoint.d/ ${HOME}/backend_entrypoint.d +COPY --chown=${USER} manage.py rqscheduler.py backend_entrypoint.sh wait_for_deps.sh ${HOME}/ +COPY --chown=${USER} utils/ ${HOME}/utils +COPY --chown=${USER} cvat/ ${HOME}/cvat +COPY --chown=${USER} components/analytics/clickhouse/init.py ${HOME}/components/analytics/clickhouse/init.py + +ARG COVERAGE_PROCESS_START +RUN if [ "${COVERAGE_PROCESS_START}" ]; then \ + echo "import coverage; coverage.process_startup()" > /opt/venv/lib/python3.12/site-packages/coverage_subprocess.pth; \ + fi + +# RUN all commands below as 'django' user. +# Use numeric UID/GID so that the image is compatible with the Kubernetes runAsNonRoot setting. +USER 1000:1000 +WORKDIR ${HOME} + +RUN mkdir -p data share keys logs /tmp/supervisord /tmp/cvat static + +EXPOSE 8080 +ENTRYPOINT ["./backend_entrypoint.sh"] diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 0000000..55ee168 --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,21 @@ +FROM cvat/server:local + +ENV DJANGO_SETTINGS_MODULE=cvat.settings.testing +USER root + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \ + build-essential \ + python3-dev \ + && \ + rm -rf /var/lib/apt/lists/*; + +COPY cvat/requirements/ /tmp/cvat/requirements/ +COPY utils/dataset_manifest/requirements.txt /tmp/utils/dataset_manifest/requirements.txt + +RUN python3 -m ensurepip +RUN DATUMARO_HEADLESS=1 python3 -m pip install --no-cache-dir -r /tmp/cvat/requirements/testing.txt + +COPY .coveragerc . + +ENTRYPOINT [] diff --git a/Dockerfile.ui b/Dockerfile.ui new file mode 100644 index 0000000..9d7e3fb --- /dev/null +++ b/Dockerfile.ui @@ -0,0 +1,40 @@ +FROM node:lts-slim AS cvat-ui + +ENV TERM=xterm \ + LANG='C.UTF-8' \ + LC_ALL='C.UTF-8' + +# Install dependencies +RUN npm install -g corepack +COPY .yarnrc.yml /tmp/ +COPY package.json /tmp/ +COPY yarn.lock /tmp/ +COPY cvat-core/package.json /tmp/cvat-core/ +COPY cvat-canvas/package.json /tmp/cvat-canvas/ +COPY cvat-canvas3d/package.json /tmp/cvat-canvas3d/ +COPY cvat-ui/package.json /tmp/cvat-ui/ +COPY cvat-data/package.json /tmp/cvat-data/ + +# Install common dependencies +WORKDIR /tmp/ +RUN DISABLE_HUSKY=1 yarn install --immutable + +# Build source code +COPY cvat-data/ /tmp/cvat-data/ +COPY cvat-core/ /tmp/cvat-core/ +COPY cvat-canvas3d/ /tmp/cvat-canvas3d/ +COPY cvat-canvas/ /tmp/cvat-canvas/ +COPY cvat-ui/ /tmp/cvat-ui/ + +ARG CLIENT_PLUGINS +ARG SOURCE_MAPS_ENABLED=false + +RUN CLIENT_PLUGINS="${CLIENT_PLUGINS}" \ +SOURCE_MAPS_ENABLED="${SOURCE_MAPS_ENABLED}" yarn run build:cvat-ui + +FROM nginxinc/nginx-unprivileged:1.31.2-alpine3.23-slim + +# Replace default.conf configuration to remove unnecessary rules +COPY cvat-ui/react_nginx.conf /etc/nginx/conf.d/default.conf +COPY cvat-ui/robots.txt /usr/share/nginx/html/ +COPY --from=cvat-ui /tmp/cvat-ui/dist /usr/share/nginx/html/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9af4363 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (C) 2018-2022 Intel Corporation +Copyright (C) 2022-2025 CVAT.ai Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3835641 --- /dev/null +++ b/README.md @@ -0,0 +1,265 @@ +[![CVAT Community header](site/content/en/images/cvat_github_header.webp)](https://app.cvat.ai) +# CVAT: Computer Vision Annotation Tool + +[![Release][release-img]][release-url] +[![GitHub stars][stars-img]][stars-url] +[![License][license-img]][license-url] +[![CI][ci-img]][ci-url] +[![server pulls][docker-server-pulls-img]][docker-server-image-url] +[![ui pulls][docker-ui-pulls-img]][docker-ui-image-url] +[![CVAT Online][online-img]][online-url] +[![CVAT Enterprise][enterprise-img]][enterprise-url] +[![Status][status-img]][status-url] +[![Discord][discord-img]][discord-url] +[![Docs][docs-img]][docs-url] + +[Website](https://www.cvat.ai/) · +[Docs](https://docs.cvat.ai/docs/) · +[Changelog](https://www.cvat.ai/resources/changelog) · +[Tutorials](https://www.cvat.ai/resources/videos) · +[Academy](https://www.cvat.ai/resources/academy) · +[Blog](https://www.cvat.ai/resources/blog) + +## What is CVAT Community? + +**CVAT Community** is the free, self-hosted open-source edition of [CVAT](https://www.cvat.ai/) — one of +the most widely used data annotation platforms for building high-quality visual datasets for +computer vision and visual AI. +Since 2018, CVAT has become one of the best-known data annotation tools in computer vision, with a +large open-source community, millions of Docker pulls, and broad adoption across research and +production AI teams. + +CVAT Community supports image, video, and 3D annotation, dataset management, team collaboration, cloud storage +integration, developer-friendly SDKs and APIs, and gives your team full control over your data +and annotation infrastructure. +The platform serves as the foundation of +[CVAT Online](https://www.cvat.ai/pricing/cvat-online) and +[CVAT Enterprise](https://www.cvat.ai/enterprise), and is actively maintained by the CVAT engineering team. + +Why teams choose CVAT Community: + +- **Own your data:** Run entirely within your own infrastructure. No data leaves your environment. +- **AI-powered annotation:** Connect your own ML models for detection, segmentation, and tracking to speed up labeling. +- **Team collaboration:** Multi-user and multi-organization support with roles, task assignments, + and review workflows. +- **MIT-licensed core:** Use, modify, and distribute CVAT Community under the permissive MIT License. Some serverless +assets and dependencies may have separate licenses. +- **Production-grade:** The foundation of all CVAT commercial products — battle-tested at scale. +- **True open-source:** Transparent development, active community, on GitHub since 2018. + +This repository contains the source code and deployment assets for CVAT Community. + +For a fully managed setup, annotation services, or enterprise features, see +[CVAT Online](https://www.cvat.ai/pricing/cvat-online), +[CVAT Enterprise](https://www.cvat.ai/enterprise) and +[CVAT Labeling Services](https://www.cvat.ai/annotation-services). + +## Getting Started + +> 💡 Want to explore CVAT before deploying anything? +> **[Try CVAT Online (Free plan)](https://app.cvat.ai)** directly in your browser. +> Feature availability and usage limits vary by plan; see +> [CVAT Online pricing](https://www.cvat.ai/pricing/cvat-online) for details. + +### Installation + +**Prerequisites:** + +- [Docker Engine](https://docs.docker.com/engine/install/) +- [Docker Compose](https://docs.docker.com/compose/install/) +- [Git](https://git-scm.com/) + +> 💡 CVAT is primarily tested with Chromium-based browsers (Google Chrome, Microsoft Edge). +> Firefox may work with some caveats; Safari/WebKit is not supported. + +**1. Start the default stack** + +Clone the repository and launch the services. + +```bash +git clone https://github.com/cvat-ai/cvat +cd cvat + +# Optional: set your IP or domain +# export CVAT_HOST=your-ip-or-domain + +docker compose up -d +``` + +**2. Create an admin account** + +```bash +docker exec -it cvat_server bash -ic 'python3 ~/manage.py createsuperuser' +``` + +See the [Installation Guide](https://docs.cvat.ai/docs/administration/community/basics/installation/) for full +instructions and OS-specific setup. + +**3. Sign in and start labeling** + +- Open [http://localhost:8080](http://localhost:8080) (or your `CVAT_HOST`) in your browser. +- Log in with your superuser account. +- Create a project or task, upload your data (images, videos, or point clouds), and define labels to start annotating. + +Learn more about annotation tools and workflows in the [CVAT Documentation](https://docs.cvat.ai/docs/) or +take our free course – [CVAT Academy](https://www.cvat.ai/resources/academy). + +_For alternative deployments (AWS, Kubernetes, external PostgreSQL, backups, upgrades), see the [Deployment Guides](https://docs.cvat.ai/docs/administration/community/advanced/)._ + +## Key Capabilities + +- **[Manual & Auto-labeling](https://docs.cvat.ai/docs/annotation/manual-annotation/):** Annotate images, videos, and + 3D point clouds with bounding boxes, polygons, masks, keypoints, cuboids, tags, and more. Speed up labeling + by connecting your own models for automatic annotation. +- **[Task Management](https://docs.cvat.ai/docs/workspace/):** Organize datasets into projects, split them into tasks + and jobs, assign work to annotators, and track progress in real time. +- **[Collaboration](https://docs.cvat.ai/docs/account_management/user-roles/):** Create organizations, invite teammates, + assign roles, and collaborate on annotations with comments and issues. +- **[Quality Control](https://docs.cvat.ai/docs/qa-analytics/manual-qa/):** Review annotations, flag issues, compare + results across annotators with consensus, and run Ground Truth and Honeypot checks through the server API. +- **[Analytics](https://docs.cvat.ai/docs/administration/community/advanced/analytics/):** Monitor user activity, + working time by job, events, and server logs with Grafana dashboards. +- **[Data Ops & Integrations](https://docs.cvat.ai/docs/dataset_management/export-datasets/):** Export/import in 20+ + formats (COCO, YOLO, Pascal VOC, KITTI, etc.), connect to cloud storage (S3, Azure, Google Cloud), and automate + via REST API and Python SDK. + +Advanced capabilities such as advanced project analytics, quality control UI, built-in auto-labeling with SAM 2 + and SAM 3, AI agents, SSO, and more are available in [CVAT Online](https://www.cvat.ai/pricing/cvat-online) + paid plans (Solo, Team) and [CVAT Enterprise](https://www.cvat.ai/enterprise). + +## Developer Tools + +CVAT is designed for automation. Beyond the Web UI, you can integrate it into your pipelines using: + +- [Python SDK](https://docs.cvat.ai/docs/api_sdk/sdk/): install with `pip install cvat-sdk` and automate task creation, +uploads, and exports from Python. +- [Command line tool](https://docs.cvat.ai/docs/api_sdk/cli/): install with `pip install cvat-cli` +and script common CVAT workflows from the terminal. +- [REST API](https://docs.cvat.ai/docs/api_sdk/api/): full programmatic control over CVAT. + +## Data and Formats + +CVAT Community supports image, video, and 3D (point cloud) annotation workflows. You can move data in and out using 20+ +industry-standard formats: CVAT (XML), COCO (JSON), YOLO (TXT), Ultralytics YOLO (TXT/YAML), Pascal VOC (XML), +KITTI (TXT), MOT (TXT), and more. + +[Full list of supported formats.](https://docs.cvat.ai/docs/dataset_management/formats/) + +## ML and AI Models + +CVAT Community supports automatic annotation via pre-built serverless models powered by Nuclio, +covering detection, segmentation, pose estimation, and tracking: + +| Model | Framework | Type | +| --- | --- | --- | +| [Segment Anything (SAM)](https://github.com/cvat-ai/cvat/tree/develop/serverless/pytorch/facebookresearch/sam/nuclio) | PyTorch | Interactor | +| [Inside-Outside Guidance (IOG)](https://github.com/cvat-ai/cvat/tree/develop/serverless/pytorch/shiyinzhang/iog/nuclio) | PyTorch | Interactor | +| [RetinaNet R101](https://github.com/cvat-ai/cvat/tree/develop/serverless/pytorch/facebookresearch/detectron2/retinanet_r101/nuclio) | PyTorch | Detector | +| [HRNet32 Whole Body Pose](https://github.com/cvat-ai/cvat/tree/develop/serverless/pytorch/mmpose/hrnet32/nuclio) | PyTorch | Pose Estimation | +| [TransT](https://github.com/cvat-ai/cvat/tree/develop/serverless/pytorch/dschoerk/transt/nuclio) | PyTorch | Tracker | +| [YOLO v7](https://github.com/cvat-ai/cvat/tree/develop/serverless/onnx/WongKinYiu/yolov7/nuclio) | ONNX | Detector | +| [Mask RCNN Inception ResNet v2](https://github.com/cvat-ai/cvat/tree/develop/serverless/openvino/omz/public/mask_rcnn_inception_resnet_v2_atrous_coco/nuclio) | OpenVINO | Detector | +| [Face Detection 0205](https://github.com/cvat-ai/cvat/tree/develop/serverless/openvino/omz/intel/face-detection-0205/nuclio) | OpenVINO | Detector | +| [Faster RCNN Inception v2](https://github.com/cvat-ai/cvat/tree/develop/serverless/tensorflow/faster_rcnn_inception_v2_coco/nuclio) | TensorFlow | Detector | + +To enable automatic annotation, add the serverless component to your deployment: + +```bash +docker compose -f docker-compose.yml -f components/serverless/docker-compose.serverless.yml up -d +``` + +This starts the serverless infrastructure. To make models available in CVAT, install `nuctl` and deploy +the functions you need, for example SAM or YOLO, as described in the [Automatic Annotation Guide](https://docs.cvat.ai/docs/annotation/auto-annotation/automatic-annotation/). + +## Which CVAT edition should I choose? + +- **CVAT Online**: the fastest way to try CVAT and start labeling without deployment. Use it to evaluate CVAT in +the browser, explore managed features, and move to cost-efficient paid plans when you need more capacity or team +workflows. +- **CVAT Community**: the MIT-licensed self-hosted edition for teams that want to run CVAT themselves, customize the +stack, and control their infrastructure. +- **CVAT Enterprise**: for organizations that need CVAT in their own cloud or internal environment, enterprise support, +security controls such as SSO, paid platform features, and SLAs. +- **Labeling Services**: for teams that want to outsource annotation work to CVAT.ai’s experienced labeling team instead +of building an internal labeling operation. Customers get trial access to CVAT Online during the project. + +For detailed plan limits and feature availability, see [CVAT Online pricing](https://www.cvat.ai/pricing/cvat-online), + [CVAT Enterprise](https://www.cvat.ai/enterprise), and [Labeling Services](https://www.cvat.ai/annotation-services). + +## Support + +- **Usage questions:** ask the community on [Discord](https://discord.com/invite/fNR3eXfk6C) or +Stack Overflow with the `cvat` tag. +- **Bugs and feature requests:** use [GitHub Issues](https://github.com/cvat-ai/cvat/issues). +- **FAQ:** [Installation, upgrades, troubleshooting](https://docs.cvat.ai/docs/faq/). + +For dedicated support, SLAs, or advanced deployments, consider [CVAT Enterprise](https://www.cvat.ai/enterprise). + +## Contributing + +We welcome all contributions: bug reports, documentation fixes, integrations, and code. + +- If you'd like to contribute to CVAT, please refer to our + [contribution documentation](https://docs.cvat.ai/docs/contributing/). +- For bug reports or feature requests, please use the [GitHub Issues](https://github.com/cvat-ai/cvat/issues) tracker. + +## Security + +- Please review our [Security Policy](https://github.com/cvat-ai/cvat/security/policy) before reporting vulnerabilities. +- For sensitive issues, contact: [secure@cvat.ai](mailto:secure@cvat.ai). + +## License + +CVAT Community is released under the MIT License. + +- Code in `/serverless` is also MIT-licensed, but may use third-party assets under separate licenses (including + non-commercial). Review those licenses before use. +- This software uses FFmpeg libraries under LGPL/GPL. See the Dockerfile and + [FFmpeg legal info](https://www.ffmpeg.org/legal.html) for details. + +## Additional Resources + +For the latest product releases, feature walkthroughs, and all things CVAT see: + + + + + + + +
CVAT BlogCVAT AcademyCase StudiesYouTubeLinkedIn
+ + + +[ci-img]: https://github.com/cvat-ai/cvat/actions/workflows/main.yml/badge.svg?branch=develop +[ci-url]: https://github.com/cvat-ai/cvat/actions + +[docs-img]: https://img.shields.io/badge/docs-docs.cvat.ai-blue?style=flat-square +[docs-url]: https://docs.cvat.ai + +[online-img]: https://img.shields.io/badge/CVAT%20Online-app.cvat.ai-success?style=flat-square +[online-url]: https://app.cvat.ai + +[release-img]: https://img.shields.io/github/v/release/cvat-ai/cvat?style=flat-square +[release-url]: https://github.com/cvat-ai/cvat/releases + +[license-img]: https://img.shields.io/github/license/cvat-ai/cvat?style=flat-square +[license-url]: https://github.com/cvat-ai/cvat/blob/develop/LICENSE + +[stars-img]: https://img.shields.io/github/stars/cvat-ai/cvat?style=flat-square +[stars-url]: https://github.com/cvat-ai/cvat/stargazers + +[status-img]: https://uptime.betterstack.com/status-badges/v2/monitor/1yl3h.svg +[status-url]: https://status.cvat.ai + +[enterprise-img]: https://img.shields.io/badge/CVAT%20Enterprise-cvat.ai-orange?style=flat-square +[enterprise-url]: https://www.cvat.ai/enterprise + +[docker-server-pulls-img]: https://img.shields.io/docker/pulls/cvat/server.svg?style=flat-square&label=server%20pulls +[docker-server-image-url]: https://hub.docker.com/r/cvat/server + +[docker-ui-pulls-img]: https://img.shields.io/docker/pulls/cvat/ui.svg?style=flat-square&label=UI%20pulls +[docker-ui-image-url]: https://hub.docker.com/r/cvat/ui + +[discord-img]: https://img.shields.io/discord/1000789942802337834?label=discord +[discord-url]: https://discord.gg/fNR3eXfk6C diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..d2d234d --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`cvat-ai/cvat` +- 原始仓库：https://github.com/cvat-ai/cvat +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..980c231 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,19 @@ +# Security Policy + +## Supported Versions + +At the moment only the latest release is supported. When you report a security issue, +be sure it can be reproduced in the supported version. + +## Reporting a Vulnerability + +If you have information about a security issue or vulnerability in the product, please +send an e-mail to [secure@cvat.ai](mailto:secure+github@cvat.ai). + +Please provide as much information as possible, including: + +- The products and versions affected +- Detailed description of the vulnerability +- Information on known exploits +- A member of the CVAT.ai Product Security Team will review your e-mail and contact you to + collaborate on resolving the issue. diff --git a/_typos.toml b/_typos.toml new file mode 100644 index 0000000..44c9a54 --- /dev/null +++ b/_typos.toml @@ -0,0 +1,47 @@ +files.extend-exclude = [ + "ai-models/agents_deployment/transformers/supported_models", + "cvat-ui/src/assets/opencv_4.8.0.js", + "cvat-data/src/ts/3rdparty/*.js", + "site/themes/docsy/", + "*.log", + "*.min.js", + "*.patch", +] +default.extend-ignore-re = [ + # test passwords + "\"md5.*?\"", + + # Line ignore with trailing "# spellchecker:disable-line" + # or trailing "// spellchecker:disable-line" + "(?Rm)^.*(#|//)\\s*spellchecker:disable-line$", + + # Line block with "# spellchecker:" + # or "// spellchecker:" + "(?s)(#|//)\\s*spellchecker:off.*?\\n\\s*(#|//)\\s*spellchecker:on" + # Taken from https://github.com/crate-ci/typos/blob/master/docs/reference.md#example-configurations + +] + +[default.extend-words] +AVOD = "AVOD" +NAM = "NAM" +blok = "block" +cheeck = "check" +configurate = "configure" +convertation = "conversion" +convertor = "converter" +cubiod = "cuboid" +cuboiud = "cuboid" +cuccessfully = "successfully" +firts = "first" +leafs = "leafs" +nd = "nd" +onject = "" +perameters = "parameters" +segway = "segway" +splitted = "splitted" +succefull = "successful" +tesk = "task" +tpic = "tpic" +trak = "trak" +typ = "type" diff --git a/ai-models/README.md b/ai-models/README.md new file mode 100644 index 0000000..f60806f --- /dev/null +++ b/ai-models/README.md @@ -0,0 +1,3 @@ +### This folder was moved to a separate repository: + +#### diff --git a/backend_entrypoint.d/cvat.conf b/backend_entrypoint.d/cvat.conf new file mode 100644 index 0000000..f743603 --- /dev/null +++ b/backend_entrypoint.d/cvat.conf @@ -0,0 +1,12 @@ +( + ["annotation"]="smokescreen" + ["chunks"]="smokescreen" + ["consensus"]="" + ["export"]="smokescreen" + ["import"]="smokescreen clamav" + ["quality_reports"]="" + ["cleaning"]="rqscheduler" + ["webhooks"]="smokescreen" + ["notifications"]="" + ["server"]="smokescreen clamav" +) diff --git a/backend_entrypoint.sh b/backend_entrypoint.sh new file mode 100755 index 0000000..442c708 --- /dev/null +++ b/backend_entrypoint.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +set -eu + +fail() { + printf >&2 "%s: %s\n" "$0" "$1" + exit 1 +} + +wait_for_db() { + wait-for-it "${CVAT_POSTGRES_HOST}:${CVAT_POSTGRES_PORT:-5432}" -t 0 +} + +wait_for_redis_inmem() { + wait-for-it "${CVAT_REDIS_INMEM_HOST}:${CVAT_REDIS_INMEM_PORT:-6379}" -t 0 +} + +wait_for_clickhouse() { + wait-for-it "${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT:-8123}" -t 0 +} + +account_for_internal_proxy() { + CVAT_NUM_PROXIES="${CVAT_NUM_PROXIES:-0}" + + if ! [[ "$CVAT_NUM_PROXIES" =~ ^[0-9]+$ ]]; then + fail "CVAT_NUM_PROXIES must be a non-negative integer" + fi + + # The user-facing setting counts proxies before the backend container. + # Add the internal nginx hop that proxies requests to uvicorn. + export CVAT_NUM_PROXIES=$((CVAT_NUM_PROXIES + 1)) +} + +cmd_bash() { + exec bash "$@" +} + +cmd_init() { + wait_for_db + ~/manage.py migrate + + wait_for_redis_inmem + ~/manage.py migrateredis + ~/manage.py syncperiodicjobs + + if [[ "${CVAT_ANALYTICS:-0}" == "1" ]]; then + wait_for_clickhouse + python components/analytics/clickhouse/init.py + fi +} + +_load_component_config() { + declare -gA merged_config=() + for config_file in ~/backend_entrypoint.d/*.conf; do + declare -A config=$(cat $config_file) + for key in "${!config[@]}"; do + if [[ -n ${merged_config[$key]+_} ]]; then + fail "Duplicated component definition: $key" + fi + merged_config[$key]=${config[$key]} + done + unset config + done +} + +_get_includes() { + extra_configs=() + for key in "$@"; do + if [[ -z ${merged_config[$key]+_} ]]; then + fail "Unexpected component: $key" + fi + + for include in ${merged_config["$key"]}; do + if ! [[ ${extra_configs[@]} =~ $include ]] && \ + ( ! [[ "$include" == "clamav" ]] || [[ "${CLAM_AV:-}" == "yes" ]] ); then + extra_configs+=("$include") + fi + done + done + + if [ ${#extra_configs[@]} -gt 0 ]; then + printf 'reusable/%s.conf ' "${extra_configs[@]}" + fi +} + +_get_reusable_includes() { + extra_configs=() + for include in "$@"; do + if ! [ -r "$HOME/supervisord/reusable/$include.conf" ]; then + fail "Unexpected supervisor include: $include" + fi + + if ! [[ ${extra_configs[@]} =~ $include ]]; then + extra_configs+=("$include") + fi + done + + if [ ${#extra_configs[@]} -gt 0 ]; then + printf 'reusable/%s.conf ' "${extra_configs[@]}" + fi +} + +cmd_run() { + if [ "$#" -eq 0 ]; then + fail "run: at least 1 argument is expected" + fi + + component="$1" + _load_component_config + + case "$component" in + server|worker|worker-pool|nginx) ;; + *) fail "Unexpected run component: $component" ;; + esac + + if [ "$component" = "nginx" ]; then + exec supervisord -c "supervisord/nginx.conf" + fi + + if [ "$component" = "server" ]; then + account_for_internal_proxy + ~/manage.py collectstatic --no-input + fi + + wait_for_db + + echo "waiting for migrations to complete..." + while ! ~/manage.py migrate --check; do + sleep 10 + done + + wait_for_redis_inmem + echo "waiting for Redis migrations to complete..." + while ! ~/manage.py migrateredis --check; do + sleep 10 + done + + supervisord_includes="" + postgres_app_name="cvat:$component" + if [ "$component" = "server" ]; then + supervisord_includes="$(_get_includes "server")$(_get_reusable_includes "${@:2}")" + elif [ "$component" = "worker" ] || [ "$component" = "worker-pool" ]; then + if [ "$#" -eq 1 ]; then + fail "run worker: expected at least 1 queue name" + fi + + queues=() + extra_flags=() + for arg in "${@:2}"; do + if [[ "$arg" == --* ]]; then + extra_flags+=("$arg") + else + queues+=("$arg") + fi + done + + if [ ${#queues[@]} -eq 0 ]; then + fail "run worker: expected at least 1 queue name" + fi + + queue_list="${queues[*]}" + echo "Workers to run: $queue_list" + if [ ${#extra_flags[@]} -gt 0 ]; then + echo "Extra rqworker flags: ${extra_flags[*]}" + fi + export CVAT_QUEUES=$queue_list + export CVAT_RQWORKER_EXTRA_FLAGS="${extra_flags[*]:-}" + + postgres_app_name+=":${queue_list// /+}" + + supervisord_includes=$(_get_includes "${queues[@]}") + fi + echo "Additional supervisor configs that will be included: $supervisord_includes" + + export CVAT_POSTGRES_APPLICATION_NAME=$postgres_app_name + export CVAT_SUPERVISORD_INCLUDES=$supervisord_includes + + exec supervisord -c "supervisord/$component.conf" +} + +if [ $# -eq 0 ]; then + echo >&2 "$0: at least one subcommand required" + echo >&2 "" + echo >&2 "available subcommands:" + echo >&2 " bash " + echo >&2 " init" + echo >&2 " run server [additional components]" + echo >&2 " run nginx" + echo >&2 " run worker " + exit 1 +fi + +for init_script in /etc/cvat/init.d/*; do + if [ -r "$init_script" ]; then + . "$init_script" + fi +done + +while [ $# -ne 0 ]; do + if [ "$(type -t "cmd_$1")" != "function" ]; then + fail "unknown subcommand: $1" + fi + + cmd_name="$1" + + shift + + "cmd_$cmd_name" "$@" +done diff --git a/changelog.d/20260527_164323_roman_rm_scheme_autodetection.md b/changelog.d/20260527_164323_roman_rm_scheme_autodetection.md new file mode 100644 index 0000000..c9a3ea5 --- /dev/null +++ b/changelog.d/20260527_164323_roman_rm_scheme_autodetection.md @@ -0,0 +1,5 @@ +### Removed + +- \[SDK, CLI\] Removed server schema autodetection, previously deprecated + in v2.36.0 + () diff --git a/changelog.d/20260619_082936_sekachev.bs_roi.md b/changelog.d/20260619_082936_sekachev.bs_roi.md new file mode 100644 index 0000000..2727b58 --- /dev/null +++ b/changelog.d/20260619_082936_sekachev.bs_roi.md @@ -0,0 +1,4 @@ +### Added + +- Added region of interest support for automatic annotation detector and interactor functions + () diff --git a/changelog.d/20260623_000000_andrei_sdk_list_filters.md b/changelog.d/20260623_000000_andrei_sdk_list_filters.md new file mode 100644 index 0000000..cbd0cc0 --- /dev/null +++ b/changelog.d/20260623_000000_andrei_sdk_list_filters.md @@ -0,0 +1,6 @@ +### Added + +- \[SDK\] `list()` methods on high-level proxy classes now accept filter parameters + via a composable `F` field DSL and keyword lookups, + which are compiled into the server-side `filter` query parameter automatically + () diff --git a/changelog.d/20260625_000000_andrei_sdk_persistent_auth.md b/changelog.d/20260625_000000_andrei_sdk_persistent_auth.md new file mode 100644 index 0000000..1838eb6 --- /dev/null +++ b/changelog.d/20260625_000000_andrei_sdk_persistent_auth.md @@ -0,0 +1,8 @@ +### Added + +- \[SDK\] Persistent authentication support with saved profiles: + `make_client_from_profile`, `make_client_from_cli`, and + `resolve_server_host` helpers let SDK users save server + credential profiles + to disk and instantiate a `Client` from a profile name without re-entering + credentials + () diff --git a/changelog.d/20260630_000000_klakhov_fix_request_null_status.md b/changelog.d/20260630_000000_klakhov_fix_request_null_status.md new file mode 100644 index 0000000..c0a0fbf --- /dev/null +++ b/changelog.d/20260630_000000_klakhov_fix_request_null_status.md @@ -0,0 +1,4 @@ +### Fixed + +- Requests API no longer returns a `null` status when a job's status cannot be + read from Redis (e.g. the job expired) () diff --git a/changelog.d/20260630_123848_klakhov_fix_draw_circles_undefined.md b/changelog.d/20260630_123848_klakhov_fix_draw_circles_undefined.md new file mode 100644 index 0000000..b3a4a83 --- /dev/null +++ b/changelog.d/20260630_123848_klakhov_fix_draw_circles_undefined.md @@ -0,0 +1,4 @@ +### Fixed + +- Fix snap to contour crashes when drawing stops (drawCircles of undefined / array of null) + () diff --git a/changelog.d/20260630_162121_roman.md b/changelog.d/20260630_162121_roman.md new file mode 100644 index 0000000..a82746d --- /dev/null +++ b/changelog.d/20260630_162121_roman.md @@ -0,0 +1,5 @@ +### Fixed + +- Fixed task creation from images hosted on Azure Blob Storage when + at least one image's header doesn't fit into 2047 bytes + () diff --git a/changelog.d/20260701_000000_andrei_task_meta_unbound_media.md b/changelog.d/20260701_000000_andrei_task_meta_unbound_media.md new file mode 100644 index 0000000..b628aa0 --- /dev/null +++ b/changelog.d/20260701_000000_andrei_task_meta_unbound_media.md @@ -0,0 +1,5 @@ +### Fixed + +- Requesting task metadata (`GET /api/tasks//data/meta`) before any + data was uploaded to the task now returns 400 with a clear message. + () diff --git a/changelog.d/20260701_150914_klakhov_fix_mask_occluded.md b/changelog.d/20260701_150914_klakhov_fix_mask_occluded.md new file mode 100644 index 0000000..e1cf128 --- /dev/null +++ b/changelog.d/20260701_150914_klakhov_fix_mask_occluded.md @@ -0,0 +1,4 @@ +### Fixed + +- Client error `Cannot read properties of undefined (reading 'occluded')` while copying masks + () diff --git a/changelog.d/20260701_164337_mzhiltsov_add_annotation_source_validation.md b/changelog.d/20260701_164337_mzhiltsov_add_annotation_source_validation.md new file mode 100644 index 0000000..488be2a --- /dev/null +++ b/changelog.d/20260701_164337_mzhiltsov_add_annotation_source_validation.md @@ -0,0 +1,6 @@ +### Fixed + +- \[Server API\] Added missing input validation for the `source` field of annotations + in the `/api/tasks//annotations` and `/api/jobs//annotations` endpoints. + The existing invalid values are returned unmodified in the server responses. + () diff --git a/changelog.d/20260701_170000_andrei_remove_invitation_patch.md b/changelog.d/20260701_170000_andrei_remove_invitation_patch.md new file mode 100644 index 0000000..0442962 --- /dev/null +++ b/changelog.d/20260701_170000_andrei_remove_invitation_patch.md @@ -0,0 +1,4 @@ +### Removed + +- Removed the unused `PATCH /api/invitations/{key}` Server API operation. + () diff --git a/changelog.d/20260702_000000_codex_track_multiple_rectangles.md b/changelog.d/20260702_000000_codex_track_multiple_rectangles.md new file mode 100644 index 0000000..f1b555f --- /dev/null +++ b/changelog.d/20260702_000000_codex_track_multiple_rectangles.md @@ -0,0 +1,4 @@ +### Fixed + +- Fixed AI tools tracking when starting tracking from multiple drawn rectangles + () diff --git a/changelog.d/20260702_112023_klakhov_fix_reading_undefined_status.md b/changelog.d/20260702_112023_klakhov_fix_reading_undefined_status.md new file mode 100644 index 0000000..58af1dd --- /dev/null +++ b/changelog.d/20260702_112023_klakhov_fix_reading_undefined_status.md @@ -0,0 +1,5 @@ +### Fixed + +- Client error thrown + when listing lambda functions failed (e.g. network + error or timeout) () diff --git a/changelog.d/20260702_131449_aleksey.zinovyev_pagination_reset.md b/changelog.d/20260702_131449_aleksey.zinovyev_pagination_reset.md new file mode 100644 index 0000000..dc217db --- /dev/null +++ b/changelog.d/20260702_131449_aleksey.zinovyev_pagination_reset.md @@ -0,0 +1,4 @@ +### Fixed + +- Fix pagination being reset when refreshing or going "back" to a resources page + () diff --git a/changelog.d/20260702_140645_aleksey.zinovyev_clickable_loading_preview.md b/changelog.d/20260702_140645_aleksey.zinovyev_clickable_loading_preview.md new file mode 100644 index 0000000..e241a89 --- /dev/null +++ b/changelog.d/20260702_140645_aleksey.zinovyev_clickable_loading_preview.md @@ -0,0 +1,4 @@ +### Fixed + +- Make loading previews on project/job cards clickable to allow opening the resource regardless of the preview state + () diff --git a/changelog.d/20260702_171804_aleksey.zinovyev_request_card_layout.md b/changelog.d/20260702_171804_aleksey.zinovyev_request_card_layout.md new file mode 100644 index 0000000..61954c5 --- /dev/null +++ b/changelog.d/20260702_171804_aleksey.zinovyev_request_card_layout.md @@ -0,0 +1,4 @@ +### Fixed + +- Prevent requests cards content overflow + () diff --git a/changelog.d/20260703_105329_andrey_optimize_init_tracks_from_db.md b/changelog.d/20260703_105329_andrey_optimize_init_tracks_from_db.md new file mode 100644 index 0000000..6fb3b99 --- /dev/null +++ b/changelog.d/20260703_105329_andrey_optimize_init_tracks_from_db.md @@ -0,0 +1,4 @@ +### Fixed + +- Improved performance of annotation responses. + () diff --git a/changelog.d/fragment.j2 b/changelog.d/fragment.j2 new file mode 100644 index 0000000..0016248 --- /dev/null +++ b/changelog.d/fragment.j2 @@ -0,0 +1,4 @@ +### {{ config.categories | join('|') }} + +- Describe your change here... + () diff --git a/changelog.d/scriv.ini b/changelog.d/scriv.ini new file mode 100644 index 0000000..752a27f --- /dev/null +++ b/changelog.d/scriv.ini @@ -0,0 +1,6 @@ +[scriv] +categories = Added, Changed, Deprecated, Removed, Fixed, Security +entry_title_template = \[{{ version }}\] - {{ date.strftime('%%Y-%%m-%%d') }} +format = md +md_header_level = 2 +new_fragment_template = file: fragment.j2 diff --git a/components/analytics/clickhouse/README.md b/components/analytics/clickhouse/README.md new file mode 100644 index 0000000..e7d9c2d --- /dev/null +++ b/components/analytics/clickhouse/README.md @@ -0,0 +1,37 @@ + + +# Clickhouse DB migrations + +This directory contains migrations for the Clickhouse DB used by CVAT. + +## Implementation details + +The Clickhouse documentation explains options to customize DB loading here: + +In the default Docker image, Clickhouse only runs these files if the DB is not initialized. +This is not the desired behavior for us, because it doesn't allow us to add new migrations +added after the DB is initialized. Instead, we run the migrations on each start using a custom +script. Therefore **make sure the migrations can be safely called multiple times** +on an existing DB. + +The directory contains scripts for the Clickhouse instance: +- `init.py`: initializes the DB and applies required migrations + +## Adding a migration + +To add a migration, create a new `migration_NNN_` function in the `init.py` script +and add it to the list of migrations. + +Recommendations on the migrations: +- **Make sure the migrations can be safely called multiple times** + and do not fail on repeated calls on the initialized DB. + Typically, this can be achieved by using "IF NOT EXISTS" at some point. +- Avoid heavy data migrations as much as possible. In most cases, changing only the table + columns should be enough and you won't need a data migration. + +As follows from the recommendations above, the current implementation is limited +to simple migrations only. diff --git a/components/analytics/clickhouse/init.py b/components/analytics/clickhouse/init.py new file mode 100644 index 0000000..f587a3e --- /dev/null +++ b/components/analytics/clickhouse/init.py @@ -0,0 +1,117 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import os +import sys +import textwrap +from argparse import ArgumentParser + +import clickhouse_connect +import clickhouse_connect.driver.client + +CLICKHOUSE_HOST_ENV_VAR = "CLICKHOUSE_HOST" +CLICKHOUSE_PORT_ENV_VAR = "CLICKHOUSE_PORT" +CLICKHOUSE_DB_ENV_VAR = "CLICKHOUSE_DB" +CLICKHOUSE_USER_ENV_VAR = "CLICKHOUSE_USER" # nosec +CLICKHOUSE_PASSWORD_ENV_VAR = "CLICKHOUSE_PASSWORD" # nosec + + +def clear_db(client: clickhouse_connect.driver.client.Client): + client.query(f"TRUNCATE DATABASE IF EXISTS {client.database};") + + +def create_db(client: clickhouse_connect.driver.client.Client, db_name: str): + client.query(f"CREATE DATABASE IF NOT EXISTS {db_name};") + + +def migration_000_initial(client: clickhouse_connect.driver.client.Client): + client.query(textwrap.dedent("""\ + CREATE TABLE IF NOT EXISTS events + ( + `scope` String NOT NULL, + `obj_name` String NULL, + `obj_id` UInt64 NULL, + `obj_val` String NULL, + `source` String NOT NULL, + `timestamp` DateTime64(3, 'Etc/UTC') NOT NULL, + `count` UInt16 NULL, + `duration` UInt32 DEFAULT toUInt32(0), + `project_id` UInt64 NULL, + `task_id` UInt64 NULL, + `job_id` UInt64 NULL, + `user_id` UInt64 NULL, + `user_name` String NULL, + `user_email` String NULL, + `org_id` UInt64 NULL, + `org_slug` String NULL, + `payload` String NULL + ) + ENGINE = MergeTree + PARTITION BY toYYYYMM(timestamp) + ORDER BY (timestamp) + SETTINGS index_granularity = 8192; + """)) + + +def migration_001_add_access_token(client: clickhouse_connect.driver.client.Client): + client.query(textwrap.dedent("""\ + ALTER TABLE events + ADD COLUMN IF NOT EXISTS + `access_token_id` Nullable(UInt64) DEFAULT NULL + """)) + + +def migration_002_add_remote_addr(client: clickhouse_connect.driver.client.Client): + client.query(textwrap.dedent("""\ + ALTER TABLE events + ADD COLUMN IF NOT EXISTS + `remote_addr` Nullable(String) DEFAULT NULL + """)) + + +migrations = [ + migration_000_initial, + migration_001_add_access_token, + migration_002_add_remote_addr, +] + + +def main(args: list[str] | None = None) -> int: + parser = ArgumentParser(description="Initializes the DB and applies migrations") + parser.add_argument("--host", help=f"Server host (env: {CLICKHOUSE_HOST_ENV_VAR})") + parser.add_argument("--port", help=f"Server port (env: {CLICKHOUSE_PORT_ENV_VAR})") + parser.add_argument("--db", help=f"Database name (env: {CLICKHOUSE_DB_ENV_VAR})") + parser.add_argument("--user", help=f"Username (env: {CLICKHOUSE_USER_ENV_VAR})") + parser.add_argument("--password", help=f"Password (env: {CLICKHOUSE_PASSWORD_ENV_VAR})") + parser.add_argument( + "--clear", action="store_true", help="Clear the existing DB and initialize a new one" + ) + parsed_args = parser.parse_args(args) + + db_name = parsed_args.db or os.getenv(CLICKHOUSE_DB_ENV_VAR) + + with clickhouse_connect.get_client( + host=parsed_args.host or os.getenv(CLICKHOUSE_HOST_ENV_VAR), + port=parsed_args.port or os.getenv(CLICKHOUSE_PORT_ENV_VAR), + username=parsed_args.user or os.getenv(CLICKHOUSE_USER_ENV_VAR), + password=parsed_args.password or os.getenv(CLICKHOUSE_PASSWORD_ENV_VAR), + ) as client: + create_db(client, db_name) + client.database = db_name # client can't be created with non existing DB + + if parsed_args.clear: + print("Clearing the DB") + clear_db(client) + print("done") + + for migration_func in migrations: + print("Applying migration", migration_func.__name__) + migration_func(client) + print("done") + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/components/analytics/grafana/dashboards/all_events.json b/components/analytics/grafana/dashboards/all_events.json new file mode 100644 index 0000000..1009837 --- /dev/null +++ b/components/analytics/grafana/dashboards/all_events.json @@ -0,0 +1,760 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 90, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "metrics": [ + { + "aggregation": "count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "metrics": [ + { + "aggregation": "count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + } + }, + "queryType": "sql", + "rawSql": "SELECT $__timeInterval(timestamp) as time, count(*)\r\nFROM events\r\nWHERE $__timeFilter(timestamp)\r\nAND scope IN (${scopes})\r\nAND source IN (${sources})\r\nAND (-1 IN (${users}) OR user_id IN (${users}))\r\nAND (-1 IN (${organizations}) OR org_id IN (${organizations}))\r\nAND (-1 IN (${projects}) OR project_id IN (${projects}))\r\nAND (-1 IN (${tasks}) OR task_id IN (${tasks}))\r\nAND (-1 IN (${jobs}) OR job_id IN (${jobs}))\r\nGROUP BY time ORDER BY time ASC", + "refId": "A" + } + ], + "title": "Overall Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 158 + } + ] + } + ] + }, + "gridPos": { + "h": 23, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 4, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [ + "*" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "IN", + "type": "String", + "value": [ + "" + ] + } + ], + "mode": "list", + "orderBy": [ + { + "dir": "ASC", + "name": "timestamp" + } + ], + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [ + "*" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "IN", + "type": "String", + "value": [ + "" + ] + } + ], + "mode": "list", + "orderBy": [ + { + "dir": "ASC", + "name": "timestamp" + } + ], + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT * \r\nFROM events \r\nWHERE $__timeFilter(timestamp)\r\n AND scope IN (${scopes})\r\n AND source IN (${sources})\r\n AND (-1 IN (${users}) OR user_id IN (${users}))\r\n AND (' ' IN (${usernames}) OR user_name IN (${usernames}))\r\n AND (-1 IN (${organizations}) OR org_id IN (${organizations}))\r\n AND (-1 IN (${projects}) OR project_id IN (${projects}))\r\n AND (-1 IN (${tasks}) OR task_id IN (${tasks}))\r\n AND (-1 IN (${jobs}) OR job_id IN (${jobs}))\r\nORDER BY timestamp DESC\r\nLIMIT 1000", + "refId": "A" + } + ], + "title": "All events", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 6, + "options": { + "barRadius": 0, + "barWidth": 0.51, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "always", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "browser", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "mode": "list", + "orderBy": [], + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "mode": "list", + "orderBy": [], + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n browser,\r\n count()\r\nFROM\r\n(\r\n SELECT\r\n concat(JSON_VALUE(payload, '$.platform.name'), ' ', JSON_VALUE(payload, '$.platform.version')) AS browser,\r\n user_id\r\n FROM cvat.events\r\n WHERE $__timeFilter(timestamp) AND (scope = 'load:cvat') AND (browser != ' ')\r\n GROUP BY\r\n user_id,\r\n browser\r\n)\r\nGROUP BY browser\r\nORDER BY count() DESC", + "refId": "A" + } + ], + "title": "Browser", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 8, + "options": { + "barRadius": 0, + "barWidth": 0.51, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "always", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "os", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "mode": "list", + "orderBy": [], + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "mode": "list", + "orderBy": [], + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n os,\r\n count()\r\nFROM\r\n(\r\n SELECT\r\n JSON_VALUE(payload, '$.platform.os') AS os,\r\n user_id\r\n FROM cvat.events\r\n WHERE $__timeFilter(timestamp) AND (scope = 'load:cvat') AND (os != '')\r\n GROUP BY\r\n user_id,\r\n os\r\n)\r\nGROUP BY os\r\nORDER BY count() DESC", + "refId": "A" + } + ], + "title": "OS", + "type": "barchart" + } + ], + "refresh": false, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT scope\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Scope", + "multi": true, + "name": "scopes", + "options": [], + "query": "SELECT DISTINCT scope\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT source\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "sources", + "options": [], + "query": "SELECT DISTINCT source\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND user_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "users", + "options": [], + "query": "SELECT DISTINCT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND user_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "' '", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "SELECT DISTINCT user_name\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Username", + "multi": true, + "name": "usernames", + "options": [], + "query": "SELECT DISTINCT user_name\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "projects", + "options": [], + "query": "SELECT DISTINCT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Task", + "multi": true, + "name": "tasks", + "options": [], + "query": "SELECT DISTINCT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "jobs", + "options": [], + "query": "SELECT DISTINCT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT org_id\nFROM events\nWHERE $__timeFilter(timestamp)\nAND org_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Organization", + "multi": true, + "name": "organizations", + "options": [], + "query": "SELECT DISTINCT org_id\nFROM events\nWHERE $__timeFilter(timestamp)\nAND org_id IS NOT NULL", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "All events", + "uid": "EIGSTDAVz", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/components/analytics/grafana/dashboards/management.json b/components/analytics/grafana/dashboards/management.json new file mode 100644 index 0000000..37a4542 --- /dev/null +++ b/components/analytics/grafana/dashboards/management.json @@ -0,0 +1,532 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "User 1" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "groupBy": [ + "user_id" + ], + "metrics": [ + { + "aggregation": "count", + "alias": "value", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "hide": false, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "groupBy": [ + "user_id" + ], + "metrics": [ + { + "aggregation": "count", + "alias": "value", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + } + }, + "queryType": "sql", + "rawSql": "SELECT $__timeInterval(timestamp) as time, toString(user_id), count() as User\r\nFROM events\r\nWHERE $__timeFilter(timestamp)\r\nGROUP BY time, user_id\r\nORDER BY time ASC, user_id ASC", + "refId": "A" + } + ], + "title": "User Activity", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 90, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "metrics": [ + { + "aggregation": "count", + "alias": "Count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(timestamp) as time, count(*) Count FROM events WHERE $__timeFilter(timestamp) GROUP BY time ORDER BY time ASC", + "refId": "A" + } + ], + "title": "Overall Activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "footer": { + "enablePagination": true, + "fields": [ + "Working time(h)", + "Activity" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n user_id as User,\r\n user_name as Username,\r\n project_id as Project,\r\n task_id as Task,\r\n job_id as Job, sum(JSONExtractUInt(payload, 'working_time')) / 1000 / 3600 as \"Working time(h)\",\r\n count() as Activity\r\nFROM events\r\nWHERE JSONHas(payload, 'working_time')\r\n AND $__timeFilter(timestamp)\r\n AND(-1 IN (${users}) OR user_id IN (${users}))\r\n AND (' ' IN (${usernames}) OR user_name IN (${usernames}))\r\n AND (-1 IN (${projects}) OR project_id IN (${projects}))\r\n AND task_id IN (${tasks})\r\n AND job_id IN (${jobs})\r\nGROUP BY user_id, user_name, project_id, task_id, job_id", + "refId": "A" + } + ], + "title": "Working time", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND source = 'client'", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "users", + "options": [], + "query": "SELECT DISTINCT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND source = 'client'", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "' '", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "SELECT DISTINCT user_name\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Username", + "multi": true, + "name": "usernames", + "options": [], + "query": "SELECT DISTINCT user_name\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL\n AND source = 'client'", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "projects", + "options": [], + "query": "SELECT DISTINCT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL\n AND source = 'client'", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL\n AND source = 'client'\n AND (-1 IN (${projects}) OR project_id IN (${projects}))", + "description": "", + "hide": 0, + "includeAll": true, + "label": "Task", + "multi": true, + "name": "tasks", + "options": [], + "query": "SELECT DISTINCT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL\n AND source = 'client'\n AND (-1 IN (${projects}) OR project_id IN (${projects}))", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL\n AND source = 'client'\n AND task_id in (${tasks})", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "jobs", + "options": [], + "query": "SELECT DISTINCT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL\n AND source = 'client'\n AND task_id in (${tasks})", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Management", + "uid": "w0if6WAVz", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/components/analytics/grafana/dashboards/monitoring.json b/components/analytics/grafana/dashboards/monitoring.json new file mode 100644 index 0000000..426ac56 --- /dev/null +++ b/components/analytics/grafana/dashboards/monitoring.json @@ -0,0 +1,1339 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "" + } + ], + "mode": "aggregate", + "orderBy": [], + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "filters": [], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "" + } + ], + "mode": "aggregate", + "orderBy": [], + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n uniqExact(user_id) as \"Active users (now)\"\r\nFROM\r\n cvat.events\r\nWHERE\r\n user_id IS NOT NULL AND\r\n timestamp >= now() - INTERVAL 15 MINUTE", + "refId": "A" + } + ], + "title": "Active users (now)", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "Show active users calculated by 15 minutes time interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 21, + "x": 3, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n time,\r\n uniqExact(user_id) Users\r\nFROM\r\n(\r\n SELECT\r\n user_id,\r\n toStartOfInterval(timestamp, INTERVAL 15 minute) as time\r\n FROM cvat.events\r\n WHERE\r\n user_id IS NOT NULL\r\n GROUP BY\r\n user_id,\r\n time\r\n ORDER BY time ASC WITH FILL STEP toIntervalMinute(15)\r\n)\r\nGROUP BY time\r\nORDER BY time", + "refId": "A" + } + ], + "title": "Active users", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "database": "cvat", + "fields": [ + "scope as Scope" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "JSONHas", + "operator": "", + "type": "string", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "$__timeFilter", + "operator": "", + "type": "datetime", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "user_id", + "operator": "IN", + "type": "", + "value": [ + "users" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "1", + "operator": "IN", + "type": "", + "value": [ + "projects", + "OR", + "project_id", + "IN", + "projects" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "task_id", + "operator": "IN", + "type": "", + "value": [ + "tasks" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "job_id", + "operator": "IN", + "type": "", + "value": [ + "jobs" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "source", + "operator": "=", + "type": "string", + "value": [ + "client" + ] + } + ], + "groupBy": [ + "user_id", + "project_id", + "task_id", + "job_id" + ], + "limit": 100, + "metrics": [ + { + "aggregation": "min", + "alias": "working_time", + "field": "JSONExtractUInt" + }, + { + "aggregation": "count", + "alias": "Activity", + "field": "as" + } + ], + "mode": "aggregate", + "table": "events" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "database": "cvat", + "fields": [ + "scope as Scope" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "JSONHas", + "operator": "", + "type": "string", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "$__timeFilter", + "operator": "", + "type": "datetime", + "value": "" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "user_id", + "operator": "IN", + "type": "", + "value": [ + "users" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "1", + "operator": "IN", + "type": "", + "value": [ + "projects", + "OR", + "project_id", + "IN", + "projects" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "task_id", + "operator": "IN", + "type": "", + "value": [ + "tasks" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "job_id", + "operator": "IN", + "type": "", + "value": [ + "jobs" + ] + }, + { + "condition": "AND", + "filterType": "custom", + "key": "source", + "operator": "=", + "type": "string", + "value": [ + "client" + ] + } + ], + "groupBy": [ + "user_id", + "project_id", + "task_id", + "job_id" + ], + "limit": 100, + "metrics": [ + { + "aggregation": "min", + "alias": "working_time", + "field": "JSONExtractUInt" + }, + { + "aggregation": "count", + "alias": "Activity", + "field": "as" + } + ], + "mode": "aggregate", + "table": "events" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n scope as Scope,\r\n source as Source,\r\n avg(duration) as \"Average duration (ms)\",\r\n min(duration) as \"Min duration (ms)\",\r\n max(duration) as \"Max duration (ms)\"\r\nFROM events\r\nWHERE duration > 0\r\n AND $__timeFilter(timestamp)\r\nGROUP BY scope, source", + "refId": "A" + } + ], + "title": "Duration of events", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 4, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n scope as Scope,\r\n source as Source,\r\n count() as Count\r\nFROM events\r\nWHERE $__timeFilter(timestamp)\r\nGROUP BY scope, source", + "refId": "A" + } + ], + "title": "Number of events", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "fields": [], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "=", + "type": "String", + "value": "send:exception" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + } + ], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 0, + "meta": { + "builderOptions": { + "fields": [], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "=", + "type": "String", + "value": "send:exception" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + } + ], + "limit": 100, + "metrics": [ + { + "aggregation": "count", + "field": "*" + } + ], + "mode": "trend", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n $__timeInterval(timestamp) as time,\r\n count(*)\r\nFROM events\r\nWHERE\r\n $__timeFilter(timestamp)\r\n AND ( scope = 'send:exception' )\r\n AND ( timestamp >= $__fromTime AND timestamp <= $__toTime )\r\n AND source IN (${sources})\r\n AND (' ' in (${usernames}) OR user_name IN (${usernames}))\r\n AND (-1 IN (${users}) OR user_id IN (${users}))\r\n AND (-1 IN (${organizations}) OR org_id IN (${organizations}))\r\n AND (-1 IN (${projects}) OR project_id IN (${projects}))\r\n AND (-1 IN (${tasks}) OR task_id IN (${tasks}))\r\n AND (-1 IN (${jobs}) OR job_id IN (${jobs}))\r\nGROUP BY time\r\nORDER BY time", + "refId": "A" + } + ], + "title": "Exceptions", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true, + "minWidth": 80 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "user_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 28 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "source" + }, + "properties": [ + { + "id": "custom.width", + "value": 68 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "project_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 61 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "task_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 55 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 115 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 153 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "custom.width", + "value": 452 + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 8, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "builderOptions": { + "fields": [ + "user_id", + "project_id", + "task_id", + "job_id", + "payload" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "=", + "type": "String", + "value": "send:exception" + } + ], + "mode": "list", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [ + "user_id", + "project_id", + "task_id", + "job_id", + "payload" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime64(3, 'Etc/UTC')", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "scope", + "operator": "=", + "type": "String", + "value": "send:exception" + } + ], + "mode": "list", + "orderBy": [], + "table": "events", + "timeField": "timestamp", + "timeFieldType": "DateTime64(3, 'Etc/UTC')" + } + }, + "queryType": "sql", + "rawSql": "SELECT\r\n timestamp,\r\n user_id,\r\n user_name,\r\n source,\r\n project_id,\r\n task_id,\r\n job_id,\r\n JSONExtractString(payload, 'message') as error,\r\n JSONExtractString(payload, 'stack') as stack,\r\n payload\r\nFROM events\r\nWHERE\r\n ( timestamp >= $__fromTime AND timestamp <= $__toTime )\r\n AND scope = 'send:exception'\r\n AND source IN (${sources})\r\n AND (-1 IN (${users}) OR user_id IN (${users}))\r\n AND (' ' IN (${usernames}) OR user_name IN (${usernames}))\r\n AND (-1 IN (${organizations}) OR org_id IN (${organizations}))\r\n AND (-1 IN (${projects}) OR project_id IN (${projects}))\r\n AND (-1 IN (${tasks}) OR task_id IN (${tasks}))\r\n AND (-1 IN (${jobs}) OR job_id IN (${jobs}))\r\n AND ('-1' IN (${errors}) OR error IN (${errors}))\r\nORDER BY timestamp DESC\r\nLIMIT 1000", + "refId": "A" + } + ], + "title": "Exceptions table", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "description": "MegaBytes per second grouped by queue: writes up, reads down", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": 5000, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.8.2", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(timestamp) AS time,\r\n concat('write:', ifNull(JSONExtractString(payload, 'queue'), 'unknown')) AS queue,\r\n sum(JSONExtractUInt(JSONExtractRaw(payload, 'cache_item'), 'size')) \r\n / $__interval_s \r\n / 1048576 AS Throughput\r\nFROM events\r\nWHERE\r\n $__timeFilter(timestamp)\r\n AND scope = 'create:cache_item'\r\nGROUP BY\r\n time,\r\n queue\r\nORDER BY\r\n time ASC", + "refId": "A" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.8.2", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(timestamp) AS time,\r\n concat('read:', ifNull(JSONExtractString(payload, 'queue'), 'unknown')) AS queue,\r\n -sum(JSONExtractUInt(JSONExtractRaw(payload, 'cache_item'), 'size')) \r\n / $__interval_s \r\n / 1048576 AS Throughput\r\nFROM events\r\nWHERE\r\n $__timeFilter(timestamp)\r\n AND scope = 'read:cache_item'\r\nGROUP BY\r\n time,\r\n queue\r\nORDER BY\r\n time ASC", + "refId": "B" + } + ], + "title": "Cache Throughput by Queue", + "transformations": [], + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT source\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "sources", + "options": [], + "query": "SELECT DISTINCT source\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND user_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "users", + "options": [], + "query": "SELECT DISTINCT user_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND user_id IS NOT NULL", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "' '", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT user_name\nFROM events\nWHERE $__timeFilter(timestamp)", + "hide": 0, + "includeAll": true, + "label": "Username", + "multi": true, + "name": "usernames", + "options": [], + "query": "SELECT DISTINCT user_name\nFROM events\nWHERE $__timeFilter(timestamp)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "projects", + "options": [], + "query": "SELECT DISTINCT project_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND project_id IS NOT NULL", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Task", + "multi": true, + "name": "tasks", + "options": [], + "query": "SELECT DISTINCT task_id\nFROM events\nWHERE $__timeFilter(timestamp) \n AND task_id IS NOT NULL", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "jobs", + "options": [], + "query": "SELECT DISTINCT job_id\nFROM events\nWHERE $__timeFilter(timestamp)\n AND job_id IS NOT NULL", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "-1", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT DISTINCT org_id\nFROM events\nWHERE $__timeFilter(timestamp)\nAND org_id IS NOT NULL", + "hide": 0, + "includeAll": true, + "label": "Organization", + "multi": true, + "name": "organizations", + "options": [], + "query": "SELECT DISTINCT org_id\nFROM events\nWHERE $__timeFilter(timestamp)\nAND org_id IS NOT NULL", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "'-1'", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "definition": "SELECT\n DISTINCT JSONExtractString(payload, 'message')\n FROM cvat.events\nWHERE $__timeFilter(timestamp)\n AND JSONHas(payload, 'message')\n AND scope='send:exception'", + "hide": 0, + "includeAll": true, + "label": "Error message", + "multi": true, + "name": "errors", + "options": [], + "query": "SELECT\n DISTINCT JSONExtractString(payload, 'message')\n FROM cvat.events\nWHERE $__timeFilter(timestamp)\n AND JSONHas(payload, 'message')\n AND scope='send:exception'", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Monitoring", + "uid": "WvDvWK04k", + "version": 4, + "weekStart": "" +} diff --git a/components/analytics/grafana/dashboards/users_exceptions.json b/components/analytics/grafana/dashboards/users_exceptions.json new file mode 100644 index 0000000..0c47e31 --- /dev/null +++ b/components/analytics/grafana/dashboards/users_exceptions.json @@ -0,0 +1,1550 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "custom": { + "width": 1 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total_errors" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Total Errors" + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "delta_percent" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "displayName", + "value": "Delta vs Prev" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 200, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": false + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "queryType": "sql", + "rawSql": "WITH\n [\n 'permissiondenied',\n 'validationerror:',\n 'django.http.response.http404',\n 'you do not have permission to perform this action.',\n 'limitreachedexception:',\n 'throttled',\n 'request failed with status code 429',\n 'matches the given query.',\n 'cloud storage instance was deleted',\n 'cvatdatasetnotfounderror',\n 'authenticationfailed',\n 'no media data found',\n 'authentication credentials were not provided',\n 'rest_framework.exceptions.notacceptable: could not satisfy the request accept header',\n 'rest_framework.exceptions.methodnotallowed',\n 'rest_framework.exceptions.notfound'\n ] AS raw_excluded_patterns,\nraw AS (\n SELECT\n timestamp,\n source,\n user_id,\n lowerUTF8(JSONExtractString(payload, 'message')) AS msg_lower\n FROM cvat.events\n PREWHERE scope = 'send:exception'\n AND source IN (${sources})\n AND timestamp >= subtractSeconds($__fromTime, dateDiff('second', $__fromTime, $__toTime))\n AND timestamp <= $__toTime\n WHERE NOT multiSearchAnyCaseInsensitive(JSONExtractString(payload, 'message'), raw_excluded_patterns)\n), filtered AS (\n SELECT\n timestamp,\n source,\n user_id,\n toUInt8(timestamp >= $__fromTime) AS period_id,\n if(timestamp >= $__fromTime, toStartOfDay(timestamp), toDateTime('1970-01-01 00:00:00', 'Etc/UTC')) AS day,\n msg_lower\n FROM raw\n WHERE length(trim(BOTH ' ' FROM msg_lower)) > 0\n), categorized AS (\n SELECT\n period_id,\n day,\n source,\n user_id,\n multiIf(\n position(msg_lower, 'network error') > 0, toUInt8(1),\n position(msg_lower, 'bad gateway') > 0, toUInt8(2),\n position(msg_lower, 'cvat.apps.webhooks.exceptions.webhookdeliveryerror') > 0, toUInt8(40),\n multiSearchAny(msg_lower, ['gateway time-out', 'gateway timeout', '504 gateway', 'code 504 received', 'timeout exceeded', 'request aborted', 'the request timed out. the cvat server did not respond in time']), toUInt8(3),\n position(msg_lower, 'server error') > 0, toUInt8(4),\n position(msg_lower, 'no available server') > 0, toUInt8(5),\n multiSearchAny(msg_lower, ['redis.exceptions', 'kvrocks']), toUInt8(6),\n multiSearchAny(msg_lower, ['django.db.utils.operationalerror', 'database system is']), toUInt8(7),\n multiSearchAny(msg_lower, ['integrityerror', 'foreign key constraint', 'unique constraint']), toUInt8(8),\n multiSearchAny(msg_lower, ['rq.timeouts.jobtimeoutexception', 'takes too long chunks']), toUInt8(9),\n source = 'client' AND multiSearchAny(msg_lower, ['cannot read properties of', 'can''t access property', 'objectstate must be provided correct label, got wrong value undefined', 'cannot set properties of undefined']), toUInt8(10),\n source = 'client' AND multiSearchAny(msg_lower, ['is not a function', 'is not iterable']), toUInt8(11),\n source = 'client' AND position(msg_lower, 'setting getter-only property') > 0, toUInt8(12),\n source = 'client' AND position(msg_lower, 'the start frame is out of the job') > 0, toUInt8(13),\n source = 'client' AND multiSearchAny(msg_lower, ['canvas', 'webgl', 'createimagebitmap', 'createobjecturl', 'getimagedata']), toUInt8(14),\n source = 'client' AND multiSearchAny(msg_lower, ['react error', '__reactfiber']), toUInt8(15),\n source = 'client' AND (startsWith(msg_lower, '**shapes**') OR startsWith(msg_lower, '**labels**')), toUInt8(16),\n source = 'client' AND multiSearchAny(msg_lower, ['queryselector', 'not a valid selector']), toUInt8(17),\n source = 'client' AND position(msg_lower, 'session has not been initialized yet. call annotations.get() or annotations.clear({ reload: true }) before') > 0, toUInt8(18),\n multiSearchAny(msg_lower, ['reached the maximum number of projects', 'cachetoolargedataerror']), toUInt8(19),\n position(msg_lower, 'decompressionbomberror') > 0, toUInt8(20),\n multiSearchAny(msg_lower, ['polygon must have at least', 'all the objects must have the same label', 'expected only one visible shape per frame', 'cannot join: not enough valid polygons']), toUInt8(21),\n multiSearchAny(msg_lower, ['action name must be unique', 'all label names must be unique']), toUInt8(22),\n position(msg_lower, 'label') > 0 AND position(msg_lower, 'is not registered for this task') > 0, toUInt8(23),\n (position(msg_lower, 'trying to save an attribute') > 0 AND position(msg_lower, 'invalid value') > 0) OR (position(msg_lower, 'spec_id') > 0 AND position(msg_lower, ' is invalid') > 0), toUInt8(24),\n multiSearchAny(msg_lower, ['unsupported media type', 'unknown media type', 'file must be file instance']), toUInt8(25),\n multiSearchAny(msg_lower, ['unexpected type of track', 'skeleton template must be specified', 'tried to set wrong number of points for a skeleton', 'cannot join these polygons: the operation would create a shape with holes, which is not supported by cvat. please select different polygons or use the mask tool.']), toUInt8(21),\n multiSearchAny(msg_lower, ['oserror: [errno 36]', 'oserror: [errno 22] invalid argument']), toUInt8(26),\n multiSearchAny(msg_lower, ['datumaro.components.errors', 'annotationexporterror', 'itemexporterror', 'repeateditemerror', 'datumaro.components.contexts.importer._importfail', 'there is no item named ''project.json'' in the archive', 'there is no item named ''task.json'' in the archive', 'no media data found', 'no image found', 'cvat.apps.dataset_manager.bindings.cvatimporterror']), toUInt8(29),\n position(msg_lower, 'cloudstoragemissingerror') > 0, toUInt8(30),\n multiSearchAny(msg_lower, ['invalidimageerror', 'unidentifiedimageerror']), toUInt8(31),\n position(msg_lower, 'runtimeerror: exception occurred during compression of image') > 0, toUInt8(32),\n position(msg_lower, 'av.error') > 0, toUInt8(33),\n multiSearchAny(msg_lower, ['filenotfounderror: [errno 2] no such file or directory', 'manifest.jsonl']), toUInt8(34),\n multiSearchAny(msg_lower, ['not found on the cloud storage', 'resource ali-']), toUInt8(35),\n position(msg_lower, 'trying to use undefined cloud storage') > 0, toUInt8(36),\n position(msg_lower, 'already being processed (action=') > 0, toUInt8(37),\n position(msg_lower, 'duplicate key value violates unique constraint') > 0, toUInt8(38),\n position(msg_lower, 'locknotownederror') > 0, toUInt8(39),\n toUInt8(255)\n ) AS category_id\n FROM filtered\n), grouped AS (\n SELECT\n grouping(period_id) AS g_period,\n grouping(day) AS g_day,\n grouping(source) AS g_source,\n grouping(category_id) AS g_category,\n period_id,\n day,\n source,\n category_id,\n count() AS error_count,\n uniqExactIf(user_id, isNotNull(user_id)) AS unique_users\n FROM categorized\n GROUP BY GROUPING SETS (\n (period_id),\n (period_id, source),\n (period_id, category_id),\n (period_id, source, category_id),\n (day, category_id)\n )\n)\nSELECT\n dataset,\n period,\n time,\n source,\n category,\n category_order,\n error_count,\n unique_users,\n if(dataset IN ('category', 'source_category') AND period = 'current', round(error_count * 100.0 / sumIf(error_count, period = 'current') OVER (PARTITION BY dataset), 2), NULL) AS percentage,\n if(dataset IN ('stats', 'source', 'category', 'source_category'), sumIf(error_count, period = 'previous') OVER (PARTITION BY dataset, source, category), NULL) AS prev_error_count,\n if(dataset IN ('stats', 'source', 'category', 'source_category'), sumIf(unique_users, period = 'previous') OVER (PARTITION BY dataset, source, category), NULL) AS prev_unique_users,\n if(dataset IN ('stats', 'source', 'category', 'source_category') AND period = 'current' AND prev_error_count != 0, round((error_count - prev_error_count) * 100.0 / prev_error_count, 2), NULL) AS delta_percent,\n if(dataset = 'stats' AND period = 'current' AND prev_unique_users != 0, round((unique_users - prev_unique_users) * 100.0 / prev_unique_users, 2), NULL) AS unique_users_delta_percent\nFROM\n(\n SELECT\n multiIf(\n g_period = 0 AND g_day = 1 AND g_source = 1 AND g_category = 1, 'stats',\n g_period = 0 AND g_day = 1 AND g_source = 0 AND g_category = 1, 'source',\n g_period = 0 AND g_day = 1 AND g_source = 1 AND g_category = 0, 'category',\n g_period = 0 AND g_day = 1 AND g_source = 0 AND g_category = 0, 'source_category',\n g_period = 1 AND g_day = 0 AND g_source = 1 AND g_category = 0, 'day_category',\n 'unknown'\n ) AS dataset,\n if(g_period = 0, if(period_id = 1, 'current', 'previous'), '') AS period,\n day AS time,\n source,\n if(g_category = 0, multiIf(\n category_id = 1, 'Network Error',\n category_id = 2, '502 Bad Gateway',\n category_id = 3, '504 Gateway Timeout',\n category_id = 4, '500 Server Error',\n category_id = 5, 'No available server / service unavailable',\n category_id = 6, 'Redis / kvrocks',\n category_id = 7, 'Postgres / DB (operational)',\n category_id = 8, 'Postgres / DB (integrity)',\n category_id = 9, 'Timeouts / background jobs',\n category_id = 10, 'JS null/undefined access',\n category_id = 11, 'JS type/method errors',\n category_id = 12, 'JS read-only property assignment',\n category_id = 13, 'Client validation / frame bounds',\n category_id = 14, 'Canvas / WebGL / memory',\n category_id = 15, 'React / DOM',\n category_id = 16, 'Debug logging / shapes dump',\n category_id = 17, 'JS DOM API / Selectors',\n category_id = 18, 'CVAT-core / Business Logic',\n category_id = 19, 'Quotas / size limits',\n category_id = 20, 'Quotas / size limits (image bomb)',\n category_id = 21, 'Geometry / annotation rules',\n category_id = 22, 'Uniqueness / names / ids',\n category_id = 23, 'Labels not registered',\n category_id = 24, 'Attributes invalid value',\n category_id = 25, 'Unsupported / invalid input',\n category_id = 26, 'OS error / file system',\n category_id = 29, 'Import/Export / Datumaro',\n category_id = 30, 'Cloud storage / file missing',\n category_id = 31, 'Invalid image file',\n category_id = 32, 'Image compression errors',\n category_id = 33, 'AV / codec errors',\n category_id = 34, 'File not found / missing file',\n category_id = 35, 'Cloud storage / file not found',\n category_id = 36, 'Cloud storage / config error',\n category_id = 37, 'Concurrent export/import / duplicate request',\n category_id = 38, 'DB duplicate key',\n category_id = 39, 'Redis lock / concurrency',\n category_id = 40, 'Webhook delivery errors',\n 'Other / uncategorized'\n ), '') AS category,\n if(g_category = 0, multiIf(\n category_id = 7, 1,\n category_id = 10, 2,\n category_id = 34, 3,\n category_id = 33, 4,\n category_id = 30, 5,\n category_id = 21, 6,\n category_id = 20, 8,\n category_id = 31, 9,\n category_id = 19, 10,\n category_id = 16, 11,\n category_id = 8, 12,\n category_id = 15, 13,\n category_id = 18, 14,\n category_id = 3, 15,\n category_id = 37, 16,\n category_id = 14, 17,\n category_id = 13, 18,\n category_id = 22, 19,\n category_id = 11, 21,\n category_id = 1, 22,\n category_id = 23, 23,\n category_id = 4, 24,\n category_id = 255, 25,\n category_id = 25, 26,\n category_id = 29, 27,\n category_id = 2, 28,\n 1000 + category_id\n ), 0) AS category_order,\n error_count,\n unique_users\n FROM grouped\n WHERE NOT (g_period = 1 AND g_day = 0 AND day = toDateTime('1970-01-01 00:00:00', 'Etc/UTC'))\n)", + "refId": "A" + } + ], + "title": "", + "type": "table", + "transparent": true + }, + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total_errors" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Total Errors" + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "delta_percent" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "displayName", + "value": "Delta vs Prev" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "panelId": 200, + "refId": "A" + } + ], + "title": "Total Errors", + "type": "stat", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "fieldName": "dataset", + "config": { + "id": "equal", + "options": { + "value": "stats" + } + } + }, + { + "fieldName": "period", + "config": { + "id": "equal", + "options": { + "value": "current" + } + } + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "dataset": true, + "period": true, + "time": true, + "source": true, + "category": true, + "unique_users": true, + "percentage": true, + "prev_error_count": true, + "prev_unique_users": true, + "unique_users_delta_percent": true, + "category_order": true + }, + "indexByName": { + "error_count": 0, + "delta_percent": 1 + }, + "renameByName": { + "error_count": "total_errors" + } + } + } + ] + }, + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "percentage" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Show drilldown for ${__data.fields.category}", + "url": "/d/${__dashboard.uid}?${__url_time_range}&${sources:queryparam}&var-category=${__data.fields.category}", + "targetBlank": false + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 18, + "w": 16, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "orientation": "horizontal", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0, + "showValue": "auto", + "groupWidth": 0.7, + "barWidth": 0.9, + "barRadius": 0, + "fullHighlight": false, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "legend": { + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "panelId": 200, + "refId": "A" + } + ], + "title": "Category Share", + "type": "barchart", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "fieldName": "dataset", + "config": { + "id": "equal", + "options": { + "value": "category" + } + } + }, + { + "fieldName": "period", + "config": { + "id": "equal", + "options": { + "value": "current" + } + } + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "dataset": true, + "period": true, + "time": true, + "source": true, + "error_count": true, + "unique_users": true, + "prev_error_count": true, + "prev_unique_users": true, + "delta_percent": true, + "unique_users_delta_percent": true, + "category_order": true + }, + "indexByName": { + "category": 0, + "percentage": 1 + }, + "renameByName": {} + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "percentage", + "desc": true + } + ] + } + } + ] + }, + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "unique_users_impacted" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Unique Users" + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "delta_percent" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "displayName", + "value": "Delta vs Prev" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "panelId": 200, + "refId": "A" + } + ], + "title": "Unique Users Impacted", + "type": "stat", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "fieldName": "dataset", + "config": { + "id": "equal", + "options": { + "value": "stats" + } + } + }, + { + "fieldName": "period", + "config": { + "id": "equal", + "options": { + "value": "current" + } + } + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "dataset": true, + "period": true, + "time": true, + "source": true, + "category": true, + "error_count": true, + "percentage": true, + "prev_error_count": true, + "prev_unique_users": true, + "delta_percent": true, + "category_order": true + }, + "indexByName": { + "unique_users": 0, + "unique_users_delta_percent": 1 + }, + "renameByName": { + "unique_users": "unique_users_impacted", + "unique_users_delta_percent": "delta_percent" + } + } + } + ] + }, + { + "id": 7, + "title": "Server vs Client", + "type": "barchart", + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "current_count" + }, + "properties": [ + { + "id": "displayName", + "value": "Current" + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "previous_count" + }, + "properties": [ + { + "id": "displayName", + "value": "Previous" + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + } + ] + }, + "options": { + "orientation": "horizontal", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0, + "showValue": "always", + "groupWidth": 0.7, + "barWidth": 0.9, + "barRadius": 0, + "fullHighlight": false, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "legend": { + "showLegend": true, + "displayMode": "list", + "placement": "bottom" + } + }, + "targets": [ + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "panelId": 200, + "refId": "A" + } + ], + "pluginVersion": "9.3.6", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "fieldName": "dataset", + "config": { + "id": "equal", + "options": { + "value": "source" + } + } + }, + { + "fieldName": "period", + "config": { + "id": "equal", + "options": { + "value": "current" + } + } + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "dataset": true, + "period": true, + "time": true, + "category": true, + "unique_users": true, + "percentage": true, + "prev_unique_users": true, + "delta_percent": true, + "unique_users_delta_percent": true, + "category_order": true + }, + "indexByName": { + "source": 0, + "error_count": 1, + "prev_error_count": 2 + }, + "renameByName": { + "error_count": "current_count", + "prev_error_count": "previous_count" + } + } + } + ] + }, + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "displayName": "${__field.labels.metric}" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "panelId": 200, + "refId": "A" + } + ], + "title": "Errors by Day (Stacked by Category)", + "type": "timeseries", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "fieldName": "dataset", + "config": { + "id": "equal", + "options": { + "value": "day_category" + } + } + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "dataset": true, + "period": true, + "source": true, + "unique_users": true, + "percentage": true, + "prev_error_count": true, + "prev_unique_users": true, + "delta_percent": true, + "unique_users_delta_percent": true, + "category_order": true + }, + "indexByName": { + "time": 0, + "error_count": 1, + "category": 2 + }, + "renameByName": { + "error_count": "value", + "category": "metric" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "metric", + "desc": false + }, + { + "field": "time", + "desc": false + } + ] + } + }, + { + "id": "prepareTimeSeries", + "options": { + "format": "multi" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^(?:error_count|value) (.*)$", + "renamePattern": "$1" + } + } + ] + }, + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "percentage" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "category" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Show drilldown for ${__data.fields.category}", + "url": "/d/${__dashboard.uid}?${__url_time_range}&${sources:queryparam}&var-category=${__data.fields.category}", + "targetBlank": false + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 5, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "error_count" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "dashboard", + "uid": "-- Dashboard --" + }, + "panelId": 200, + "refId": "A" + } + ], + "title": "Category Summary", + "type": "table", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "fieldName": "dataset", + "config": { + "id": "equal", + "options": { + "value": "source_category" + } + } + }, + { + "fieldName": "period", + "config": { + "id": "equal", + "options": { + "value": "current" + } + } + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "dataset": true, + "period": true, + "time": true, + "unique_users_delta_percent": true, + "category_order": true + }, + "indexByName": { + "source": 0, + "category": 1, + "error_count": 2, + "percentage": 3, + "unique_users": 4, + "prev_error_count": 5, + "delta_percent": 6, + "prev_unique_users": 7 + }, + "renameByName": {} + } + } + ] + }, + { + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 201, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "[Show all drilldown categories](/d/${__dashboard.uid}?from=${__from}&to=${__to}&${sources:queryparam}&var-category=__all)", + "mode": "markdown" + }, + "pluginVersion": "10.1.2", + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 6, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "error_count" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "format": 1, + "queryType": "sql", + "rawSql": "WITH\n [\n 'permissiondenied',\n 'validationerror:',\n 'django.http.response.http404',\n 'you do not have permission to perform this action.',\n 'limitreachedexception:',\n 'throttled',\n 'request failed with status code 429',\n 'matches the given query.',\n 'cloud storage instance was deleted',\n 'cvatdatasetnotfounderror',\n 'authenticationfailed',\n 'no media data found',\n 'authentication credentials were not provided',\n 'rest_framework.exceptions.notacceptable: could not satisfy the request accept header',\n 'rest_framework.exceptions.methodnotallowed',\n 'rest_framework.exceptions.notfound'\n ] AS raw_excluded_patterns,\nraw AS (\n SELECT\n source,\n user_id,\n lowerUTF8(JSONExtractString(payload, 'message')) AS msg_lower\n FROM cvat.events\n PREWHERE scope = 'send:exception'\n AND source IN (${sources})\n AND $__timeFilter(timestamp)\n WHERE NOT multiSearchAnyCaseInsensitive(JSONExtractString(payload, 'message'), raw_excluded_patterns)\n), filtered AS (\n SELECT\n source,\n user_id,\n msg_lower\n FROM raw\n WHERE length(trim(BOTH ' ' FROM msg_lower)) > 0\n), categorized AS (\n SELECT\n source,\n user_id,\n msg_lower,\n multiIf(\n position(msg_lower, 'network error') > 0, 'Network Error',\n position(msg_lower, 'bad gateway') > 0, '502 Bad Gateway',\n position(msg_lower, 'cvat.apps.webhooks.exceptions.webhookdeliveryerror') > 0, 'Webhook delivery errors',\n position(msg_lower, 'gateway time-out') > 0 OR position(msg_lower, 'gateway timeout') > 0 OR position(msg_lower, '504 gateway') > 0 OR position(msg_lower, 'code 504 received') > 0 OR position(msg_lower, 'timeout exceeded') > 0 OR position(msg_lower, 'request aborted') > 0 OR position(msg_lower, 'the request timed out. the cvat server did not respond in time') > 0, '504 Gateway Timeout',\n position(msg_lower, 'server error') > 0, '500 Server Error',\n position(msg_lower, 'no available server') > 0, 'No available server / service unavailable',\n position(msg_lower, 'redis.exceptions') > 0 OR position(msg_lower, 'kvrocks') > 0, 'Redis / kvrocks',\n position(msg_lower, 'django.db.utils.operationalerror') > 0 OR position(msg_lower, 'database system is') > 0, 'Postgres / DB (operational)',\n position(msg_lower, 'integrityerror') > 0 OR position(msg_lower, 'foreign key constraint') > 0 OR position(msg_lower, 'unique constraint') > 0, 'Postgres / DB (integrity)',\n position(msg_lower, 'rq.timeouts.jobtimeoutexception') > 0 OR position(msg_lower, 'takes too long chunks') > 0, 'Timeouts / background jobs',\n source = 'client' AND (position(msg_lower, 'cannot read properties of') > 0 OR position(msg_lower, 'can''t access property') > 0 OR position(msg_lower, 'objectstate must be provided correct label, got wrong value undefined') > 0 OR position(msg_lower, 'cannot set properties of undefined') > 0), 'JS null/undefined access',\n source = 'client' AND (position(msg_lower, 'is not a function') > 0 OR position(msg_lower, 'is not iterable') > 0), 'JS type/method errors',\n source = 'client' AND position(msg_lower, 'setting getter-only property') > 0, 'JS read-only property assignment',\n source = 'client' AND position(msg_lower, 'the start frame is out of the job') > 0, 'Client validation / frame bounds',\n source = 'client' AND (position(msg_lower, 'canvas') > 0 OR position(msg_lower, 'webgl') > 0 OR position(msg_lower, 'createimagebitmap') > 0 OR position(msg_lower, 'createobjecturl') > 0 OR position(msg_lower, 'getimagedata') > 0), 'Canvas / WebGL / memory',\n source = 'client' AND (position(msg_lower, 'react error') > 0 OR position(msg_lower, '__reactfiber') > 0), 'React / DOM',\n source = 'client' AND (startsWith(msg_lower, '**shapes**') OR startsWith(msg_lower, '**labels**')), 'Debug logging / shapes dump',\n source = 'client' AND (position(msg_lower, 'queryselector') > 0 OR position(msg_lower, 'not a valid selector') > 0), 'JS DOM API / Selectors',\n source = 'client' AND position(msg_lower, 'session has not been initialized yet. call annotations.get() or annotations.clear({ reload: true }) before') > 0, 'CVAT-core / Business Logic',\n position(msg_lower, 'reached the maximum number of projects') > 0 OR position(msg_lower, 'cachetoolargedataerror') > 0, 'Quotas / size limits',\n position(msg_lower, 'decompressionbomberror') > 0, 'Quotas / size limits (image bomb)',\n position(msg_lower, 'polygon must have at least') > 0 OR position(msg_lower, 'all the objects must have the same label') > 0 OR position(msg_lower, 'expected only one visible shape per frame') > 0 OR position(msg_lower, 'cannot join: not enough valid polygons') > 0, 'Geometry / annotation rules',\n position(msg_lower, 'action name must be unique') > 0 OR position(msg_lower, 'all label names must be unique') > 0, 'Uniqueness / names / ids',\n position(msg_lower, 'label') > 0 AND position(msg_lower, 'is not registered for this task') > 0, 'Labels not registered',\n (position(msg_lower, 'trying to save an attribute') > 0 AND position(msg_lower, 'invalid value') > 0) OR (position(msg_lower, 'spec_id') > 0 AND position(msg_lower, ' is invalid') > 0), 'Attributes invalid value',\n position(msg_lower, 'unsupported media type') > 0 OR position(msg_lower, 'unknown media type') > 0 OR position(msg_lower, 'file must be file instance') > 0, 'Unsupported / invalid input',\n position(msg_lower, 'unexpected type of track') > 0 OR position(msg_lower, 'skeleton template must be specified') > 0 OR position(msg_lower, 'tried to set wrong number of points for a skeleton') > 0 OR position(msg_lower, 'cannot join these polygons: the operation would create a shape with holes, which is not supported by cvat. please select different polygons or use the mask tool.') > 0, 'Geometry / annotation rules',\n position(msg_lower, 'oserror: [errno 36]') > 0 OR position(msg_lower, 'oserror: [errno 22] invalid argument') > 0, 'OS error / file system',\n position(msg_lower, 'datumaro.components.errors') > 0 OR position(msg_lower, 'annotationexporterror') > 0 OR position(msg_lower, 'itemexporterror') > 0 OR position(msg_lower, 'repeateditemerror') > 0 OR position(msg_lower, 'datumaro.components.contexts.importer._importfail') > 0 OR position(msg_lower, 'there is no item named ''project.json'' in the archive') > 0 OR position(msg_lower, 'there is no item named ''task.json'' in the archive') > 0 OR position(msg_lower, 'no media data found') > 0 OR position(msg_lower, 'no image found') > 0 OR position(msg_lower, 'cvat.apps.dataset_manager.bindings.cvatimporterror') > 0, 'Import/Export / Datumaro',\n position(msg_lower, 'cloudstoragemissingerror') > 0, 'Cloud storage / file missing',\n position(msg_lower, 'invalidimageerror') > 0 OR position(msg_lower, 'unidentifiedimageerror') > 0, 'Invalid image file',\n position(msg_lower, 'runtimeerror: exception occurred during compression of image') > 0, 'Image compression errors',\n position(msg_lower, 'av.error') > 0, 'AV / codec errors',\n position(msg_lower, 'filenotfounderror: [errno 2] no such file or directory') > 0 OR position(msg_lower, 'manifest.jsonl') > 0, 'File not found / missing file',\n position(msg_lower, 'not found on the cloud storage') > 0 OR position(msg_lower, 'resource ali-') > 0, 'Cloud storage / file not found',\n position(msg_lower, 'trying to use undefined cloud storage') > 0, 'Cloud storage / config error',\n position(msg_lower, 'already being processed (action=') > 0, 'Concurrent export/import / duplicate request',\n position(msg_lower, 'duplicate key value violates unique constraint') > 0, 'DB duplicate key',\n position(msg_lower, 'locknotownederror') > 0, 'Redis lock / concurrency',\n 'Other / uncategorized'\n ) AS category\n FROM filtered\n)\nSELECT\n category,\n msg_lower AS error,\n source,\n count() AS error_count,\n uniqExactIf(user_id, isNotNull(user_id)) AS unique_users\nFROM categorized\nWHERE '${category}' = '__all' OR category = '${category}'\nGROUP BY category, msg_lower, source\nORDER BY error_count DESC, category ASC, error ASC, source ASC\nLIMIT 200", + "refId": "A" + } + ], + "title": "Category Drilldown (Top Exceptions)", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "exceptions" + ], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "client,server", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "sources", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "client", + "value": "client" + }, + { + "selected": false, + "text": "server", + "value": "server" + } + ], + "query": "client,server", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "custom" + }, + { + "allValue": "__all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 2, + "includeAll": true, + "label": "Category", + "multi": false, + "name": "category", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "Network Error", + "value": "Network Error" + }, + { + "selected": false, + "text": "JS null/undefined access", + "value": "JS null/undefined access" + }, + { + "selected": false, + "text": "500 Server Error", + "value": "500 Server Error" + }, + { + "selected": false, + "text": "502 Bad Gateway", + "value": "502 Bad Gateway" + }, + { + "selected": false, + "text": "504 Gateway Timeout", + "value": "504 Gateway Timeout" + }, + { + "selected": false, + "text": "Other / uncategorized", + "value": "Other / uncategorized" + }, + { + "selected": false, + "text": "AV / codec errors", + "value": "AV / codec errors" + }, + { + "selected": false, + "text": "Attributes invalid value", + "value": "Attributes invalid value" + }, + { + "selected": false, + "text": "CVAT-core / Business Logic", + "value": "CVAT-core / Business Logic" + }, + { + "selected": false, + "text": "Canvas / WebGL / memory", + "value": "Canvas / WebGL / memory" + }, + { + "selected": false, + "text": "Cloud storage / config error", + "value": "Cloud storage / config error" + }, + { + "selected": false, + "text": "Cloud storage / file missing", + "value": "Cloud storage / file missing" + }, + { + "selected": false, + "text": "Cloud storage / file not found", + "value": "Cloud storage / file not found" + }, + { + "selected": false, + "text": "Concurrent export/import / duplicate request", + "value": "Concurrent export/import / duplicate request" + }, + { + "selected": false, + "text": "DB duplicate key", + "value": "DB duplicate key" + }, + { + "selected": false, + "text": "Debug logging / shapes dump", + "value": "Debug logging / shapes dump" + }, + { + "selected": false, + "text": "File not found / missing file", + "value": "File not found / missing file" + }, + { + "selected": false, + "text": "Geometry / annotation rules", + "value": "Geometry / annotation rules" + }, + { + "selected": false, + "text": "Image compression errors", + "value": "Image compression errors" + }, + { + "selected": false, + "text": "Import/Export / Datumaro", + "value": "Import/Export / Datumaro" + }, + { + "selected": false, + "text": "Invalid image file", + "value": "Invalid image file" + }, + { + "selected": false, + "text": "JS DOM API / Selectors", + "value": "JS DOM API / Selectors" + }, + { + "selected": false, + "text": "JS type/method errors", + "value": "JS type/method errors" + }, + { + "selected": false, + "text": "Labels not registered", + "value": "Labels not registered" + }, + { + "selected": false, + "text": "No available server / service unavailable", + "value": "No available server / service unavailable" + }, + { + "selected": false, + "text": "OS error / file system", + "value": "OS error / file system" + }, + { + "selected": false, + "text": "Postgres / DB (integrity)", + "value": "Postgres / DB (integrity)" + }, + { + "selected": false, + "text": "Postgres / DB (operational)", + "value": "Postgres / DB (operational)" + }, + { + "selected": false, + "text": "Quotas / size limits", + "value": "Quotas / size limits" + }, + { + "selected": false, + "text": "Quotas / size limits (image bomb)", + "value": "Quotas / size limits (image bomb)" + }, + { + "selected": false, + "text": "React / DOM", + "value": "React / DOM" + }, + { + "selected": false, + "text": "Redis / kvrocks", + "value": "Redis / kvrocks" + }, + { + "selected": false, + "text": "Redis lock / concurrency", + "value": "Redis lock / concurrency" + }, + { + "selected": false, + "text": "Timeouts / background jobs", + "value": "Timeouts / background jobs" + }, + { + "selected": false, + "text": "Uniqueness / names / ids", + "value": "Uniqueness / names / ids" + }, + { + "selected": false, + "text": "Unsupported / invalid input", + "value": "Unsupported / invalid input" + }, + { + "selected": false, + "text": "Webhook delivery errors", + "value": "Webhook delivery errors" + }, + { + "selected": false, + "text": "JS read-only property assignment", + "value": "JS read-only property assignment" + }, + { + "selected": false, + "text": "Client validation / frame bounds", + "value": "Client validation / frame bounds" + } + ], + "query": "Network Error,JS null/undefined access,500 Server Error,502 Bad Gateway,504 Gateway Timeout,Other / uncategorized,AV / codec errors,Attributes invalid value,CVAT-core / Business Logic,Canvas / WebGL / memory,Cloud storage / config error,Cloud storage / file missing,Cloud storage / file not found,Concurrent export/import / duplicate request,DB duplicate key,Debug logging / shapes dump,File not found / missing file,Geometry / annotation rules,Image compression errors,Import/Export / Datumaro,Invalid image file,JS DOM API / Selectors,JS type/method errors,Labels not registered,No available server / service unavailable,OS error / file system,Postgres / DB (integrity),Postgres / DB (operational),Quotas / size limits,Quotas / size limits (image bomb),React / DOM,Redis / kvrocks,Redis lock / concurrency,Timeouts / background jobs,Uniqueness / names / ids,Unsupported / invalid input,Webhook delivery errors,JS read-only property assignment,Client validation / frame bounds", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "custom", + "definition": "Network Error,JS null/undefined access,500 Server Error,502 Bad Gateway,504 Gateway Timeout,Other / uncategorized,AV / codec errors,Attributes invalid value,CVAT-core / Business Logic,Canvas / WebGL / memory,Cloud storage / config error,Cloud storage / file missing,Cloud storage / file not found,Concurrent export/import / duplicate request,DB duplicate key,Debug logging / shapes dump,File not found / missing file,Geometry / annotation rules,Image compression errors,Import/Export / Datumaro,Invalid image file,JS DOM API / Selectors,JS type/method errors,Labels not registered,No available server / service unavailable,OS error / file system,Postgres / DB (integrity),Postgres / DB (operational),Quotas / size limits,Quotas / size limits (image bomb),React / DOM,Redis / kvrocks,Redis lock / concurrency,Timeouts / background jobs,Uniqueness / names / ids,Unsupported / invalid input,Webhook delivery errors,JS read-only property assignment,Client validation / frame bounds" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Users Exceptions", + "uid": "user-exceptions", + "version": 4, + "weekStart": "" +} diff --git a/components/analytics/grafana_conf.yml b/components/analytics/grafana_conf.yml new file mode 100644 index 0000000..f36a300 --- /dev/null +++ b/components/analytics/grafana_conf.yml @@ -0,0 +1,39 @@ +http: + routers: + grafana: + entryPoints: + - web + middlewares: + - analytics-auth + - strip-prefix + service: grafana + rule: Host(`{{ env "CVAT_HOST" }}`) && PathPrefix(`/analytics`) + grafana_https: + entryPoints: + - websecure + middlewares: + - analytics-auth + - strip-prefix + service: grafana + tls: {} + rule: Host(`{{ env "CVAT_HOST" }}`) && PathPrefix(`/analytics`) + + middlewares: + analytics-auth: + forwardauth: + address: http://cvat_server:8080/analytics + authRequestHeaders: + - "Cookie" + - "Authorization" + + strip-prefix: + stripprefix: + prefixes: + - /analytics + + services: + grafana: + loadBalancer: + servers: + - url: http://{{ env "DJANGO_LOG_VIEWER_HOST" }}:{{ env "DJANGO_LOG_VIEWER_PORT" }} + passHostHeader: false diff --git a/components/analytics/vector/vector.toml b/components/analytics/vector/vector.toml new file mode 100644 index 0000000..c6288b4 --- /dev/null +++ b/components/analytics/vector/vector.toml @@ -0,0 +1,47 @@ +data_dir = "/vector-data-dir" + +[sources.http-events] +type = "http_server" +address = "0.0.0.0:8282" +encoding = "json" + +# Uncomment for debug +# [sinks.console] +# type = "console" +# inputs = [ "http-events" ] +# target = "stdout" + +# [sinks.console.encoding] +# codec = "json" + +[sinks.clickhouse] +inputs = [ "http-events" ] +type = "clickhouse" +database = "${CLICKHOUSE_DB}" +table = "events" +auth.strategy = "basic" +auth.user = "${CLICKHOUSE_USER}" +auth.password = "${CLICKHOUSE_PASSWORD}" +endpoint = "http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}" +request.concurrency = "adaptive" +encoding.only_fields = [ + "scope", + "obj_name", + "obj_id", + "obj_val", + "source", + "timestamp", + "count", + "duration", + "project_id", + "task_id", + "job_id", + "user_id", + "user_name", + "user_email", + "org_id", + "org_slug", + "payload", + "access_token_id", + "remote_addr", +] diff --git a/components/serverless/README.md b/components/serverless/README.md new file mode 100644 index 0000000..30cc53f --- /dev/null +++ b/components/serverless/README.md @@ -0,0 +1,8 @@ +## Serverless for Computer Vision Annotation Tool (CVAT) + +### Run docker container + +```bash +# From project root directory +docker compose -f docker-compose.yml -f components/serverless/docker-compose.serverless.yml up -d +``` diff --git a/components/serverless/docker-compose.serverless.yml b/components/serverless/docker-compose.serverless.yml new file mode 100644 index 0000000..8f03ff3 --- /dev/null +++ b/components/serverless/docker-compose.serverless.yml @@ -0,0 +1,30 @@ +services: + nuclio: + container_name: nuclio + image: quay.io/nuclio/dashboard:1.16.3-amd64 + restart: always + networks: + - cvat + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + http_proxy: + https_proxy: + no_proxy: ${no_proxy:-} + NUCLIO_CHECK_FUNCTION_CONTAINERS_HEALTHINESS: 'true' + NUCLIO_DASHBOARD_DEFAULT_FUNCTION_MOUNT_MODE: 'volume' + logging: + driver: "json-file" + options: + max-size: 100m + max-file: "3" + + cvat_server: + environment: + CVAT_SERVERLESS: 1 + extra_hosts: + - "host.docker.internal:host-gateway" + + cvat_worker_annotation: + extra_hosts: + - "host.docker.internal:host-gateway" diff --git a/cvat-canvas/.eslintrc.cjs b/cvat-canvas/.eslintrc.cjs new file mode 100644 index 0000000..463e1e6 --- /dev/null +++ b/cvat-canvas/.eslintrc.cjs @@ -0,0 +1,27 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +const { join } = require('path'); + +module.exports = { + ignorePatterns: [ + '.eslintrc.cjs', + 'webpack.config.js', + 'node_modules/**', + 'dist/**', + ], + parserOptions: { + project: './tsconfig.json', + tsconfigRootDir: __dirname, + }, + rules: { + 'import/no-extraneous-dependencies': [ + 'error', + { + packageDir: [__dirname, join(__dirname, '../')] + }, + ], + } +}; diff --git a/cvat-canvas/README.md b/cvat-canvas/README.md new file mode 100644 index 0000000..26811e5 --- /dev/null +++ b/cvat-canvas/README.md @@ -0,0 +1,134 @@ +# Module CVAT-CANVAS + +## Description + +The CVAT module written in TypeScript language. +It presents a canvas to viewing, drawing and editing of annotations. + +## Commands + +- Building of the module from sources in the `dist` directory: + +```bash +yarn run build +yarn run build --mode=development # without a minification +``` + +### API Methods + +For API methods, their arguments and return types, please look at ``canvas.ts``. + +### API CSS + +- All drawn objects (shapes, tracks) have an id `cvat_canvas_shape_{objectState.clientID}` +- Drawn shapes and tracks have classes `cvat_canvas_shape`, + `cvat_canvas_shape_activated`, + `cvat_canvas_shape_selection`, + `cvat_canvas_shape_merging`, + `cvat_canvas_shape_drawing`, + `cvat_canvas_shape_occluded` +- Drawn review ROIs have an id `cvat_canvas_issue_region_{issue.id}` +- Drawn review roi has the class `cvat_canvas_issue_region` +- Drawn texts have the class `cvat_canvas_text` +- Tags have the class `cvat_canvas_tag` +- Canvas image has ID `cvat_canvas_image` +- Grid on the canvas has ID `cvat_canvas_grid` and `cvat_canvas_grid_pattern` +- Crosshair during a draw has class `cvat_canvas_crosshair` +- To stick something to a specific position you can use an element with id `cvat_canvas_attachment_board` + +### Events + +Standard JS events are used. + +```js + - canvas.setup + - canvas.activated => {state: ObjectState} + - canvas.clicked => {state: ObjectState} + - canvas.moved => {states: ObjectState[], x: number, y: number} + - canvas.find => {states: ObjectState[], x: number, y: number} + - canvas.drawn => {state: DrawnData} + - canvas.interacted => {shapes: InteractionResult[]} + - canvas.editstart + - canvas.edited => {state: ObjectState, points: number[], rotation?: number} + - canvas.splitted => {state: ObjectState, frame: number, duration: number} + - canvas.grouped => {states: ObjectState[], duration: number} + - canvas.joined => {states: ObjectState[], points: number[], duration: number} + - canvas.sliced => {state: ObjectState, results: number[][], duration: number} + - canvas.merged => {states: ObjectState[], duration: number} + - canvas.canceled + - canvas.dragstart + - canvas.dragstop + - canvas.zoomstart + - canvas.zoomstop + - canvas.zoom + - canvas.reshape + - canvas.fit + - canvas.regionselected => {points: number[]} + - canvas.dragshape => {duration: number, state: ObjectState} + - canvas.roiselected => {points: number[]} + - canvas.resizeshape => {duration: number, state: ObjectState} + - canvas.contextmenu => { mouseEvent: MouseEvent, objectState: ObjectState, pointID: number } + - canvas.message => { messages: { type: 'text' | 'list'; content: string | string[]; className?: string; icon?: 'info' | 'loading' }[] | null, topic: string } + - canvas.error => { exception: Error, domain?: string } + - canvas.destroy +``` + +### WEB + +```js +// Create an instance of a canvas +const canvas = new window.canvas.Canvas(); + +console.log('Version ', window.canvas.CanvasVersion); +console.log('Current mode is ', window.canvas.mode()); + +// Put canvas to a html container +htmlContainer.appendChild(canvas.html()); +canvas.fitCanvas(); + +// Next you can use its API methods. For example: +canvas.rotate(270); +canvas.draw({ + enabled: true, + shapeType: 'rectangle', + crosshair: true, + rectDrawingMethod: window.Canvas.RectDrawingMethod.CLASSIC, +}); +``` + + + +## API Reaction + +| | IDLE | GROUP | SPLIT | DRAW | MERGE | EDIT | DRAG | RESIZE | ZOOM_CANVAS | DRAG_CANVAS | INTERACT | JOIN | SLICE | SELECT_REGION | +| -------------- | ---- | ----- | ----- | ---- | ----- | ---- | ---- | ------ | ----------- | ----------- | -------- | ---- | ----- | ------------- | +| setup() | + | + | + | +/- | + | +/- | +/- | +/- | + | + | + | + | + | + | +| activate() | + | - | - | - | - | - | - | - | - | - | - | - | - | - | +| rotate() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| focus() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| fit() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| grid() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| draw() | + | - | - | + | - | - | - | - | - | - | - | - | - | - | +| interact() | + | - | - | - | - | - | - | - | - | - | + | - | - | - | +| split() | + | - | + | - | - | - | - | - | - | - | - | - | - | - | +| group() | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +| merge() | + | - | - | - | + | - | - | - | - | - | - | - | - | - | +| edit() | + | - | - | - | - | + | - | - | - | - | - | - | - | - | +| join() | + | - | - | - | - | - | - | - | - | - | - | + | - | - | +| slice() | + | - | - | - | - | - | - | - | - | - | - | - | + | - | +| selectRegion() | + | - | - | - | - | - | - | - | - | - | - | - | - | + | +| fitCanvas() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| dragCanvas() | + | - | - | - | - | - | + | - | - | + | - | - | - | - | +| zoomCanvas() | + | - | - | - | - | - | - | + | + | - | - | - | - | - | +| cancel() | - | + | + | + | + | + | + | + | + | + | + | + | + | + | +| configure() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| bitmap() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| setZLayer() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | +| destroy() | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + + + +You can call setup() during editing, dragging, and resizing only to update objects, not to change a frame. +You can change frame during draw only when you do not redraw an existing object + +Other methods do not change state and can be used at any time. diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json new file mode 100644 index 0000000..a51c650 --- /dev/null +++ b/cvat-canvas/package.json @@ -0,0 +1,28 @@ +{ + "name": "cvat-canvas", + "version": "2.20.10", + "type": "module", + "description": "Part of Computer Vision Annotation Tool which presents its canvas library", + "main": "src/canvas.ts", + "scripts": { + "build": "tsc && webpack --config ./webpack.config.cjs" + }, + "author": "CVAT.ai", + "license": "MIT", + "browserslist": [ + "Chrome >= 99", + "Firefox >= 110", + "not IE 11", + "> 2%" + ], + "dependencies": { + "@types/polylabel": "^1.0.5", + "martinez-polygon-clipping": "^0.8.1", + "polylabel": "^1.1.0", + "svg.draggable.js": "2.2.2", + "svg.draw.js": "^2.0.4", + "svg.js": "2.7.1", + "svg.resize.js": "1.4.3", + "svg.select.js": "3.0.1" + } +} diff --git a/cvat-canvas/src/scss/canvas.scss b/cvat-canvas/src/scss/canvas.scss new file mode 100644 index 0000000..a0bad09 --- /dev/null +++ b/cvat-canvas/src/scss/canvas.scss @@ -0,0 +1,484 @@ +// Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/* stylelint-disable selector-class-pattern, selector-id-pattern */ + +.cvat_canvas_hidden { + display: none; +} + +.cvat_canvas_shape { + stroke-opacity: 1; +} + +g.cvat_canvas_shape { + > circle { + fill-opacity: 1; + } +} + +polyline.cvat_canvas_shape { + fill-opacity: 0; +} + +.cvat_shape_action_opacity { + fill-opacity: 0.5; + stroke-opacity: 1; +} + +polyline.cvat_shape_action_opacity { + fill-opacity: 0; +} + +.cvat_shape_drawing_opacity { + stroke-opacity: 1; +} + +polyline.cvat_shape_drawing_opacity { + fill-opacity: 0; +} + +.cvat_shape_action_dasharray { + stroke-dasharray: 4 1 2 3; +} + +.cvat_canvas_text { + font-weight: bold; + fill: white; + cursor: default; + filter: drop-shadow(1px 1px 1px black) drop-shadow(-1px -1px 1px black); + user-select: none; + pointer-events: none; +} + +.cvat_canvas_text_dimensions { + fill: lightskyblue; +} + +.cvat_canvas_text_layer { + fill: #73d13d; +} + +.cvat_canvas_text_description { + fill: yellow; + font-style: oblique 40deg; +} + +.cvat_canvas_text_score { + fill: #FFB347; + font-style: oblique 10deg; +} + +.cvat_canvas_crosshair { + stroke: red; +} + +.cvat_canvas_shape_selection { + @extend .cvat_shape_action_dasharray; + @extend .cvat_shape_action_opacity; + + fill: #fcfbfc; +} + +image.cvat_canvas_shape_selection { + visibility: hidden; +} + +.cvat_canvas_selection_box { + fill: white; + fill-opacity: 0.1; + stroke: white; +} + +.cvat_canvas_shape_region_selection { + @extend .cvat_shape_action_dasharray; + @extend .cvat_shape_action_opacity; + + fill: white; + stroke: white; +} + +.cvat_canvas_issue_region { + pointer-events: none; + stroke-width: 0; +} + +circle.cvat_canvas_issue_region { + opacity: 1 !important; +} + +polyline.cvat_canvas_shape_selection { + @extend .cvat_shape_action_dasharray; + @extend .cvat_shape_action_opacity; + + stroke: darkmagenta; +} + +.cvat_canvas_shape_merging { + @extend .cvat_shape_action_dasharray; + @extend .cvat_shape_action_opacity; + + fill: blue; + + > circle[data-node-id] { + fill: blue; + } +} + +polyline.cvat_canvas_shape_merging { + @extend .cvat_shape_action_dasharray; + @extend .cvat_shape_action_opacity; + + stroke: blue; +} + +polyline.cvat_canvas_shape_splitting { + @extend .cvat_shape_action_dasharray; + @extend .cvat_shape_action_opacity; + + stroke: dodgerblue; +} + +.cvat_canvas_shape_splitting { + @extend .cvat_shape_action_dasharray; + @extend .cvat_shape_action_opacity; + + fill: dodgerblue; +} + +.cvat_canvas_shape_drawing { + @extend .cvat_shape_drawing_opacity; + + fill: white; +} + +.cvat_canvas_zoom_selection { + @extend .cvat_shape_action_dasharray; + + stroke: #096dd9; + fill-opacity: 0; +} + +.cvat_canvas_shape_occluded { + stroke-dasharray: 5; +} + +.cvat_canvas_ground_truth { + stroke-dasharray: 1; +} + +.cvat_canvas_conflicted { + stroke: #ff4800; + fill: #ff4800; + + rect, + ellipse, + polygon, + polyline, + line { + fill: #ff4800; + stroke: #ff4800; + } + + circle { + fill: #ff4800; + } +} + +.cvat_canvas_warned { + stroke: #ff7301; + fill: #ff7301; + + rect, + ellipse, + polygon, + polyline, + line { + fill: #ff7301; + stroke: #ff7301; + } + + circle { + fill: #ff7301; + } +} + +.cvat_canvas_shape_occluded_point { + stroke-dasharray: 1 !important; + stroke: white; +} + +circle.cvat_canvas_shape_occluded { + @extend .cvat_canvas_shape_occluded_point; +} + +g.cvat_canvas_shape_occluded { + > rect { + stroke-dasharray: 5; + } + + > circle { + @extend .cvat_canvas_shape_occluded_point; + } +} + +.svg_select_points_rot { + fill: white; +} + +.cvat_canvas_shape .svg_select_points, +.cvat_canvas_shape .cvat_canvas_cuboid_projections { + stroke-dasharray: none; +} + +.cvat_canvas_autoborder_point { + transition: opacity 0.05s; + opacity: 1; +} + +.cvat_canvas_autoborder_point:hover { + opacity: 0.5; +} + +.cvat_canvas_autoborder_point:active { + opacity: 1; +} + +.cvat_canvas_interact_intermediate_shape { + @extend .cvat_canvas_shape; +} + +.cvat_interaction_delete_button { + opacity: 0.25; + + &:hover { + opacity: 1; + } +} + +.cvat_interaction_rectangle { + @extend .cvat_canvas_shape_drawing; +} + +.cvat_interaction_delete_button, .cvat_interaction_point { + transition: stroke-width 0.2s ease-in-out, opacity 0.2s ease-in-out; + filter: drop-shadow(0 1px 3px rgba(0, 0, 0, 30%)); +} + +.cvat_canvas_removable_interaction_point { + cursor: + url( + 'data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAiIGhlaWdodD0iMjAiIHZpZXdCb3g9IjAgMCAxMCAxMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTEgMUw5IDlNMSA5TDkgMSIgc3Ryb2tlPSJibGFjayIvPgo8L3N2Zz4K' + ) 10 10, + auto; +} + +.svg_select_boundingRect { + opacity: 0; + pointer-events: none; +} + +.svg_select_points_lb:hover, +.svg_select_points_rt:hover { + cursor: nesw-resize; +} + +.svg_select_points_lt:hover, +.svg_select_points_rb:hover { + cursor: nwse-resize; +} + +.svg_select_points_l:hover, +.svg_select_points_r:hover, +.svg_select_points_ew:hover { + cursor: ew-resize; +} + +.svg_select_points_t:hover, +.svg_select_points_b:hover { + cursor: ns-resize; +} + +.cvat_canvas_shape_draggable:hover { + cursor: move; +} + +.cvat_canvas_first_poly_point { + fill: lightgray; +} + +.cvat_canvas_poly_direction { + fill: lightgray; + stroke: black; + + &:hover { + fill: black; + stroke: lightgray; + } + + &:active { + fill: lightgray; + stroke: black; + } +} + +.cvat_canvas_skeleton_wrapping_rect { + // wrapping rect must not apply transform attribute from selectize.js + // otherwise it rotated twice, because we apply the same rotation value to parent element (skeleton itself) + transform: none !important; +} + +.cvat_canvas_shape > .cvat_canvas_skeleton_wrapping_rect { + visibility: hidden; +} + +.cvat_canvas_shape.cvat_canvas_shape_activated > .cvat_canvas_skeleton_wrapping_rect { + visibility: initial; +} + +.cvat_canvas_pixelized { + image-rendering: optimizeSpeed; + image-rendering: optimize-contrast; + image-rendering: crisp-edges; + image-rendering: pixelated; +} + +.cvat_canvas_removed_image { + filter: saturate(0) brightness(1.2) contrast(0.75) !important; +} + +#cvat_canvas_wrapper { + width: calc(100% - 10px); + height: calc(100% - 10px); + margin: 5px; + border-radius: 5px; + background-color: inherit; + overflow: hidden; + position: relative; +} + +.cvat-canvas-highlight-enabled { + svg { + >rect:not(.cvat_canvas_issue_region,.cvat_canvas_conflicted,.cvat_canvas_warned), + >ellipse:not(.cvat_canvas_issue_region,.cvat_canvas_conflicted,.cvat_canvas_warned), + >polygon:not(.cvat_canvas_issue_region,.cvat_canvas_conflicted,.cvat_canvas_warned), + >polyline:not(.cvat_canvas_issue_region,.cvat_canvas_conflicted,.cvat_canvas_warned), + >line:not(.cvat_canvas_issue_region,.cvat_canvas_conflicted,.cvat_canvas_warned) { + fill: gray; + stroke: gray; + } + + >circle:not(.cvat_canvas_issue_region,.cvat_canvas_conflicted,.cvat_canvas_warned) { + fill: gray; + } + + >g:not(.cvat_canvas_issue_region,.cvat_canvas_conflicted,.cvat_canvas_warned) { + rect, + ellipse, + polygon, + polyline, + line { + fill: gray; + stroke: gray; + } + + circle { + fill: gray; + } + } + } +} + +#cvat_canvas_text_content { + text-rendering: optimizeSpeed; + position: absolute; + z-index: 3; + pointer-events: none; + width: 100%; + height: 100%; +} + +#cvat_canvas_background { + position: absolute; + z-index: 0; + background-repeat: no-repeat; + width: 100%; + height: 100%; + box-shadow: 2px 2px 5px 0 rgba(0, 0, 0, 75%); +} + +#cvat_canvas_bitmap { + @extend .cvat_canvas_pixelized; + + pointer-events: none; + position: absolute; + z-index: 4; + background: black; + width: 100%; + height: 100%; + box-shadow: 2px 2px 5px 0 rgba(0, 0, 0, 75%); +} + +#cvat_canvas_grid { + position: absolute; + z-index: 2; + pointer-events: none; + width: 100%; + height: 100%; +} + +#cvat_canvas_grid_pattern { + opacity: 1; + stroke: white; +} + +#cvat_canvas_content { + @extend .cvat_canvas_pixelized; + + filter: contrast(120%) saturate(150%); + position: absolute; + z-index: 2; + outline: 10px solid black; + width: 100%; + height: 100%; +} + +.cvat_masks_canvas_wrapper { + @extend .cvat_canvas_pixelized; + + z-index: 3; + display: none; +} + +#cvat_canvas_attachment_board { + position: absolute; + z-index: 4; + pointer-events: none; + width: 100%; + height: 100%; + user-select: none; +} + +.cvat_canvas_shape_darken { + fill: #838383; + stroke: #838383; +} + +.cvat_canvas_sliced_contour { + fill-opacity: 0.01; +} + +.cvat_canvas_slicing_line { + pointer-events: none; + fill-opacity: 0; +} + +.cvat-canvas-notification-list-warning { + color: orange; +} + +.cvat-canvas-notification-list-shortcuts { + color: yellow; +} diff --git a/cvat-canvas/src/typescript/autoborderHandler.ts b/cvat-canvas/src/typescript/autoborderHandler.ts new file mode 100644 index 0000000..51363b9 --- /dev/null +++ b/cvat-canvas/src/typescript/autoborderHandler.ts @@ -0,0 +1,505 @@ +// Copyright (C) CVAT.ai Corporation +// Copyright (C) 2020-2022 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; + +import consts from './consts'; +import { Configuration, Geometry } from './canvasModel'; +import { translateToSVG } from './shared'; + +interface TransformedShape { + points: number[][]; + color: string; +} + +export interface AutoborderHandler { + autoborder(enabled: boolean, currentShape?: SVG.Shape, excludedClientId?: number): void; + configure(configuration: Configuration): void; + transform(geometry: Geometry): void; + updateObjects(): void; +} + +function collectSegmentPoints( + points: number[][], + startPointID: number, + endPointID: number, + direction: -1 | 1 | null, +): number[][] { + if (startPointID === endPointID) { + return []; + } + + if ( + Math.abs(startPointID - endPointID) === 1 || + Math.abs(startPointID - endPointID) === points.length - 1 + ) { + // adjacent points, no need to calculate anything + return [points[endPointID]]; + } + + const walk = (step: number): { + length: number; + points: number[][]; + } => { + let length = 0; + let previousPoint = points[startPointID]; + const curvePoints: number[][] = []; + + for (let i = startPointID + step; ; i += step) { + if (i < 0) { + i = points.length - 1; + } else if (i === points.length) { + i = 0; + } + + const currentPoint = points[i]; + const dx = currentPoint[0] - previousPoint[0]; + const dy = currentPoint[1] - previousPoint[1]; + + length += Math.hypot(dx, dy); + curvePoints.push([...currentPoint]); + + if (i === endPointID) { + break; + } + + previousPoint = currentPoint; + } + + return { + length, + points: curvePoints, + }; + }; + + if (direction === 1) { + return walk(1).points; + } + + if (direction === -1) { + return walk(-1).points; + } + + const forward = walk(1); + const backward = walk(-1); + return forward.length <= backward.length ? forward.points : backward.points; +} + +type PointIndexesAndDirection = { + addedPointIndexes: Set; + direction: -1 | 1 | null; +}; + +function collectAddedPointIndexesAndDirection( + points: number[][], + pointsToRevertPreview: number[][], +): PointIndexesAndDirection { + const pointKey = (point: number[]): string => `${point[0]},${point[1]}`; + const addedPointIndexes = new Set(); + const indexesByPointKey = new Map(); + + for (let i = 0; i < points.length; i++) { + const key = pointKey(points[i]); + const indexes = indexesByPointKey.get(key); + + if (indexes) { + indexes.push(i); + } else { + indexesByPointKey.set(key, [i]); + } + } + + const originalPointKeys = new Set(pointsToRevertPreview.map(pointKey)); + for (let i = 0; i < points.length; i++) { + if (originalPointKeys.has(pointKey(points[i]))) { + addedPointIndexes.add(i); + } + } + + if (points.length < 2 || pointsToRevertPreview.length < 2) { + return { + addedPointIndexes, + direction: null, + }; + } + + const previousKey = pointKey(pointsToRevertPreview[pointsToRevertPreview.length - 2]); + const lastKey = pointKey(pointsToRevertPreview[pointsToRevertPreview.length - 1]); + const previousIndexes = indexesByPointKey.get(previousKey) || []; + const lastIndexes = new Set(indexesByPointKey.get(lastKey) || []); + + for (const previousIndex of previousIndexes) { + const nextIndex = (previousIndex + 1) % points.length; + const prevIndex = (previousIndex - 1 + points.length) % points.length; + + if (lastIndexes.has(nextIndex)) { + return { + addedPointIndexes, + direction: 1, + }; + } + + if (lastIndexes.has(prevIndex)) { + return { + addedPointIndexes, + direction: -1, + }; + } + } + + return { + addedPointIndexes, + direction: null, + }; +} + +export class AutoborderHandlerImpl implements AutoborderHandler { + private currentShape: SVG.Shape | null; + private excludedClientId?: number; + private container: SVGSVGElement; + private pointGroups: SVGGElement[]; + private enabled: boolean; + private scale: number; + private controlPointsSize: number; + private visibleShapes: TransformedShape[]; + private currentClick: { groupIdx: number; pointIdx: number; } | null; + private currentPreview: { pointIdx: number; shapePoints: number[][]; } | null; + private pointsToRevertPreview: number[][] | null; + private listeners: Map void; }>; + private isCtrlKeyDown: (() => boolean) | null; + + public constructor(container: SVGSVGElement, isCtrlKeyDown: () => boolean) { + this.currentShape = null; + this.excludedClientId = undefined; + this.container = container; + this.pointGroups = []; + this.enabled = false; + this.scale = 1; + this.controlPointsSize = consts.BASE_POINT_SIZE; + this.visibleShapes = []; + this.currentClick = null; + this.currentPreview = null; + this.pointsToRevertPreview = null; + this.listeners = new Map(); + this.isCtrlKeyDown = isCtrlKeyDown; + } + + private removeMarkers(): void { + this.pointGroups.forEach((group: SVGGElement): void => { + Array.from(group.children).forEach((circle: SVGCircleElement): void => { + const listeners = this.listeners.get(circle); + if (listeners) { + circle.removeEventListener('mousedown', listeners.mousedown); + } + circle.remove(); + }); + + group.remove(); + }); + + this.pointGroups = []; + this.currentClick = null; + this.listeners.clear(); + } + + private readCurrentShapePoints(): number[][] { + return (this.currentShape as any).array().valueOf().map( + (shapePoint: number[]): number[] => [...shapePoint], + ); + } + + private replaceCurrentShapePoints(points: number[][]): void { + const paintHandler = this.currentShape?.remember('_paintHandler'); + if (!paintHandler) { + return; + } + + const lastPoint = (this.currentShape as any).array().valueOf().slice(-1)[0]; + (this.currentShape as any).plot([...points, lastPoint]); + + paintHandler.drawCircles(); + paintHandler.set.members.forEach((el: SVG.Circle): void => { + el.attr('stroke-width', 1 / this.scale).attr('r', 2.5 / this.scale); + el.attr('r', `${this.controlPointsSize / this.scale}`); + }); + } + + private raiseMarkers(): void { + for (let i = 0; i < this.pointGroups.length; i++) { + this.container.appendChild(this.pointGroups[i]); + } + + if (this.currentClick !== null) { + this.container.appendChild(this.pointGroups[this.currentClick.groupIdx]); + } + } + + private drawMarkers(): void { + this.removeMarkers(); + + const ns = 'http://www.w3.org/2000/svg'; + this.pointGroups = this.visibleShapes.map( + (shape: TransformedShape, groupIdx: number): SVGGElement => { + const group = document.createElementNS(ns, 'g'); + + const circles = shape.points.map( + (point: number[], pointIdx: number): SVGCircleElement => { + const [x, y] = point; + const circle = document.createElementNS(ns, 'circle'); + circle.classList.add('cvat_canvas_autoborder_point'); + circle.setAttribute('fill', shape.color); + circle.setAttribute('stroke', 'black'); + circle.setAttribute('stroke-width', `${consts.POINTS_STROKE_WIDTH / this.scale}`); + circle.setAttribute('cx', `${x}`); + circle.setAttribute('cy', `${y}`); + circle.setAttribute('r', `${this.controlPointsSize / this.scale}`); + + const mousedown = (event: MouseEvent): void => { + if (event.button !== 0) { + return; + } + + if (this.isCtrlKeyDown && this.isCtrlKeyDown()) { + return; + } + + event.stopPropagation(); + if (this.currentClick?.groupIdx !== groupIdx) { + // first click on this group of points + + // svg.draw.js initializes the internal paint handler lazily, + // so the first autoborder click needs to bootstrap it first. + const handler = this.currentShape.remember('_paintHandler'); + if (!handler || !handler.startPoint) { + (this.currentShape as any).draw('point', event); + (this.currentShape as any).draw('undo'); + } + + this.container.appendChild(group); // raise in DOM over other groups + const originalPoints = this.readCurrentShapePoints().slice(0, -1); + this.replaceCurrentShapePoints([...originalPoints, [x, y]]); + this.currentClick = { groupIdx, pointIdx }; + } + }; + + circle.addEventListener('mousedown', mousedown); + this.listeners.set(circle, { mousedown }); + return circle; + }, + ); + + group.append(...circles); + return group; + }, + ); + + this.container.append(...this.pointGroups); + this.raiseMarkers(); + } + + private findClosestPointInClickedGroup(e: MouseEvent): number { + let closestPointIdx = -1; + if (this.currentClick === null) { + // group not selected, do nothing + return closestPointIdx; + } + + const { points } = this.visibleShapes[this.currentClick.groupIdx]; + const [x, y] = translateToSVG(this.container, [e.clientX, e.clientY]); + let distance = (consts.BASE_POINT_SIZE * 2) / this.scale; + + for (let i = 0; i < points.length; i++) { + const point = points[i]; + const dx = point[0] - x; + const dy = point[1] - y; + const currentDistance = Math.hypot(dx, dy); + if (currentDistance <= distance) { + distance = currentDistance; + closestPointIdx = i; + } + } + + return closestPointIdx; + } + + private onContainerMouseMove = (e: MouseEvent): void => { + if (this.isCtrlKeyDown && this.isCtrlKeyDown()) { + return; + } + + const closestPointIdx = this.findClosestPointInClickedGroup(e); + + if (this.pointsToRevertPreview) { + this.replaceCurrentShapePoints(this.pointsToRevertPreview); + this.pointsToRevertPreview = null; + this.currentPreview = null; + } + + if (closestPointIdx === -1) { + return; + } + + const { points } = this.visibleShapes[this.currentClick.groupIdx]; + const { + addedPointIndexes, + direction, + } = collectAddedPointIndexesAndDirection(points, this.readCurrentShapePoints().slice(0, -1)); + + if (addedPointIndexes.has(closestPointIdx)) { + return; + } + + const curvePoints = collectSegmentPoints(points, this.currentClick.pointIdx, closestPointIdx, direction); + this.pointsToRevertPreview = this.readCurrentShapePoints().slice(0, -1); + this.currentPreview = { + pointIdx: closestPointIdx, + shapePoints: [...this.pointsToRevertPreview, ...curvePoints], + }; + this.replaceCurrentShapePoints(this.currentPreview.shapePoints); + }; + + private onContainerMouseDown = (e: MouseEvent): void => { + if (this.isCtrlKeyDown && this.isCtrlKeyDown()) { + return; + } + + if (e.button !== 0 || this.currentPreview === null) { + return; + } + + const closestPointIdx = this.findClosestPointInClickedGroup(e); + if (closestPointIdx === this.currentPreview.pointIdx) { + e.stopPropagation(); + this.currentPreview = null; + this.pointsToRevertPreview = null; + this.currentClick.pointIdx = closestPointIdx; + } + }; + + private setObjectsFromContainer(): void { + const shapes = Array.from(this.container.getElementsByClassName('cvat_canvas_shape')).filter( + (shape: HTMLElement): boolean => !shape.classList.contains('cvat_canvas_hidden'), + ); + + this.visibleShapes = shapes.map((shape: HTMLElement): TransformedShape | null => { + const color = shape.getAttribute('fill'); + const clientId = shape.getAttribute('clientID'); + + const isSupportedType = + shape.tagName === 'polyline' || + shape.tagName === 'polygon' || + shape.tagName === 'rect'; + + if ( + color === null || clientId === null || !isSupportedType || + (typeof this.excludedClientId === 'number' && +clientId === this.excludedClientId) + ) { + return null; + } + + let points = ''; + if (shape.tagName === 'polyline' || shape.tagName === 'polygon') { + points = shape.getAttribute('points'); + } else if (shape.tagName === 'rect') { + const x = +shape.getAttribute('x'); + const y = +shape.getAttribute('y'); + const width = +shape.getAttribute('width'); + const height = +shape.getAttribute('height'); + + if (Number.isNaN(x) || Number.isNaN(y) || Number.isNaN(width) || Number.isNaN(height)) { + return null; + } + + const svgElement = shape as unknown as SVGGraphicsElement; + const ctm = svgElement.getCTM(); + + const localCorners = [ + { x, y }, + { x: x + width, y }, + { x: x + width, y: y + height }, + { x, y: y + height }, + ]; + + if (ctm && (ctm.a !== 1 || ctm.b !== 0 || ctm.c !== 0 || ctm.d !== 1)) { + const transformedCorners = localCorners.map((corner) => { + const transformedX = ctm.a * corner.x + ctm.c * corner.y + ctm.e; + const transformedY = ctm.b * corner.x + ctm.d * corner.y + ctm.f; + return [transformedX, transformedY]; + }); + + return { + color, + points: transformedCorners, + }; + } + + points = `${x},${y} ${x + width},${y} ${x + width},${y + height} ${x},${y + height}`; + } + + return { + color, + points: points.trim().split(/\s/).map( + (shapePoint: string): number[] => shapePoint.split(',') + .map((coordinate: string): number => +coordinate), + ), + }; + }).filter((state: TransformedShape | null): boolean => state !== null); + } + + public updateObjects(): void { + if (this.enabled) { + this.setObjectsFromContainer(); + this.drawMarkers(); + } + } + + public autoborder(enabled: boolean, currentShape?: SVG.Shape, excludedClientId?: number): void { + if (enabled && !this.enabled && currentShape) { + this.enabled = true; + this.currentShape = currentShape; + this.excludedClientId = excludedClientId; + this.updateObjects(); + + this.container.addEventListener('mousemove', this.onContainerMouseMove); + this.container.addEventListener('mousedown', this.onContainerMouseDown, { capture: true }); + this.currentShape.on('undopoint', (): void => { + this.currentClick = null; + this.currentPreview = null; + + if (this.pointsToRevertPreview) { + this.replaceCurrentShapePoints(this.pointsToRevertPreview); + this.pointsToRevertPreview = null; + } + }); + + this.currentShape.on('drawpoint', (): void => { + this.currentClick = null; + this.currentPreview = null; + this.pointsToRevertPreview = null; + }); + } else { + this.container.removeEventListener('mousemove', this.onContainerMouseMove); + this.container.removeEventListener('mousedown', this.onContainerMouseDown); + this.removeMarkers(); + this.enabled = false; + this.currentShape = null; + } + } + + public transform(geometry: Geometry): void { + this.scale = geometry.scale; + + this.pointGroups.forEach((group: SVGGElement): void => { + Array.from(group.children).forEach((circle: SVGCircleElement): void => { + circle.setAttribute('r', `${this.controlPointsSize / this.scale}`); + circle.setAttribute('stroke-width', `${consts.BASE_STROKE_WIDTH / this.scale}`); + }); + }); + } + + public configure(configuration: Configuration): void { + this.controlPointsSize = configuration.controlPointsSize ?? consts.BASE_POINT_SIZE; + } +} diff --git a/cvat-canvas/src/typescript/canvas.ts b/cvat-canvas/src/typescript/canvas.ts new file mode 100644 index 0000000..b759f88 --- /dev/null +++ b/cvat-canvas/src/typescript/canvas.ts @@ -0,0 +1,204 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { + DrawData, MergeData, SplitData, GroupData, + JoinData, SliceData, MasksEditData, + InteractionData as _InteractionData, + InteractionResult as _InteractionResult, + CanvasModel, CanvasModelImpl, RectDrawingMethod, + CuboidDrawingMethod, Configuration, Geometry, Mode, + HighlightSeverity as _HighlightSeverity, CanvasHint as _CanvasHint, + PolyEditData, RenderData as _RenderData, +} from './canvasModel'; +import { Master } from './master'; +import { CanvasController, CanvasControllerImpl } from './canvasController'; +import { CanvasView, CanvasViewImpl } from './canvasView'; + +import '../scss/canvas.scss'; + +interface Canvas { + html(): HTMLDivElement; + setup(frameData: any, objectStates: any[], renderData?: RenderData): void; + setupIssueRegions(issueRegions: Record): void; + translateFromSVG(points: number[]): number[]; + setupConflictRegions(clientID: number): number[]; + activate(clientID: number | null, attributeID?: number): void; + highlight(clientIDs: number[] | null, severity: HighlightSeverity | null): void; + rotate(rotationAngle: number): void; + focus(clientID: number, padding?: number): void; + fit(): void; + grid(stepX: number, stepY: number): void; + + interact(interactionData: InteractionData): void; + draw(drawData: DrawData): void; + edit(editData: MasksEditData | PolyEditData): void; + group(groupData: GroupData): void; + join(joinData: JoinData): void; + slice(sliceData: SliceData): void; + split(splitData: SplitData): void; + merge(mergeData: MergeData): void; + select(objectState: any): void; + + fitCanvas(): void; + bitmap(enable: boolean): void; + selectRegion(enable: boolean): void; + dragCanvas(enable: boolean): void; + zoomCanvas(enable: boolean): void; + + mode(): Mode; + cancel(): void; + configure(configuration: Configuration): void; + isAbleToChangeFrame(): boolean; + destroy(): void; + + readonly geometry: Geometry; +} + +class CanvasImpl implements Canvas { + private model: CanvasModel & Master; + private controller: CanvasController; + private view: CanvasView; + + public constructor() { + this.model = new CanvasModelImpl(); + this.controller = new CanvasControllerImpl(this.model); + this.view = new CanvasViewImpl(this.model, this.controller); + } + + public html(): HTMLDivElement { + return this.view.html(); + } + + public setup(frameData: any, objectStates: any[], renderData?: RenderData): void { + this.model.setup(frameData, objectStates, renderData); + } + + public setupIssueRegions(issueRegions: Record): void { + this.model.setupIssueRegions(issueRegions); + } + + public translateFromSVG(points: number[]): number[] { + return this.view.translateFromSVG(points); + } + + public setupConflictRegions(clientID: number): number[] { + return this.view.setupConflictRegions(clientID); + } + + public fitCanvas(): void { + this.model.fitCanvas(this.view.html().clientWidth, this.view.html().clientHeight); + } + + public bitmap(enable: boolean): void { + this.model.bitmap(enable); + } + + public selectRegion(enable: boolean): void { + this.model.selectRegion(enable); + } + + public dragCanvas(enable: boolean): void { + this.model.dragCanvas(enable); + } + + public zoomCanvas(enable: boolean): void { + this.model.zoomCanvas(enable); + } + + public activate(clientID: number | null, attributeID: number | null = null): void { + this.model.activate(clientID, attributeID); + } + + public highlight(clientIDs: number[], severity: HighlightSeverity | null = null): void { + this.model.highlight(clientIDs, severity); + } + + public rotate(rotationAngle: number): void { + this.model.rotate(rotationAngle); + } + + public focus(clientID: number, padding = 0): void { + this.model.focus(clientID, padding); + } + + public fit(): void { + this.model.fit(); + } + + public grid(stepX: number, stepY: number): void { + this.model.grid(stepX, stepY); + } + + public interact(interactionData: InteractionData): void { + this.model.interact(interactionData); + } + + public draw(drawData: DrawData): void { + this.model.draw(drawData); + } + + public edit(editData: MasksEditData | PolyEditData): void { + this.model.edit(editData); + } + + public split(splitData: SplitData): void { + this.model.split(splitData); + } + + public group(groupData: GroupData): void { + this.model.group(groupData); + } + + public join(joinData: JoinData): void { + this.model.join(joinData); + } + + public slice(sliceData: SliceData): void { + this.model.slice(sliceData); + } + + public merge(mergeData: MergeData): void { + this.model.merge(mergeData); + } + + public select(objectState: any): void { + this.model.select(objectState); + } + + public mode(): Mode { + return this.model.mode; + } + + public cancel(): void { + this.model.cancel(); + } + + public configure(configuration: Configuration): void { + this.model.configure(configuration); + } + + public isAbleToChangeFrame(): boolean { + return this.model.isAbleToChangeFrame(); + } + + public get geometry(): Geometry { + return this.model.geometry; + } + + public destroy(): void { + this.model.destroy(); + } +} + +export type InteractionData = _InteractionData; +export type CanvasHint = _CanvasHint; +export type InteractionResult = _InteractionResult; +export type HighlightSeverity = _HighlightSeverity; +export type RenderData = _RenderData; + +export { + CanvasImpl as Canvas, RectDrawingMethod, CuboidDrawingMethod, Mode as CanvasMode, +}; diff --git a/cvat-canvas/src/typescript/canvasController.ts b/cvat-canvas/src/typescript/canvasController.ts new file mode 100644 index 0000000..8745b70 --- /dev/null +++ b/cvat-canvas/src/typescript/canvasController.ts @@ -0,0 +1,189 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { + CanvasModel, + Geometry, + Position, + FocusData, + ActiveElement, + DrawData, + MergeData, + SplitData, + GroupData, + JoinData, + SliceData, + Mode, + InteractionData, + Configuration, + MasksEditData, + HighlightedElements, + PolyEditData, + RenderData, +} from './canvasModel'; + +export interface CanvasController { + readonly objects: any[]; + readonly renderData: RenderData; + readonly issueRegions: Record; + readonly focusData: FocusData; + readonly activeElement: ActiveElement; + readonly highlightedElements: HighlightedElements; + readonly drawData: DrawData; + readonly editData: MasksEditData | PolyEditData; + readonly interactionData: InteractionData; + readonly mergeData: MergeData; + readonly splitData: SplitData; + readonly groupData: GroupData; + readonly joinData: JoinData; + readonly sliceData: SliceData; + readonly selected: any; + readonly configuration: Configuration; + mode: Mode; + geometry: Geometry; + + zoom(x: number, y: number, deltaY: number): void; + draw(drawData: DrawData): void; + edit(editData: MasksEditData | PolyEditData): void; + enableDrag(x: number, y: number): void; + drag(x: number, y: number): void; + disableDrag(): void; + fit(): void; + focus(clientId: number, padding: number): void; +} + +export class CanvasControllerImpl implements CanvasController { + private model: CanvasModel; + private lastDragPosition: Position; + private isDragging: boolean; + + public constructor(model: CanvasModel) { + this.model = model; + } + + public zoom(x: number, y: number, deltaY: number): void { + this.model.zoom(x, y, deltaY); + } + + public fit(): void { + this.model.fit(); + } + + public enableDrag(x: number, y: number): void { + this.lastDragPosition = { + x, + y, + }; + this.isDragging = true; + } + + public drag(x: number, y: number): void { + if (this.isDragging) { + const topOffset: number = y - this.lastDragPosition.y; + const leftOffset: number = x - this.lastDragPosition.x; + this.lastDragPosition = { + x, + y, + }; + this.model.move(topOffset, leftOffset); + } + } + + public disableDrag(): void { + this.isDragging = false; + } + + public draw(drawData: DrawData): void { + this.model.draw(drawData); + } + + public edit(editData: MasksEditData | PolyEditData): void { + this.model.edit(editData); + } + + public focus(clientID: number, padding: number): void { + this.model.focus(clientID, padding); + } + + public get geometry(): Geometry { + return this.model.geometry; + } + + public set geometry(geometry: Geometry) { + this.model.geometry = geometry; + } + + public get issueRegions(): Record { + return this.model.issueRegions; + } + + public get objects(): any[] { + return this.model.objects; + } + + public get renderData(): RenderData { + return this.model.renderData; + } + + public get focusData(): FocusData { + return this.model.focusData; + } + + public get activeElement(): ActiveElement { + return this.model.activeElement; + } + + public get highlightedElements(): HighlightedElements { + return this.model.highlightedElements; + } + + public get drawData(): DrawData { + return this.model.drawData; + } + + public get editData(): MasksEditData | PolyEditData { + return this.model.editData; + } + + public get interactionData(): InteractionData { + return this.model.interactionData; + } + + public get mergeData(): MergeData { + return this.model.mergeData; + } + + public get splitData(): SplitData { + return this.model.splitData; + } + + public get groupData(): GroupData { + return this.model.groupData; + } + + public get joinData(): JoinData { + return this.model.joinData; + } + + public get sliceData(): SliceData { + return this.model.sliceData; + } + + public get selected(): any { + return this.model.selected; + } + + public get configuration(): Configuration { + return this.model.configuration; + } + + public set mode(value: Mode) { + this.model.mode = value; + } + + public get mode(): Mode { + return this.model.mode; + } +} diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts new file mode 100644 index 0000000..37c0e44 --- /dev/null +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -0,0 +1,1184 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import consts from './consts'; +import { MasterImpl } from './master'; + +export interface Size { + width: number; + height: number; +} + +export interface Image { + renderWidth: number; + renderHeight: number; + imageData: ImageBitmap; +} + +export interface Position { + x: number; + y: number; +} + +export interface CanvasHint { + type: 'text' | 'list'; + content: string | string[]; + className?: string; + icon?: 'info' | 'loading'; +} + +export interface RenderData { + visibleSkeletonElements: Record; +} + +export interface Geometry { + image: Size; + canvas: Size; + grid: Size; + top: number; + left: number; + scale: number; + offset: number; + angle: number; +} + +export interface FocusData { + clientID: number; +} + +export interface ActiveElement { + clientID: number | null; + attributeID: number | null; +} + +export enum HighlightSeverity { + ERROR = 'error', + WARNING = 'warning', +} + +export interface HighlightedElements { + elementsIDs: number[]; + severity: HighlightSeverity | null; +} + +export enum RectDrawingMethod { + CLASSIC = 'By 2 points', + EXTREME_POINTS = 'By 4 points', +} + +export enum CuboidDrawingMethod { + CLASSIC = 'From rectangle', + CORNER_POINTS = 'By 4 points', +} + +export enum ColorBy { + INSTANCE = 'Instance', + GROUP = 'Group', + LABEL = 'Label', +} + +export interface Configuration { + smoothImage?: boolean; + autoborders?: boolean; + snapToPoint?: boolean; + adaptiveZoom?: boolean; + displayAllText?: boolean; + textFontSize?: number; + textPosition?: 'auto' | 'center'; + textContent?: string; + undefinedAttrValue?: string; + showProjections?: boolean; + showConflicts?: boolean; + forceDisableEditing?: boolean; + intelligentPolygonCrop?: boolean; + forceFrameUpdate?: boolean; + CSSImageFilter?: string; + colorBy?: ColorBy; + selectedShapeOpacity?: number; + shapeOpacity?: number; + controlPointsSize?: number; + outlinedBorders?: string | false; + resetZoom?: boolean; + hideEditedObject?: boolean; + focusedObjectPadding?: number; + snapRadius?: number; +} + +export interface BrushTool { + type: 'brush' | 'eraser' | 'polygon-plus' | 'polygon-minus'; + color: string; + form: 'circle' | 'square'; + size: number; + onBlockUpdated: (blockedTools: Record<'eraser' | 'polygon-minus', boolean>) => void; +} + +export interface DrawData { + enabled: boolean; + continue?: boolean; + shapeType?: string; + rectDrawingMethod?: RectDrawingMethod; + cuboidDrawingMethod?: CuboidDrawingMethod; + skeletonSVG?: SVGSVGElement; + numberOfPoints?: number; + initialState?: any; + crosshair?: boolean; + brushTool?: BrushTool; + redraw?: number; + simplifyPoly?: boolean; + onDrawDone?: (data: object) => void; + onUpdateConfiguration?: (configuration: { brushTool?: Pick }) => void; +} + +export interface InteractionData { + enabled: boolean; + command?: 'draw_points' | 'draw_box' | 'put_shapes' | 'refine'; + payload?: { + shapes: { + shapeType: string; + points: ArrayLike; + }[]; + }; + settings?: { + crosshair?: boolean; // default is false + points_type?: 'any' | 'positive' | 'negative'; // default is any + removalStrategy?: 'any' | 'last'; // default is any + appendCursorPositionAsPoint?: boolean; // default is false + hint?: string; + regionOfInterest?: [number, number, number, number]; + }; +} + +export interface InteractionResult { + points: number[]; + shapeType: string; + type: 'positive' | 'negative'; +} + +export interface PolyEditData { + enabled: boolean; + state?: any; + pointID?: number; +} + +export interface MasksEditData { + enabled: boolean; + state?: any; + brushTool?: BrushTool; + onUpdateConfiguration?: (configuration: { brushTool?: Pick }) => void; +} + +export interface GroupData { + enabled: boolean; +} + +export interface MergeData { + enabled: boolean; +} + +export interface SplitData { + enabled: boolean; +} + +export interface JoinData { + enabled: boolean; +} + +export interface SliceData { + enabled: boolean; + clientID?: number; + getContour?: (state: any) => Promise<[number, number][]>; +} + +export enum FrameZoom { + MIN = 0.1, + MAX = 10, +} + +export enum UpdateReasons { + IMAGE_CHANGED = 'image_changed', + IMAGE_ZOOMED = 'image_zoomed', + IMAGE_FITTED = 'image_fitted', + IMAGE_MOVED = 'image_moved', + IMAGE_ROTATED = 'image_rotated', + GRID_UPDATED = 'grid_updated', + + ISSUE_REGIONS_UPDATED = 'issue_regions_updated', + OBJECTS_UPDATED = 'objects_updated', + SHAPE_ACTIVATED = 'shape_activated', + SHAPE_FOCUSED = 'shape_focused', + SHAPE_HIGHLIGHTED = 'shape_highlighted', + + FITTED_CANVAS = 'fitted_canvas', + + INTERACT = 'interact', + DRAW = 'draw', + EDIT = 'edit', + MERGE = 'merge', + SPLIT = 'split', + GROUP = 'group', + JOIN = 'join', + SLICE = 'slice', + SELECT = 'select', + CANCEL = 'cancel', + BITMAP = 'bitmap', + SELECT_REGION = 'select_region', + DRAG_CANVAS = 'drag_canvas', + ZOOM_CANVAS = 'zoom_canvas', + CONFIG_UPDATED = 'config_updated', + DATA_FAILED = 'data_failed', + DESTROY = 'destroy', +} + +export enum Mode { + IDLE = 'idle', + DRAG = 'drag', + RESIZE = 'resize', + DRAW = 'draw', + EDIT = 'edit', + MERGE = 'merge', + SPLIT = 'split', + GROUP = 'group', + JOIN = 'join', + SLICE = 'slice', + INTERACT = 'interact', + SELECT_REGION = 'select_region', + DRAG_CANVAS = 'drag_canvas', + ZOOM_CANVAS = 'zoom_canvas', +} + +export interface CanvasModel { + readonly imageBitmap: boolean; + readonly imageIsDeleted: boolean; + readonly image: Image | null; + readonly issueRegions: Record; + readonly objects: any[]; + readonly renderData: RenderData; + readonly gridSize: Size; + readonly focusData: FocusData; + readonly activeElement: ActiveElement; + readonly highlightedElements: HighlightedElements; + readonly drawData: DrawData; + readonly editData: MasksEditData | PolyEditData; + readonly interactionData: InteractionData; + readonly mergeData: MergeData; + readonly splitData: SplitData; + readonly groupData: GroupData; + readonly joinData: JoinData; + readonly sliceData: SliceData; + readonly configuration: Configuration; + readonly selected: any; + geometry: Geometry; + mode: Mode; + exception: Error | null; + + zoom(x: number, y: number, deltaY: number): void; + move(topOffset: number, leftOffset: number): void; + + setup(frameData: any, objectStates: any[], renderData?: RenderData): void; + setupIssueRegions(issueRegions: Record): void; + activate(clientID: number | null, attributeID: number | null): void; + highlight(clientIDs: number[], severity: HighlightSeverity): void; + rotate(rotationAngle: number): void; + focus(clientID: number, padding: number): void; + fit(): void; + grid(stepX: number, stepY: number): void; + + draw(drawData: DrawData): void; + edit(editData: MasksEditData | PolyEditData): void; + group(groupData: GroupData): void; + join(joinData: JoinData): void; + slice(sliceData: SliceData): void; + split(splitData: SplitData): void; + merge(mergeData: MergeData): void; + select(objectState: any): void; + interact(interactionData: InteractionData): void; + + fitCanvas(width: number, height: number): void; + bitmap(enabled: boolean): void; + selectRegion(enabled: boolean): void; + dragCanvas(enable: boolean): void; + zoomCanvas(enable: boolean): void; + + isAbleToChangeFrame(): boolean; + configure(configuration: Configuration): void; + cancel(): void; + destroy(): void; +} + +const defaultData = { + drawData: { + enabled: false, + }, + editData: { + enabled: false, + }, + interactionData: { + enabled: false, + }, + mergeData: { + enabled: false, + }, + groupData: { + enabled: false, + }, + splitData: { + enabled: false, + }, + joinData: { + enabled: false, + }, + sliceData: { + enabled: false, + }, +}; + +function hasShapeIsBeingDrawn(): boolean { + const [element] = window.document.getElementsByClassName('cvat_canvas_shape_drawing'); + if (element) { + return !!(element as any).instance.remember('_paintHandler'); + } + + return false; +} + +function disableInternalSVGDrawing(data: DrawData | MasksEditData, currentData: DrawData | MasksEditData): boolean { + // P.S. spaghetti code, but probably significant refactoring needed to find a better solution + // when it is a mask drawing/editing using polygon fill + // a user needs to close drawing/editing twice + // first close stops internal drawing/editing with svg.js + // the second one stops drawing/editing mask itself + + return !data.enabled && currentData.enabled && + (('shapeType' in currentData && currentData.shapeType === 'mask') || + ('state' in currentData && currentData.state.shapeType === 'mask')) && + currentData.brushTool?.type?.startsWith('polygon-') && + hasShapeIsBeingDrawn(); +} + +export class CanvasModelImpl extends MasterImpl implements CanvasModel { + private data: { + activeElement: ActiveElement; + highlightedElements: HighlightedElements; + angle: number; + canvasSize: Size; + configuration: Configuration; + imageBitmap: boolean; + image: Image | null; + imageID: number | null; + imageOffset: number; + imageSize: Size; + imageIsDeleted: boolean; + focusData: FocusData; + gridSize: Size; + objects: any[]; + renderData: RenderData; + issueRegions: Record; + scale: number; + top: number; + left: number; + fittedScale: number; + drawData: DrawData; + editData: MasksEditData | PolyEditData; + interactionData: InteractionData; + mergeData: MergeData; + groupData: GroupData; + joinData: JoinData; + sliceData: SliceData; + splitData: SplitData; + selected: any; + mode: Mode; + exception: Error | null; + }; + + public constructor() { + super(); + + this.data = { + activeElement: { + clientID: null, + attributeID: null, + }, + highlightedElements: { + elementsIDs: [], + severity: null, + }, + angle: 0, + canvasSize: { + height: 0, + width: 0, + }, + configuration: { + smoothImage: true, + autoborders: false, + snapToPoint: false, + snapRadius: 10, + adaptiveZoom: true, + displayAllText: false, + showProjections: false, + showConflicts: false, + forceDisableEditing: false, + intelligentPolygonCrop: false, + forceFrameUpdate: false, + CSSImageFilter: '', + colorBy: ColorBy.LABEL, + selectedShapeOpacity: 0.5, + shapeOpacity: 0.2, + outlinedBorders: false, + resetZoom: true, + textFontSize: consts.DEFAULT_SHAPE_TEXT_SIZE, + controlPointsSize: consts.BASE_POINT_SIZE, + textPosition: consts.DEFAULT_SHAPE_TEXT_POSITION, + textContent: consts.DEFAULT_SHAPE_TEXT_CONTENT, + undefinedAttrValue: consts.DEFAULT_UNDEFINED_ATTR_VALUE, + hideEditedObject: false, + focusedObjectPadding: 50, + }, + imageBitmap: false, + image: null, + imageID: null, + imageOffset: 0, + imageSize: { + height: 0, + width: 0, + }, + imageIsDeleted: false, + focusData: { + clientID: 0, + }, + gridSize: { + height: 100, + width: 100, + }, + objects: [], + renderData: { + visibleSkeletonElements: {}, + }, + issueRegions: {}, + scale: 1, + top: 0, + left: 0, + fittedScale: 0, + selected: null, + mode: Mode.IDLE, + exception: null, + ...defaultData, + }; + } + + public zoom(x: number, y: number, deltaY: number): void { + const basicZoomCoef = 6 / 5; // historical value + // less value of adjust coef, means zoomin/zoomout smoother + // we need a trade-off between speed and smoothness, value 1 / 10 is good enough + const adjustCoef = 1 / 10; + const oldScale: number = this.data.scale; + let scaleFactor = basicZoomCoef ** (-deltaY * adjustCoef); + + if (!this.data.configuration.adaptiveZoom) { + // old alogithm, just multiplies to 6/5 or 5/6 + scaleFactor = basicZoomCoef ** (Math.sign(-deltaY)); + } + const newScale: number = oldScale * scaleFactor; + this.data.scale = Math.min(Math.max(newScale, FrameZoom.MIN), FrameZoom.MAX); + + const { angle } = this.data; + + const multiplier = Math.sin((angle * Math.PI) / 180) + Math.cos((angle * Math.PI) / 180); + if ((angle / 90) % 2) { + // 90, 270, .. + const topMultiplier = (x - this.data.imageSize.width / 2) * (oldScale / this.data.scale - 1); + const leftMultiplier = (y - this.data.imageSize.height / 2) * (oldScale / this.data.scale - 1); + this.data.top += multiplier * topMultiplier * this.data.scale; + this.data.left -= multiplier * leftMultiplier * this.data.scale; + } else { + const leftMultiplier = (x - this.data.imageSize.width / 2) * (oldScale / this.data.scale - 1); + const topMultiplier = (y - this.data.imageSize.height / 2) * (oldScale / this.data.scale - 1); + this.data.left += multiplier * leftMultiplier * this.data.scale; + this.data.top += multiplier * topMultiplier * this.data.scale; + } + + this.notify(UpdateReasons.IMAGE_ZOOMED); + } + + public move(topOffset: number, leftOffset: number): void { + this.data.top += topOffset; + this.data.left += leftOffset; + this.notify(UpdateReasons.IMAGE_MOVED); + } + + public fitCanvas(width: number, height: number): void { + this.data.canvasSize.height = height; + this.data.canvasSize.width = width; + + this.data.imageOffset = Math.floor( + Math.max(this.data.canvasSize.height / FrameZoom.MIN, this.data.canvasSize.width / FrameZoom.MIN), + ); + + this.notify(UpdateReasons.FITTED_CANVAS); + this.notify(UpdateReasons.OBJECTS_UPDATED); + this.notify(UpdateReasons.ISSUE_REGIONS_UPDATED); + } + + public bitmap(enabled: boolean): void { + this.data.imageBitmap = enabled; + this.notify(UpdateReasons.BITMAP); + } + + public selectRegion(enable: boolean): void { + if (enable && this.data.mode !== Mode.IDLE) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (!enable && this.data.mode !== Mode.SELECT_REGION) { + throw Error(`Canvas is not in the region selecting mode. Action: ${this.data.mode}`); + } + + this.data.mode = enable ? Mode.SELECT_REGION : Mode.IDLE; + this.notify(UpdateReasons.SELECT_REGION); + } + + public dragCanvas(enable: boolean): void { + if (enable && this.data.mode !== Mode.IDLE) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (!enable && this.data.mode !== Mode.DRAG_CANVAS) { + throw Error(`Canvas is not in the drag mode. Action: ${this.data.mode}`); + } + + this.data.mode = enable ? Mode.DRAG_CANVAS : Mode.IDLE; + this.notify(UpdateReasons.DRAG_CANVAS); + } + + public zoomCanvas(enable: boolean): void { + if (enable && this.data.mode !== Mode.IDLE) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (!enable && this.data.mode !== Mode.ZOOM_CANVAS) { + throw Error(`Canvas is not in the zoom mode. Action: ${this.data.mode}`); + } + + this.data.mode = enable ? Mode.ZOOM_CANVAS : Mode.IDLE; + this.notify(UpdateReasons.ZOOM_CANVAS); + } + + public setup(frameData: any, objectStates: any[], renderData: RenderData = { + visibleSkeletonElements: {}, + }): void { + if (this.data.imageID !== frameData.number) { + if ([Mode.EDIT, Mode.DRAG, Mode.RESIZE].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + } + if (frameData.number === this.data.imageID && + frameData.deleted === this.data.imageIsDeleted && + !this.data.configuration.forceFrameUpdate + ) { + this.data.objects = objectStates; + this.data.renderData = renderData; + if (this.data.image) { + // display objects only if there is a drawn image + // if there is not, UpdateReasons.OBJECTS_UPDATED will be triggered after image is set + // it covers cases when annotations are changed while image is being received from the server + // e.g. with UI buttons (lock, unlock), shortcuts, delete/restore frames, + // and anytime when a list of objects updated in cvat-ui + this.notify(UpdateReasons.OBJECTS_UPDATED); + } + return; + } + + this.data.imageID = frameData.number; + this.data.imageIsDeleted = frameData.deleted; + if (this.data.imageIsDeleted) { + this.data.angle = 0; + } + + const { objects: prevObjects, renderData: prevRenderData } = this.data; + frameData + .data((): void => { + this.data.image = null; + this.notify(UpdateReasons.IMAGE_CHANGED); + }) + .then((data: Image): void => { + if (frameData.number !== this.data.imageID) { + // check that request is still relevant after async image data fetching + return; + } + + const relativeScaling = this.data.scale / this.data.fittedScale; + const prevImageLeft = this.data.left; + const prevImageTop = this.data.top; + const prevImageWidth = this.data.imageSize.width; + const prevImageHeight = this.data.imageSize.height; + + this.data.imageSize = { + height: frameData.height as number, + width: frameData.width as number, + }; + + this.data.image = data; + this.resetScale(); + + // restore correct image position after switching to a new frame + // if corresponding option is disabled + // prevImageHeight and prevImageWidth are initialized by 0 by default + if (prevImageHeight !== 0 && prevImageWidth !== 0 && !this.data.configuration.resetZoom) { + const leftOffset = Math.round((this.data.imageSize.width - prevImageWidth) / 2); + const topOffset = Math.round((this.data.imageSize.height - prevImageHeight) / 2); + this.data.left = prevImageLeft - leftOffset; + this.data.top = prevImageTop - topOffset; + this.data.scale *= relativeScaling; + } + + this.notify(UpdateReasons.IMAGE_CHANGED); + + if ( + prevObjects === this.data.objects && + prevRenderData === this.data.renderData + ) { + // check the request is relevant, other setup() may have been called while promise resolving + this.data.objects = objectStates; + this.data.renderData = renderData; + } + + this.notify(UpdateReasons.OBJECTS_UPDATED); + }) + .catch((exception: unknown): void => { + if (typeof exception !== 'number') { + // don't notify when the frame is no longer needed + if (exception instanceof Error) { + this.data.exception = exception; + } else { + this.data.exception = new Error('Unknown error occurred when fetching image data'); + } + this.notify(UpdateReasons.DATA_FAILED); + } + }); + } + + public setupIssueRegions(issueRegions: Record): void { + this.data.issueRegions = issueRegions; + this.notify(UpdateReasons.ISSUE_REGIONS_UPDATED); + } + + public activate(clientID: number | null, attributeID: number | null): void { + if (this.data.activeElement.clientID === clientID && this.data.activeElement.attributeID === attributeID) { + return; + } + + if (this.data.mode !== Mode.IDLE && clientID !== null) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (typeof clientID === 'number') { + const [state] = this.objects.filter((_state: any): boolean => _state.clientID === clientID); + if (!state || state.objectType === 'tag') { + return; + } + } + + this.data.activeElement = { + clientID, + attributeID, + }; + + this.notify(UpdateReasons.SHAPE_ACTIVATED); + } + + public highlight(clientIDs: number[], severity: HighlightSeverity | null): void { + const elementsIDs = clientIDs.filter((id: number): boolean => ( + this.objects.find((_state: any): boolean => _state.clientID === id) + )); + + this.data.highlightedElements = { + elementsIDs, + severity, + }; + + this.notify(UpdateReasons.SHAPE_HIGHLIGHTED); + } + + public rotate(rotationAngle: number): void { + if (this.data.angle !== rotationAngle && !this.data.imageIsDeleted) { + this.data.angle = (360 + Math.floor(rotationAngle / 90) * 90) % 360; + this.notify(UpdateReasons.IMAGE_ROTATED); + } + } + + public focus(clientID: number): void { + this.data.focusData = { clientID }; + this.notify(UpdateReasons.SHAPE_FOCUSED); + } + + private resetScale(): boolean { + const { angle } = this.data; + + let updatedScale = this.data.scale; + if ((angle / 90) % 2) { + // 90, 270, .. + updatedScale = Math.min( + this.data.canvasSize.width / this.data.imageSize.height, + this.data.canvasSize.height / this.data.imageSize.width, + ); + } else { + updatedScale = Math.min( + this.data.canvasSize.width / this.data.imageSize.width, + this.data.canvasSize.height / this.data.imageSize.height, + ); + } + + updatedScale = Math.min(Math.max(updatedScale, FrameZoom.MIN), FrameZoom.MAX); + const updatedTop = this.data.canvasSize.height / 2 - this.data.imageSize.height / 2; + const updatedLeft = this.data.canvasSize.width / 2 - this.data.imageSize.width / 2; + + if (updatedScale !== this.data.scale || updatedTop !== this.data.top || updatedLeft !== this.data.left) { + this.data.scale = updatedScale; + this.data.top = updatedTop; + this.data.left = updatedLeft; + + // scale is changed during zooming or translating + // so, remember fitted scale to compute fit-relative scaling + this.data.fittedScale = this.data.scale; + return true; + } + + return false; + } + + public fit(): void { + if (this.resetScale()) { + this.notify(UpdateReasons.IMAGE_FITTED); + } + } + + public grid(stepX: number, stepY: number): void { + this.data.gridSize = { + height: stepY, + width: stepX, + }; + + this.notify(UpdateReasons.GRID_UPDATED); + } + + public draw(drawData: DrawData): void { + const supportedShapes = [ + 'rectangle', 'polygon', 'polyline', 'points', 'ellipse', 'cuboid', 'skeleton', 'mask', + ]; + if (![Mode.IDLE, Mode.DRAW].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (drawData.enabled) { + if (drawData.shapeType === 'skeleton' && !drawData.skeletonSVG) { + throw new Error('Skeleton template must be specified when drawing a skeleton'); + } + + if (!drawData.shapeType && !drawData.initialState) { + throw new Error('A shape type is not specified'); + } + + if (drawData.shapeType && !supportedShapes.includes(drawData.shapeType)) { + throw new Error(`Drawing method for type "${drawData.shapeType}" is not implemented`); + } + + if (typeof drawData.numberOfPoints !== 'undefined') { + if (drawData.shapeType === 'polygon' && drawData.numberOfPoints < 3) { + throw new Error('A polygon consists of at least 3 points'); + } else if (drawData.shapeType === 'polyline' && drawData.numberOfPoints < 2) { + throw new Error('A polyline consists of at least 2 points'); + } + } + } + + if (typeof drawData.redraw === 'number') { + const clientID = drawData.redraw; + const [state] = this.data.objects.filter((_state: any): boolean => _state.clientID === clientID); + + if (state) { + this.data.drawData = { ...drawData }; + this.data.drawData.shapeType = state.shapeType; + } else { + return; + } + } else { + if (disableInternalSVGDrawing(drawData, this.data.drawData)) { + this.notify(UpdateReasons.DRAW); + return; + } + + this.data.drawData = { ...drawData }; + if (this.data.drawData.initialState) { + this.data.drawData.shapeType = this.data.drawData.initialState.shapeType; + } + } + + // install default values for drawing method + if (drawData.enabled) { + if (drawData.shapeType === 'rectangle') { + this.data.drawData.rectDrawingMethod = drawData.rectDrawingMethod || RectDrawingMethod.CLASSIC; + } + if (drawData.shapeType === 'cuboid') { + this.data.drawData.cuboidDrawingMethod = drawData.cuboidDrawingMethod || CuboidDrawingMethod.CLASSIC; + } + } + + this.notify(UpdateReasons.DRAW); + } + + public edit(editData: MasksEditData | PolyEditData): void { + if (![Mode.IDLE, Mode.EDIT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (editData.enabled && !editData.state) { + throw Error('State must be specified when call edit() editing process'); + } + + if (this.data.editData.enabled && editData.enabled && + editData.state.clientID !== this.data.editData.state.clientID + ) { + throw Error('State cannot be updated during editing, need to finish current editing first'); + } + + if (editData.enabled) { + this.data.editData = { ...editData }; + } else if (disableInternalSVGDrawing(editData, this.data.editData)) { + this.notify(UpdateReasons.EDIT); + return; + } else { + this.data.editData = { enabled: false }; + } + + this.notify(UpdateReasons.EDIT); + } + + public interact(interactionData: InteractionData): void { + if (![Mode.IDLE, Mode.INTERACT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + this.data.interactionData = interactionData; + this.notify(UpdateReasons.INTERACT); + } + + public split(splitData: SplitData): void { + if (![Mode.IDLE, Mode.SPLIT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if ((this.data.splitData.enabled && splitData.enabled) || ( + !this.data.splitData.enabled && !splitData.enabled + )) { + return; + } + + this.data.splitData = { ...splitData }; + this.notify(UpdateReasons.SPLIT); + } + + public group(groupData: GroupData): void { + if (![Mode.IDLE, Mode.GROUP].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if ((this.data.groupData.enabled && groupData.enabled) || ( + !this.data.groupData.enabled && !groupData.enabled + )) { + return; + } + + this.data.groupData = { ...groupData }; + this.notify(UpdateReasons.GROUP); + } + + public join(joinData: JoinData): void { + if (![Mode.IDLE, Mode.JOIN].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if ((this.data.joinData.enabled && joinData.enabled) || ( + !this.data.joinData.enabled && !joinData.enabled + )) { + return; + } + + this.data.joinData = { ...joinData }; + this.notify(UpdateReasons.JOIN); + } + + public slice(sliceData: SliceData): void { + if (![Mode.IDLE, Mode.SLICE].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if ((this.data.sliceData.enabled && sliceData.enabled) || ( + !this.data.sliceData.enabled && !sliceData.enabled + )) { + return; + } + + if (sliceData.enabled && !sliceData.getContour) { + throw Error('Contours computing method was not provided'); + } + + this.data.sliceData = { ...sliceData }; + this.notify(UpdateReasons.SLICE); + } + + public merge(mergeData: MergeData): void { + if (![Mode.IDLE, Mode.MERGE].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if ((this.data.mergeData.enabled && mergeData.enabled) || ( + !this.data.mergeData.enabled && !mergeData.enabled + )) { + return; + } + + this.data.mergeData = { ...mergeData }; + this.notify(UpdateReasons.MERGE); + } + + public select(objectState: any): void { + this.data.selected = objectState; + this.notify(UpdateReasons.SELECT); + this.data.selected = null; + } + + public configure(configuration: Configuration): void { + if (typeof configuration.displayAllText === 'boolean') { + this.data.configuration.displayAllText = configuration.displayAllText; + } + + if (typeof configuration.textFontSize === 'number' && configuration.textFontSize >= consts.MINIMUM_TEXT_FONT_SIZE) { + this.data.configuration.textFontSize = configuration.textFontSize; + } + + if (typeof configuration.controlPointsSize === 'number') { + this.data.configuration.controlPointsSize = configuration.controlPointsSize; + } + + if (['auto', 'center'].includes(configuration.textPosition)) { + this.data.configuration.textPosition = configuration.textPosition; + } + + if (typeof configuration.textContent === 'string') { + const splitted = configuration.textContent.split(',').filter((entry: string) => !!entry); + if (splitted.every((entry: string) => ( + ['id', 'label', 'attributes', 'source', 'descriptions', 'dimensions', 'layer', 'zOrder'].includes(entry) + ))) { + this.data.configuration.textContent = configuration.textContent; + } + } + + if (typeof configuration.showProjections === 'boolean') { + this.data.configuration.showProjections = configuration.showProjections; + } + if (typeof configuration.autoborders === 'boolean') { + this.data.configuration.autoborders = configuration.autoborders; + } + if (typeof configuration.snapToPoint === 'boolean') { + this.data.configuration.snapToPoint = configuration.snapToPoint; + } + if (typeof configuration.snapRadius === 'number' && configuration.snapRadius > 0) { + this.data.configuration.snapRadius = configuration.snapRadius; + } + if (typeof configuration.adaptiveZoom === 'boolean') { + this.data.configuration.adaptiveZoom = configuration.adaptiveZoom; + } + if (typeof configuration.smoothImage === 'boolean') { + this.data.configuration.smoothImage = configuration.smoothImage; + } + if (typeof configuration.undefinedAttrValue === 'string') { + this.data.configuration.undefinedAttrValue = configuration.undefinedAttrValue; + } + if (typeof configuration.forceDisableEditing === 'boolean') { + this.data.configuration.forceDisableEditing = configuration.forceDisableEditing; + } + if (typeof configuration.intelligentPolygonCrop === 'boolean') { + this.data.configuration.intelligentPolygonCrop = configuration.intelligentPolygonCrop; + } + if (typeof configuration.forceFrameUpdate === 'boolean') { + this.data.configuration.forceFrameUpdate = configuration.forceFrameUpdate; + } + if (typeof configuration.resetZoom === 'boolean') { + this.data.configuration.resetZoom = configuration.resetZoom; + } + if (typeof configuration.selectedShapeOpacity === 'number') { + this.data.configuration.selectedShapeOpacity = configuration.selectedShapeOpacity; + } + if (typeof configuration.shapeOpacity === 'number') { + this.data.configuration.shapeOpacity = configuration.shapeOpacity; + } + if (['string', 'boolean'].includes(typeof configuration.outlinedBorders)) { + this.data.configuration.outlinedBorders = configuration.outlinedBorders; + } + if (Object.values(ColorBy).includes(configuration.colorBy)) { + this.data.configuration.colorBy = configuration.colorBy; + } + + if (typeof configuration.showConflicts === 'boolean') { + this.data.configuration.showConflicts = configuration.showConflicts; + } + + if (typeof configuration.CSSImageFilter === 'string') { + this.data.configuration.CSSImageFilter = configuration.CSSImageFilter; + } + + if (typeof configuration.hideEditedObject === 'boolean') { + this.data.configuration.hideEditedObject = configuration.hideEditedObject; + } + + if (typeof configuration.focusedObjectPadding === 'number') { + this.data.configuration.focusedObjectPadding = Math.max( + configuration.focusedObjectPadding, 0, + ); + } + + this.notify(UpdateReasons.CONFIG_UPDATED); + } + + public isAbleToChangeFrame(): boolean { + const isUnable = [Mode.SLICE, Mode.DRAG, Mode.EDIT, Mode.RESIZE, Mode.INTERACT].includes(this.data.mode) || + (this.data.mode === Mode.DRAW && typeof this.data.drawData.redraw === 'number'); + + return !isUnable; + } + + public cancel(): void { + this.data = { + ...this.data, + ...defaultData, + }; + this.notify(UpdateReasons.CANCEL); + } + + public destroy(): void { + this.notify(UpdateReasons.DESTROY); + } + + public get configuration(): Configuration { + return { ...this.data.configuration }; + } + + public get geometry(): Geometry { + return { + angle: this.data.angle, + canvas: { ...this.data.canvasSize }, + image: { ...this.data.imageSize }, + grid: { ...this.data.gridSize }, + left: this.data.left, + offset: this.data.imageOffset, + scale: this.data.scale, + top: this.data.top, + }; + } + + public set geometry(geometry: Geometry) { + this.data.angle = geometry.angle; + this.data.canvasSize = { ...geometry.canvas }; + this.data.imageSize = { ...geometry.image }; + this.data.gridSize = { ...geometry.grid }; + this.data.left = geometry.left; + this.data.top = geometry.top; + this.data.imageOffset = geometry.offset; + this.data.scale = geometry.scale; + + this.data.imageOffset = Math.floor( + Math.max(this.data.canvasSize.height / FrameZoom.MIN, this.data.canvasSize.width / FrameZoom.MIN), + ); + } + + public get imageBitmap(): boolean { + return this.data.imageBitmap; + } + + public get imageIsDeleted(): boolean { + return this.data.imageIsDeleted; + } + + public get image(): Image | null { + return this.data.image; + } + + public get issueRegions(): Record { + return { ...this.data.issueRegions }; + } + + public get objects(): any[] { + return this.data.objects; + } + + public get renderData(): RenderData { + return { + visibleSkeletonElements: { ...this.data.renderData.visibleSkeletonElements }, + }; + } + + public get gridSize(): Size { + return { ...this.data.gridSize }; + } + + public get focusData(): FocusData { + return { ...this.data.focusData }; + } + + public get activeElement(): ActiveElement { + return { ...this.data.activeElement }; + } + + public get highlightedElements(): HighlightedElements { + return { ...this.data.highlightedElements }; + } + + public get drawData(): DrawData { + return { ...this.data.drawData }; + } + + public get editData(): MasksEditData | PolyEditData { + return { ...this.data.editData }; + } + + public get interactionData(): InteractionData { + return { ...this.data.interactionData }; + } + + public get mergeData(): MergeData { + return { ...this.data.mergeData }; + } + + public get splitData(): SplitData { + return { ...this.data.splitData }; + } + + public get joinData(): JoinData { + return { ...this.data.joinData }; + } + + public get sliceData(): SliceData { + return { ...this.data.sliceData }; + } + + public get groupData(): GroupData { + return { ...this.data.groupData }; + } + + public get selected(): any { + return this.data.selected; + } + + public set mode(value: Mode) { + this.data.mode = value; + } + + public get mode(): Mode { + return this.data.mode; + } + + public get exception(): Error | null { + return this.data.exception; + } +} diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts new file mode 100644 index 0000000..31fd8a5 --- /dev/null +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -0,0 +1,3896 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import polylabel from 'polylabel'; +import { fabric } from 'fabric'; +import * as SVG from 'svg.js'; +import * as martinez from 'martinez-polygon-clipping'; + +import 'svg.draggable.js'; +import 'svg.resize.js'; +import 'svg.select.js'; + +import { CanvasController } from './canvasController'; +import { Listener, Master } from './master'; +import { DrawHandler, DrawHandlerImpl } from './drawHandler'; +import { MasksHandler, MasksHandlerImpl } from './masksHandler'; +import { EditHandler, EditHandlerImpl } from './editHandler'; +import { MergeHandler, MergeHandlerImpl } from './mergeHandler'; +import { SplitHandler, SplitHandlerImpl } from './splitHandler'; +import { ObjectSelector, ObjectSelectorImpl } from './objectSelector'; +import { GroupHandler, GroupHandlerImpl } from './groupHandler'; +import { SliceHandler, SliceHandlerImpl } from './sliceHandler'; +import { RegionSelector, RegionSelectorImpl } from './regionSelector'; +import { ZoomHandler, ZoomHandlerImpl } from './zoomHandler'; +import { InteractionHandler, InteractionHandlerImpl } from './interactionHandler'; +import { AutoborderHandler, AutoborderHandlerImpl } from './autoborderHandler'; +import consts from './consts'; +import { + translateToSVG, translateFromSVG, translateToCanvas, translateFromCanvas, + pointsToNumberArray, parsePoints, displayShapeSize, scalarProduct, + vectorLength, ShapeSizeElement, DrawnState, rotate2DPoints, + readPointsFromShape, setupSkeletonEdges, makeSVGFromTemplate, + imageDataToDataURL, RLEToImageData, stringifyPoints, imageDataToRLE, + composeShapeDimensions, getRoundedRotation, + clamp, validateUnionResult, processPolygonUnionResult, + applySnapToShapePoint, isPolygonSelfIntersecting, +} from './shared'; +import { + CanvasModel, Geometry, UpdateReasons, FrameZoom, ActiveElement, + DrawData, MergeData, SplitData, Mode, Size, Configuration, + InteractionResult, InteractionData, ColorBy, HighlightedElements, + HighlightSeverity, GroupData, JoinData, CanvasHint, +} from './canvasModel'; + +export interface CanvasView { + html(): HTMLDivElement; + setupConflictRegions(clientID: number): number[]; + translateFromSVG(points: number[]): number[]; +} + +export class CanvasViewImpl implements CanvasView, Listener { + private text: SVGSVGElement; + private adoptedText: SVG.Container; + private background: HTMLCanvasElement; + private masksContent: HTMLCanvasElement; + private bitmap: HTMLCanvasElement; + private bitmapUpdateReqId: number; + private grid: SVGSVGElement; + private content: SVGSVGElement; + private attachmentBoard: HTMLDivElement; + private adoptedContent: SVG.Container; + private canvas: HTMLDivElement; + private gridPath: SVGPathElement; + private gridPattern: SVGPatternElement; + private controller: CanvasController; + private svgShapes: Record; + private svgTexts: Record; + private isImageLoading: boolean; + private issueRegionPattern_1: SVG.Pattern; + private issueRegionPattern_2: SVG.Pattern; + private drawnStates: Record; + private drawnIssueRegions: Record; + private geometry: Geometry; + private drawHandler: DrawHandler; + private masksHandler: MasksHandler; + private editHandler: EditHandler; + private mergeHandler: MergeHandler; + private splitHandler: SplitHandler; + private groupHandler: GroupHandler; + private sliceHandler: SliceHandler; + private regionSelector: RegionSelector; + private objectSelector: ObjectSelector; + private zoomHandler: ZoomHandler; + private autoborderHandler: AutoborderHandler; + private interactionHandler: InteractionHandler; + private activeElement: ActiveElement; + private highlightedElements: HighlightedElements; + private configuration: Configuration; + private snapToAngleResize: number; + private draggableShape: SVG.Shape | null; + private resizableShape: SVG.Shape | null; + private ctrlPressed: boolean; + private innerObjectsFlags: { + drawHidden: Record; + editHidden: Record; + sliceHidden: Record; + }; + + private set mode(value: Mode) { + this.controller.mode = value; + } + + private get mode(): Mode { + return this.controller.mode; + } + + private onMessage = (messages: CanvasHint[] | null, topic: string): void => { + this.canvas.dispatchEvent( + new CustomEvent('canvas.message', { + bubbles: false, + cancelable: true, + detail: { + topic, + messages, + }, + }), + ); + }; + + private onError = (exception: unknown, domain?: string): void => { + this.canvas.dispatchEvent( + new CustomEvent('canvas.error', { + bubbles: false, + cancelable: true, + detail: { + domain, + exception: exception instanceof Error ? + exception : new Error(`Unknown exception: "${exception}"`), + }, + }), + ); + }; + + private onWarning = (message: string, domain?: string): void => { + this.canvas.dispatchEvent( + new CustomEvent('canvas.warning', { + bubbles: false, + cancelable: true, + detail: { + domain, + message, + }, + }), + ); + }; + + private stateIsLocked(state: any): boolean { + const { configuration } = this.controller; + return state.lock || configuration.forceDisableEditing; + } + + private translateToCanvas(points: number[]): number[] { + const { offset } = this.controller.geometry; + return translateToCanvas(offset, points); + } + + private translateFromCanvas(points: number[]): number[] { + const { offset } = this.controller.geometry; + return translateFromCanvas(offset, points); + } + + private translatePointsFromRotatedShape( + shape: SVG.Shape, points: number[], cx: number = null, cy: number = null, + ): number[] { + const { rotation } = shape.transform(); + // currently shape is rotated and SHIFTED somehow additionally (css transform property) + // let's remove rotation to get correct transformation matrix (element -> screen) + // correct means that we do not consider points to be rotated + // because rotation property is stored separately and already saved + if (cx !== null && cy !== null) { + shape.rotate(0, cx, cy); + } else { + shape.rotate(0); + } + + const result = []; + + try { + // get each point and apply a couple of matrix transformation to it + const point = this.content.createSVGPoint(); + // matrix to convert from ELEMENT coordinate system to CLIENT coordinate system + const ctm = ( + (shape.node as any) as SVGRectElement | SVGPolygonElement | SVGPolylineElement | SVGGElement + ).getScreenCTM(); + // matrix to convert from CLIENT coordinate system to CANVAS coordinate system + const ctm1 = this.content.getScreenCTM().inverse(); + // NOTE: I tried to use element.getCTM(), but this way does not work on firefox + + for (let i = 0; i < points.length; i += 2) { + point.x = points[i]; + point.y = points[i + 1]; + let transformedPoint = point.matrixTransform(ctm); + transformedPoint = transformedPoint.matrixTransform(ctm1); + + result.push(transformedPoint.x, transformedPoint.y); + } + } finally { + if (cx !== null && cy !== null) { + shape.rotate(rotation, cx, cy); + } else { + shape.rotate(rotation); + } + } + + return result; + } + + private isInnerHidden(clientID: number): boolean { + return this.innerObjectsFlags.drawHidden[clientID] || + this.innerObjectsFlags.editHidden[clientID] || + this.innerObjectsFlags.sliceHidden[clientID] || + false; + } + + private setupInnerFlags(clientID: number, path: keyof CanvasViewImpl['innerObjectsFlags'], value: boolean): void { + this.innerObjectsFlags[path][clientID] = value; + const shape = this.svgShapes[clientID]; + const text = this.svgTexts[clientID]; + const state = this.drawnStates[clientID]; + + if (value && clientID === this.controller.activeElement.clientID) { + this.deactivate(); + } + + if (value) { + if (shape) { + (state.shapeType === 'points' ? shape.remember('_selectHandler').nested : shape).addClass( + 'cvat_canvas_hidden', + ); + } + + if (text) { + text.addClass('cvat_canvas_hidden'); + } + } else { + delete this.innerObjectsFlags[path][clientID]; + + if (state) { + if (!state.outside && !state.hidden) { + if (shape) { + (state.shapeType === 'points' ? shape.remember('_selectHandler').nested : shape).removeClass( + 'cvat_canvas_hidden', + ); + } + + if (text) { + text.removeClass('cvat_canvas_hidden'); + this.updateTextPosition(text); + } + } + } + } + } + + private dispatchCanceledEvent(): void { + this.mode = Mode.IDLE; + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.canvas.dispatchEvent(event); + } + + private resetViewPosition(clientID: number): void { + const drawnState = this.drawnStates[clientID]; + const drawnShape = this.svgShapes[clientID]; + + if (drawnState && drawnShape) { + const { shapeType, points } = drawnState; + const translatedPoints: number[] = this.translateToCanvas(points); + const stringified = stringifyPoints(translatedPoints); + if (shapeType === 'cuboid') { + drawnShape.attr('points', stringified); + } else if (['polygon', 'polyline', 'points'].includes(shapeType)) { + (drawnShape as SVG.PolyLine | SVG.Polygon).plot(stringified); + if (shapeType === 'points') { + this.selectize(false, drawnShape); + this.setupPoints(drawnShape as SVG.PolyLine, drawnState); + } + } else if (shapeType === 'rectangle') { + const [xtl, ytl, xbr, ybr] = translatedPoints; + drawnShape.rotate(0); + drawnShape.size(xbr - xtl, ybr - ytl).move(xtl, ytl); + drawnShape.rotate(drawnState.rotation); + } else if (shapeType === 'ellipse') { + const [cx, cy, rightX, topY] = translatedPoints; + const [rx, ry] = [rightX - cx, cy - topY]; + drawnShape.rotate(0); + drawnShape.size(rx * 2, ry * 2).center(cx, cy); + drawnShape.rotate(drawnState.rotation); + } else if (shapeType === 'skeleton') { + drawnShape.rotate(0); + for (const child of (drawnShape as SVG.G).children()) { + if (child.type === 'circle') { + const childClientID = child.attr('data-client-id'); + const element = drawnState.elements.find((el: any) => el.clientID === childClientID); + const [x, y] = this.translateToCanvas(element.points); + child.center(x, y); + } + } + drawnShape.rotate(drawnState.rotation); + } else if (shapeType === 'mask') { + const [left, top] = points.slice(-4); + drawnShape.move(this.geometry.offset + left, this.geometry.offset + top); + } else { + throw new Error('Not implemented'); + } + } + } + + private onInteraction = ( + shapes: InteractionResult[] | null, + finished = false, + ): void => { + // whenever prompts are updated, interactor sends corresponding event with prompts + // when finished, it also sends finish flag equals to "true" + // when cancelled or closed, shapes equals to "null" + if (Array.isArray(shapes) || finished) { + const event: CustomEvent = new CustomEvent('canvas.interacted', { + bubbles: false, + cancelable: true, + detail: { + finished, + shapes, + }, + }); + + this.canvas.dispatchEvent(event); + } + + if (shapes === null) { + this.dispatchCanceledEvent(); + } + }; + + private onDrawDone = ( + data: any | null, + duration: number, + continueDraw?: boolean, + prevDrawData?: DrawData, + ): void => { + const hiddenBecauseOfDraw = Object.keys(this.innerObjectsFlags.drawHidden) + .map((_clientID): number => +_clientID); + if (hiddenBecauseOfDraw.length) { + for (const hidden of hiddenBecauseOfDraw) { + this.setupInnerFlags(hidden, 'drawHidden', false); + } + } + + if (data) { + const { clientID, elements } = data as any; + const points = data.points || elements.map((el: any) => el.points).flat(); + if (typeof clientID === 'number') { + const [state] = this.controller.objects + .filter((_state: any): boolean => _state.clientID === clientID); + this.onEditDone(state, points); + this.dispatchCanceledEvent(); + return; + } + + const event: CustomEvent = new CustomEvent('canvas.drawn', { + bubbles: false, + cancelable: true, + detail: { + state: data, + continue: continueDraw, + simplifyPoly: data?.simplifyPoly || false, + duration, + }, + }); + + this.canvas.dispatchEvent(event); + } else if (!continueDraw) { + this.dispatchCanceledEvent(); + } + + if (continueDraw) { + this.canvas.dispatchEvent( + new CustomEvent('canvas.drawstart', { + bubbles: false, + cancelable: true, + detail: { + drawData: prevDrawData, + }, + }), + ); + } else { + // when draw stops from inside canvas (for example if use predefined number of points) + this.mode = Mode.IDLE; + this.canvas.style.cursor = ''; + } + }; + + private onEditStart = (state?: any): void => { + this.canvas.style.cursor = 'crosshair'; + this.deactivate(); + this.canvas.dispatchEvent( + new CustomEvent('canvas.editstart', { + bubbles: false, + cancelable: true, + detail: { + state, + }, + }), + ); + + if (state && state.shapeType === 'mask') { + this.setupInnerFlags(state.clientID, 'editHidden', true); + } + + this.mode = Mode.EDIT; + }; + + private onEditDone = (state: any, points: number[], rotation?: number): void => { + this.canvas.style.cursor = ''; + this.mode = Mode.IDLE; + if (state && points) { + // we need to store "updated" and set "points" to an empty array + // as this information is used to define "updated" objects in diff logic during canvas objects setup + // if because of any reason updating was actually rejected somewhere, we must reset view inside this logic + + // there is one more deeper issue: + // somewhere canvas updates drawn views and then sends request, + // updating internal CVAT state (e.g. drag, resize) + // somewhere, however, it just sends request to update internal CVAT state + // (e.g. remove point, edit polygon/polyline) + // if object view was not changed by canvas and points accepted as is without any changes + // the view will not be updated during objects setup if we just set points as is here + // that is why we need to set points to an empty array (something that can't normally come from CVAT) + // I do not think it can be easily fixed now, however in the future we should refactor code + if (Number.isInteger(state.parentID)) { + const { elements } = this.drawnStates[state.parentID]; + const drawnElement = elements.find((el) => el.clientID === state.clientID); + drawnElement.updated = 0; + drawnElement.points = []; + + this.drawnStates[state.parentID].updated = 0; + this.drawnStates[state.parentID].points = []; + } else { + this.drawnStates[state.clientID].updated = 0; + this.drawnStates[state.clientID].points = []; + } + + const event: CustomEvent = new CustomEvent('canvas.edited', { + bubbles: false, + cancelable: true, + detail: { + state, + points, + rotation: typeof rotation === 'number' ? rotation : state.rotation, + }, + }); + + this.canvas.dispatchEvent(event); + } else { + this.dispatchCanceledEvent(); + } + + for (const clientID of Object.keys(this.innerObjectsFlags.editHidden)) { + this.setupInnerFlags(+clientID, 'editHidden', false); + } + }; + + private onMergeDone = (objects: any[] | null, duration?: number): void => { + if (objects) { + const event: CustomEvent = new CustomEvent('canvas.merged', { + bubbles: false, + cancelable: true, + detail: { + duration, + states: objects, + }, + }); + + this.mode = Mode.IDLE; + this.canvas.dispatchEvent(event); + } else { + this.dispatchCanceledEvent(); + } + }; + + private onSplitDone = (object?: any, duration?: number): void => { + if (object && typeof duration !== 'undefined') { + const event: CustomEvent = new CustomEvent('canvas.splitted', { + bubbles: false, + cancelable: true, + detail: { + duration, + state: object, + frame: object.frame, + }, + }); + + this.canvas.style.cursor = ''; + this.mode = Mode.IDLE; + this.splitHandler.split({ enabled: false }); + this.canvas.dispatchEvent(event); + } else { + this.dispatchCanceledEvent(); + } + }; + + private onSelectDone = (objects?: any[], duration?: number): void => { + if (this.mode === Mode.JOIN) { + this.onMessage(null, 'join'); + } + + if (objects && typeof duration !== 'undefined' && objects.length > 1) { + if (this.mode === Mode.GROUP) { + this.mode = Mode.IDLE; + this.canvas.dispatchEvent(new CustomEvent('canvas.grouped', { + bubbles: false, + cancelable: true, + detail: { + duration, + states: objects, + }, + })); + } else if (this.mode === Mode.JOIN) { + this.mode = Mode.IDLE; + + const { shapeType } = objects[0]; + if (shapeType === 'polygon') { + this.joinPolygons(objects, duration); + } else if (shapeType === 'mask') { + this.joinMasks(objects, duration); + } + } + } else { + this.dispatchCanceledEvent(); + } + }; + + private joinPolygons(objects: any[], duration: number): void { + try { + const validObjects: any[] = []; + const selfIntersectingIndices: number[] = []; + + objects.forEach((state, idx) => { + if (isPolygonSelfIntersecting(state.points)) { + selfIntersectingIndices.push(idx); + } else { + validObjects.push(state); + } + }); + + if (selfIntersectingIndices.length > 0) { + const excludedIds = selfIntersectingIndices.map((idx) => objects[idx].clientID).join(', '); + this.onWarning( + `${selfIntersectingIndices.length} self-intersecting polygon${selfIntersectingIndices.length > 1 ? 's' : ''} excluded from merge ` + + `(IDs: ${excludedIds}).`, + 'Join operation', + ); + } + + if (validObjects.length < 2) { + throw new Error('Cannot join: not enough valid polygons (need at least 2 non-self-intersecting polygons)'); + } + + // Convert CVAT polygon format to martinez format + // CVAT format: [x1, y1, x2, y2, ...] (flat array) + // martinez format: [[[x1, y1], [x2, y2], ...]] (GeoJSON Polygon) + const polygons: martinez.Polygon[] = validObjects.map((state) => { + const { points } = state; + const coords: martinez.Position[] = []; + + for (let i = 0; i < points.length; i += 2) { + coords.push([points[i], points[i + 1]]); + } + + const firstPoint = coords[0]; + const lastPoint = coords[coords.length - 1]; + // Martinez library requires closed polygons (first point === last point) + // CVAT stores polygons in open format, so we need to close them + if (firstPoint[0] !== lastPoint[0] || firstPoint[1] !== lastPoint[1]) { + coords.push([firstPoint[0], firstPoint[1]]); + } + + return [coords]; + }); + + let result: martinez.Geometry = polygons[0]; + for (let i = 1; i < polygons.length; i += 1) { + result = martinez.union(result, polygons[i]); + if (!result) { + throw new Error('Union operation failed - polygons may be invalid'); + } + } + + if (!result || result.length === 0) { + throw new Error('Union operation resulted in empty polygon'); + } + + validateUnionResult(result); + + const processedResults = processPolygonUnionResult(result); + + // Show warning if merge resulted in multiple disjoint polygons + if (processedResults.length > 1) { + this.onWarning( + `Merge resulted in ${processedResults.length} separate polygons.`, + 'Join operation', + ); + } + + const pointsArray = processedResults.map(({ points }) => points); + + this.canvas.dispatchEvent(new CustomEvent('canvas.joined', { + bubbles: false, + cancelable: true, + detail: { + duration, + states: validObjects, + points: pointsArray, + shapeType: 'polygon', + }, + })); + } catch (error) { + this.onError(error); + this.dispatchCanceledEvent(); + } + } + + private joinMasks(objects: any[], duration: number): void { + let [left, top, right, bottom] = objects[0].points.slice(-4); + objects.forEach((state) => { + const [curLeft, curTop, curRight, curBottom] = state.points.slice(-4); + left = Math.min(left, curLeft); + top = Math.min(top, curTop); + right = Math.max(right, curRight); + bottom = Math.max(bottom, curBottom); + }); + + Promise.all(objects.map((state) => { + const [curLeft, , curRight] = state.points.slice(-4, -1); + const image = new ImageData( + RLEToImageData(255, 255, 255, state.points), curRight - curLeft + 1, + ); + return createImageBitmap(image); + })).then((results) => { + const canvas = new OffscreenCanvas(right - left + 1, bottom - top + 1); + const ctx = canvas.getContext('2d'); + + results.forEach((bitmap, idx) => { + const [curLeft, curTop] = objects[idx].points.slice(-4, -2); + ctx.drawImage(bitmap, curLeft - left, curTop - top); + bitmap.close(); + }); + + const imageData = ctx.getImageData(0, 0, right - left + 1, bottom - top + 1); + const rle = imageDataToRLE(imageData.data); + rle.push(left, top, right, bottom); + + this.canvas.dispatchEvent(new CustomEvent('canvas.joined', { + bubbles: false, + cancelable: true, + detail: { + duration, + states: objects, + points: [rle], + shapeType: 'mask', + }, + })); + }).catch((error) => { + this.onError(error); + this.dispatchCanceledEvent(); + }); + } + + private onSliceDone = (state?: any, results?: number[][], duration?: number): void => { + if (state && results && typeof duration !== 'undefined') { + this.mode = Mode.IDLE; + this.sliceHandler.slice({ enabled: false }); + this.canvas.dispatchEvent(new CustomEvent('canvas.sliced', { + bubbles: false, + cancelable: true, + detail: { + state, + results, + duration, + }, + })); + } else { + this.dispatchCanceledEvent(); + } + }; + + private onRegionSelected = (points?: number[]): void => { + if (points) { + this.canvas.dispatchEvent(new CustomEvent('canvas.regionselected', { + bubbles: false, + cancelable: true, + detail: { + points, + }, + })); + } else { + this.dispatchCanceledEvent(); + } + }; + + private onFindObject = (e: MouseEvent): void => { + if (e.button === 0) { + const { offset } = this.controller.geometry; + const [x, y] = translateToSVG(this.content, [e.clientX, e.clientY]); + const event: CustomEvent = new CustomEvent('canvas.find', { + bubbles: false, + cancelable: true, + detail: { + x: x - offset, + y: y - offset, + states: this.controller.objects, + }, + }); + + this.canvas.dispatchEvent(event); + e.preventDefault(); + } + }; + + private onFocusRegion = (x: number, y: number, width: number, height: number): void => { + // First of all, compute and apply scale + let scale = null; + + if ((this.geometry.angle / 90) % 2) { + // 90, 270, .. + scale = Math.min( + Math.max( + Math.min(this.geometry.canvas.width / height, this.geometry.canvas.height / width), + FrameZoom.MIN, + ), + FrameZoom.MAX, + ); + } else { + scale = Math.min( + Math.max( + Math.min(this.geometry.canvas.width / width, this.geometry.canvas.height / height), + FrameZoom.MIN, + ), + FrameZoom.MAX, + ); + } + + this.geometry = { ...this.geometry, scale }; + this.transformCanvas(); + + const [canvasX, canvasY] = translateFromSVG(this.content, [x + width / 2, y + height / 2]); + + const canvasOffset = this.canvas.getBoundingClientRect(); + const [cx, cy] = [ + this.canvas.clientWidth / 2 + canvasOffset.left, + this.canvas.clientHeight / 2 + canvasOffset.top, + ]; + + const dragged = { + ...this.geometry, + top: this.geometry.top + cy - canvasY, + left: this.geometry.left + cx - canvasX, + scale, + }; + + this.controller.geometry = dragged; + this.geometry = dragged; + this.moveCanvas(); + + this.canvas.dispatchEvent( + new CustomEvent('canvas.zoom', { + bubbles: false, + cancelable: true, + }), + ); + }; + + private moveCanvas(): void { + for (const obj of [this.background, this.grid, this.bitmap]) { + obj.style.top = `${this.geometry.top}px`; + obj.style.left = `${this.geometry.left}px`; + } + + for (const obj of [this.content, this.text, this.attachmentBoard]) { + obj.style.top = `${this.geometry.top - this.geometry.offset}px`; + obj.style.left = `${this.geometry.left - this.geometry.offset}px`; + } + + // Transform handlers + this.drawHandler.transform(this.geometry); + this.masksHandler.transform(this.geometry); + this.editHandler.transform(this.geometry); + this.zoomHandler.transform(this.geometry); + this.autoborderHandler.transform(this.geometry); + this.interactionHandler.transform(this.geometry); + this.regionSelector.transform(this.geometry); + this.objectSelector.transform(this.geometry); + this.sliceHandler.transform(this.geometry); + } + + private transformCanvas(): void { + // Transform canvas + for (const obj of [ + this.background, + this.grid, + this.content, + this.bitmap, + this.attachmentBoard, + ]) { + obj.style.transform = `scale(${this.geometry.scale}) rotate(${this.geometry.angle}deg)`; + } + + // Transform grid + this.gridPath.setAttribute('stroke-width', `${consts.BASE_GRID_WIDTH / this.geometry.scale}px`); + + // Transform all shape points + for (const element of [ + ...window.document.getElementsByClassName('svg_select_points'), + ...window.document.getElementsByClassName('svg_select_points_rot'), + ...window.document.getElementsByClassName('svg_select_boundingRect'), + ]) { + element.setAttribute('stroke-width', `${consts.POINTS_STROKE_WIDTH / this.geometry.scale}`); + element.setAttribute('r', `${this.configuration.controlPointsSize / this.geometry.scale}`); + } + + for (const element of window.document.getElementsByClassName('cvat_canvas_poly_direction')) { + const angle = (element as any).instance.data('angle'); + + (element as any).instance.style({ + transform: `scale(${1 / this.geometry.scale}) rotate(${angle}deg)`, + }); + } + + for (const element of window.document.getElementsByClassName('cvat_canvas_selected_point')) { + const previousWidth = element.getAttribute('stroke-width') as string; + element.setAttribute('stroke-width', `${+previousWidth * 2}`); + } + + // Transform all drawn shapes and text + for (const key of Object.keys(this.svgShapes)) { + const clientID = +key; + const object = this.svgShapes[clientID]; + object.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + }); + if (object.type === 'circle') { + object.attr('r', `${this.configuration.controlPointsSize / this.geometry.scale}`); + } + if (clientID in this.svgTexts) { + this.updateTextPosition(this.svgTexts[clientID]); + } + } + + // Transform skeleton edges + for (const skeletonEdge of window.document.getElementsByClassName('cvat_canvas_skeleton_edge')) { + skeletonEdge.setAttribute('stroke-width', `${consts.BASE_STROKE_WIDTH / this.geometry.scale}`); + } + + // Transform all drawn issues region + for (const issueRegion of Object.values(this.drawnIssueRegions)) { + ((issueRegion as any) as SVG.Shape).attr('r', `${(consts.BASE_POINT_SIZE * 3) / this.geometry.scale}`); + ((issueRegion as any) as SVG.Shape).attr( + 'stroke-width', + `${consts.BASE_STROKE_WIDTH / this.geometry.scale}`, + ); + } + + // Transform patterns + for (const pattern of [this.issueRegionPattern_1, this.issueRegionPattern_2]) { + pattern.attr({ + width: consts.BASE_PATTERN_SIZE / this.geometry.scale, + height: consts.BASE_PATTERN_SIZE / this.geometry.scale, + }); + + pattern.children().forEach((element: SVG.Element): void => { + element.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale); + }); + } + + // Transform handlers + this.drawHandler.transform(this.geometry); + this.masksHandler.transform(this.geometry); + this.editHandler.transform(this.geometry); + this.zoomHandler.transform(this.geometry); + this.autoborderHandler.transform(this.geometry); + this.interactionHandler.transform(this.geometry); + this.regionSelector.transform(this.geometry); + } + + private resizeCanvas(): void { + for (const obj of [this.background, this.masksContent, this.grid, this.bitmap]) { + obj.style.width = `${this.geometry.image.width}px`; + obj.style.height = `${this.geometry.image.height}px`; + } + + for (const obj of [this.content, this.text, this.attachmentBoard]) { + obj.style.width = `${this.geometry.image.width + this.geometry.offset * 2}px`; + obj.style.height = `${this.geometry.image.height + this.geometry.offset * 2}px`; + } + } + + private setupIssueRegions(issueRegions: Record): void { + for (const issueRegion of Object.keys(this.drawnIssueRegions)) { + if (!(issueRegion in issueRegions) || !+issueRegion) { + this.drawnIssueRegions[+issueRegion].remove(); + delete this.drawnIssueRegions[+issueRegion]; + } + } + + for (const issueRegion of Object.keys(issueRegions)) { + if (issueRegion in this.drawnIssueRegions) continue; + const points = this.translateToCanvas(issueRegions[+issueRegion].points); + if (points.length === 2) { + this.drawnIssueRegions[+issueRegion] = this.adoptedContent + .circle((consts.BASE_POINT_SIZE * 3 * 2) / this.geometry.scale) + .center(points[0], points[1]) + .addClass('cvat_canvas_issue_region') + .attr({ + id: `cvat_canvas_issue_region_${issueRegion}`, + fill: 'url(#cvat_issue_region_pattern_1)', + }); + } else if (points.length === 4) { + const stringified = stringifyPoints([ + points[0], + points[1], + points[2], + points[1], + points[2], + points[3], + points[0], + points[3], + ]); + this.drawnIssueRegions[+issueRegion] = this.adoptedContent + .polygon(stringified) + .addClass('cvat_canvas_issue_region') + .attr({ + id: `cvat_canvas_issue_region_${issueRegion}`, + fill: 'url(#cvat_issue_region_pattern_1)', + 'stroke-width': `${consts.BASE_STROKE_WIDTH / this.geometry.scale}`, + }); + } else { + const stringified = stringifyPoints(points); + this.drawnIssueRegions[+issueRegion] = this.adoptedContent + .polygon(stringified) + .addClass('cvat_canvas_issue_region') + .attr({ + id: `cvat_canvas_issue_region_${issueRegion}`, + fill: 'url(#cvat_issue_region_pattern_1)', + 'stroke-width': `${consts.BASE_STROKE_WIDTH / this.geometry.scale}`, + }); + } + + if (issueRegions[+issueRegion].hidden) { + this.drawnIssueRegions[+issueRegion].style({ display: 'none' }); + } + } + } + + private getVisibleSkeletonElements(clientID: number): number[] | null { + const visibleElements = this.controller.renderData.visibleSkeletonElements[clientID]; + return Array.isArray(visibleElements) ? [...visibleElements] : null; + } + + private getVisibleSkeletonElementIDs(clientID: number): Set | null { + const visibleElements = this.getVisibleSkeletonElements(clientID); + return visibleElements ? new Set(visibleElements) : null; + } + + private skeletonElementsVisibilityChanged(state: any): boolean { + if (state.shapeType !== 'skeleton') { + return false; + } + + const drawnVisibleElements = this.drawnStates[state.clientID].visibleSkeletonElements ?? null; + const visibleElements = this.getVisibleSkeletonElements(state.clientID); + return ( + drawnVisibleElements?.length !== visibleElements?.length || + (drawnVisibleElements || []).some((clientID, idx) => clientID !== visibleElements?.[idx]) + ); + } + + private setupObjects(states: any[]): void { + const created = []; + const updated = []; + + for (const state of states) { + if (!(state.clientID in this.drawnStates)) { + created.push(state); + } else { + const drawnState = this.drawnStates[state.clientID]; + // object has been changed or changed frame for a track + if ( + drawnState.updated !== state.updated || + drawnState.frame !== state.frame || + this.skeletonElementsVisibilityChanged(state) + ) { + updated.push(state); + } + } + } + const newIDs = states.map((state: any): number => state.clientID); + const deleted = Object.keys(this.drawnStates) + .map((clientID: string): number => +clientID) + .filter((id: number): boolean => !newIDs.includes(id)) + .map((id: number): any => this.drawnStates[id]); + + if (deleted.length || updated.length || created.length) { + if (this.activeElement.clientID !== null) { + this.deactivate(); + } + + this.deleteObjects(deleted); + this.addObjects(created); + + const updatedSkeletons = updated.filter((state: any): boolean => state.shapeType === 'skeleton'); + const updatedNotSkeletons = updated.filter((state: any): boolean => state.shapeType !== 'skeleton'); + // todo: implement updateObjects for skeletons, add group and color to updateObjects function + // change colors if necessary (for example when instance color is changed) + this.updateObjects(updatedNotSkeletons); + + this.deleteObjects(updatedSkeletons); + this.addObjects(updatedSkeletons); + + this.sortObjects(); + + if (this.controller.activeElement.clientID !== null) { + const { clientID } = this.controller.activeElement; + if (states.map((state: any): number => state.clientID).includes(clientID)) { + this.activate(this.controller.activeElement); + } + } + + this.autoborderHandler.updateObjects(); + } + } + + private hideDirection(shape: SVG.Polygon | SVG.PolyLine): void { + /* eslint class-methods-use-this: 0 */ + const handler = shape.remember('_selectHandler'); + if (!handler || !handler.nested) return; + const nested = handler.nested as SVG.Parent; + if (nested.children().length) { + nested.children()[0].removeClass('cvat_canvas_first_poly_point'); + } + + const node = nested.node as SVG.LinkedHTMLElement; + const directions = node.getElementsByClassName('cvat_canvas_poly_direction'); + for (const direction of directions) { + const { instance } = direction as any; + instance.off('click'); + instance.remove(); + } + } + + private showDirection(state: any, shape: SVG.Polygon | SVG.PolyLine): void { + const path = consts.ARROW_PATH; + + const points = parsePoints(state.points); + const handler = shape.remember('_selectHandler'); + + if (!handler || !handler.nested) return; + const firstCircle = handler.nested.children()[0]; + const secondCircle = handler.nested.children()[1]; + firstCircle.addClass('cvat_canvas_first_poly_point'); + + const [cx, cy] = [(secondCircle.cx() + firstCircle.cx()) / 2, (secondCircle.cy() + firstCircle.cy()) / 2]; + const [firstPoint, secondPoint] = points.slice(0, 2); + const xAxis = { i: 1, j: 0 }; + const baseVector = { i: secondPoint.x - firstPoint.x, j: secondPoint.y - firstPoint.y }; + const baseVectorLength = vectorLength(baseVector); + let cosinus = 0; + + if (baseVectorLength !== 0) { + // two points have the same coordinates + cosinus = scalarProduct(xAxis, baseVector) / (vectorLength(xAxis) * baseVectorLength); + } + const angle = (Math.acos(cosinus) * (Math.sign(baseVector.j) || 1) * 180) / Math.PI; + + const pathElement = handler.nested + .path(path) + .fill('white') + .stroke({ + width: 1, + color: 'black', + }) + .addClass('cvat_canvas_poly_direction') + .style({ + 'transform-origin': `${cx}px ${cy}px`, + transform: `scale(${1 / this.geometry.scale}) rotate(${angle}deg)`, + }) + .move(cx, cy); + + pathElement.on('click', (e: MouseEvent): void => { + if (e.button === 0) { + e.stopPropagation(); + if (state.shapeType === 'polygon') { + const reversedPoints = [points[0], ...points.slice(1).reverse()]; + this.onEditDone(state, pointsToNumberArray(reversedPoints)); + } else { + const reversedPoints = points.reverse(); + this.onEditDone(state, pointsToNumberArray(reversedPoints)); + } + } + }); + + pathElement.data('angle', angle); + pathElement.dmove(-pathElement.width() / 2, -pathElement.height() / 2); + } + + private selectize(value: boolean, shape: SVG.Element): void { + const mousedownHandler = (e: MouseEvent): void => { + if (e.button !== 0) return; + e.preventDefault(); + + if (this.activeElement.clientID !== null) { + const pointID = Array.prototype.indexOf.call( + ((e.target as HTMLElement).parentElement as HTMLElement).children, + e.target, + ); + const [state] = this.controller.objects.filter( + (_state: any): boolean => _state.clientID === this.activeElement.clientID, + ); + + if (['polygon', 'polyline', 'points'].includes(state.shapeType)) { + if (state.shapeType === 'points' && (e.altKey || e.ctrlKey)) { + const selectedClientID = +((e.target as HTMLElement).parentElement as HTMLElement).getAttribute('clientID'); + + if (state.clientID !== selectedClientID) { + return; + } + } + if (e.altKey) { + const { points } = state; + if ( + (state.shapeType === 'polygon' && state.points.length > 6) || + (state.shapeType === 'polyline' && state.points.length > 4) || + (state.shapeType === 'points' && state.points.length > 2) + ) { + this.onEditDone(state, points.slice(0, pointID * 2).concat(points.slice(pointID * 2 + 2))); + } + } else if (e.shiftKey) { + this.onEditStart(state); + this.editHandler.edit({ + enabled: true, + state, + pointID, + }); + } + } + } + }; + + const dblClickHandler = (e: MouseEvent): void => { + if (this.activeElement.clientID !== null) { + const [state] = this.controller.objects.filter( + (_state: any): boolean => _state.clientID === this.activeElement.clientID, + ); + + if (state.shapeType === 'cuboid') { + e.preventDefault(); + e.stopPropagation(); + if (e.shiftKey) { + const points = this.translateFromCanvas( + pointsToNumberArray((e.target as any).parentElement.parentElement.instance.attr('points')), + ); + this.onEditDone(state, points); + } + } + } + }; + + const contextMenuHandler = (e: MouseEvent): void => { + const pointID = Array.prototype.indexOf.call( + ((e.target as HTMLElement).parentElement as HTMLElement).children, + e.target, + ); + if (this.activeElement.clientID !== null) { + const [state] = this.controller.objects.filter( + (_state: any): boolean => _state.clientID === this.activeElement.clientID, + ); + this.canvas.dispatchEvent( + new CustomEvent('canvas.contextmenu', { + bubbles: false, + cancelable: true, + detail: { + mouseEvent: e, + objectState: state, + pointID, + }, + }), + ); + } + e.preventDefault(); + }; + + if (value) { + const getGeometry = (): Geometry => this.geometry; + const getController = (): CanvasController => this.controller; + const getActiveElement = (): ActiveElement => this.activeElement; + (shape as any).selectize(value, { + deepSelect: true, + pointSize: (2 * this.configuration.controlPointsSize) / this.geometry.scale, + rotationPoint: shape.type === 'rect' || shape.type === 'ellipse', + pointsExclude: shape.type === 'image' ? ['lt', 'rt', 'rb', 'lb', 't', 'r', 'b', 'l'] : [], + pointType(cx: number, cy: number): SVG.Circle { + const circle: SVG.Circle = this.nested + .circle(this.options.pointSize) + .stroke('black') + .fill('inherit') + .center(cx, cy) + .attr({ + 'fill-opacity': 1, + 'stroke-width': consts.POINTS_STROKE_WIDTH / getGeometry().scale, + }); + circle.on('mouseenter', (e: MouseEvent): void => { + const activeElement = getActiveElement(); + if (activeElement !== null && (e.altKey || e.ctrlKey)) { + const [state] = getController().objects.filter( + (_state: any): boolean => _state.clientID === activeElement.clientID, + ); + if (state?.shapeType === 'points') { + const selectedClientID = +((e.target as HTMLElement).parentElement as HTMLElement).getAttribute('clientID'); + if (state.clientID !== selectedClientID) { + return; + } + } + } + + circle.attr({ + 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / getGeometry().scale, + }); + + circle.on('dblclick', dblClickHandler); + circle.on('mousedown', mousedownHandler); + circle.on('contextmenu', contextMenuHandler); + circle.addClass('cvat_canvas_selected_point'); + }); + + circle.on('mouseleave', (): void => { + circle.attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / getGeometry().scale, + }); + + circle.off('dblclick', dblClickHandler); + circle.off('mousedown', mousedownHandler); + circle.off('contextmenu', contextMenuHandler); + circle.removeClass('cvat_canvas_selected_point'); + }); + return circle; + }, + }); + } else { + (shape as any).selectize(false, { + deepSelect: true, + }); + } + + const handler = shape.remember('_selectHandler'); + if (handler && handler.nested) { + handler.nested.fill(shape.attr('fill')); + } + + const [rotationPoint] = window.document.getElementsByClassName('svg_select_points_rot'); + const [topPoint] = window.document.getElementsByClassName('svg_select_points_t'); + if (rotationPoint && !rotationPoint.children.length) { + if (topPoint) { + const rotY = +(rotationPoint as SVGEllipseElement).getAttribute('cy'); + const topY = +(topPoint as SVGEllipseElement).getAttribute('cy'); + (rotationPoint as SVGCircleElement).style.transform = `translate(0px, -${rotY - topY + 20}px)`; + } + + const title = document.createElementNS('http://www.w3.org/2000/svg', 'title'); + title.textContent = 'Hold Shift to snap angle'; + rotationPoint.appendChild(title); + } + + if (value && shape.type === 'image') { + const [boundingRect] = window.document.getElementsByClassName('svg_select_boundingRect'); + if (boundingRect) { + (boundingRect as SVGRectElement).style.opacity = '1'; + boundingRect.setAttribute('fill', 'none'); + boundingRect.setAttribute('stroke', shape.attr('stroke')); + boundingRect.setAttribute('stroke-width', `${consts.BASE_STROKE_WIDTH / this.geometry.scale}px`); + if (shape.hasClass('cvat_canvas_shape_occluded')) { + boundingRect.setAttribute('stroke-dasharray', '5'); + } + } + } + } + + private draggable( + state: any, + shape: SVG.Shape, + onDragStart: () => void = () => {}, + onDragMove: () => void = () => {}, + onDragEnd: () => void = () => {}, + ): void { + let draggableInstance = shape; + if (shape.classes().includes('cvat_canvas_shape_skeleton')) { + // for skeletons we use wrapping rectangle to drag the skeleton itself + draggableInstance = (shape as any).children().find((child: SVG.Element) => child.type === 'rect'); + } + + if (state) { + let start = Date.now(); + let aborted = false; + let skeletonSVGTemplate: SVG.G = null; + shape.addClass('cvat_canvas_shape_draggable'); + (draggableInstance as any).draggable({ + ...(state.shapeType === 'mask' ? { snapToGrid: 1 } : {}), + }); + + let startCenter = null; + draggableInstance.on('dragstart', (): void => { + onDragStart(); + this.draggableShape = shape; + const { cx, cy } = shape.bbox(); + startCenter = { x: cx, y: cy }; + start = Date.now(); + }).on('dragmove', (e: CustomEvent): void => { + onDragMove(); + if (state.shapeType === 'skeleton' && e.target) { + const { instance } = e.target as any; + const [x, y] = [instance.x(), instance.y()]; + const prevXtl = +draggableInstance.attr('data-xtl'); + const prevYtl = +draggableInstance.attr('data-ytl'); + + for (const child of (shape as SVG.G).children()) { + if (child.type === 'circle') { + const childClientID = child.attr('data-client-id'); + if (state.elements.find((el: any) => el.clientID === childClientID).lock || false) { + continue; + } + child.center(child.cx() - prevXtl + x, child.cy() - prevYtl + y); + } + } + + draggableInstance.attr('data-xtl', x); + draggableInstance.attr('data-ytl', y); + draggableInstance.attr('data-xbr', x + instance.width()); + draggableInstance.attr('data-ybr', y + instance.height()); + + skeletonSVGTemplate = skeletonSVGTemplate ?? makeSVGFromTemplate(state.label.structure.svg); + setupSkeletonEdges(shape as SVG.G, skeletonSVGTemplate); + } + }).on('dragend', (): void => { + if (aborted) { + this.resetViewPosition(state.clientID); + return; + } + + onDragEnd(); + this.draggableShape = null; + const { cx, cy } = shape.bbox(); + + const dx2 = (startCenter.x - cx) ** 2; + const dy2 = (startCenter.y - cy) ** 2; + if (Math.sqrt(dx2 + dy2) > 0) { + if (state.shapeType === 'mask') { + const { points } = state; + const x = Math.trunc(shape.x()) - this.geometry.offset; + const y = Math.trunc(shape.y()) - this.geometry.offset; + points.splice(-4); + points.push(x, y, x + shape.width() - 1, y + shape.height() - 1); + this.onEditDone(state, points); + } else if (state.shapeType === 'skeleton') { + const points = []; + state.elements.forEach((element: any) => { + const elementShape = (shape as SVG.G).children() + .find((child: SVG.Shape) => ( + child.id() === `cvat_canvas_shape_${element.clientID}` + )); + + if (elementShape) { + points.push(...this.translateFromCanvas(readPointsFromShape(elementShape))); + } + }); + this.onEditDone(state, points); + } else { + // these points does not take into account possible transformations, applied on the element + // so, if any (like rotation) we need to map them to canvas coordinate space + let points = readPointsFromShape(shape); + const { rotation } = shape.transform(); + if (rotation) { + points = this.translatePointsFromRotatedShape(shape, points); + } + + this.onEditDone(state, this.translateFromCanvas(points)); + } + + this.canvas.dispatchEvent( + new CustomEvent('canvas.dragshape', { + bubbles: false, + cancelable: true, + detail: { + state, + duration: Date.now() - start, + }, + }), + ); + } + }).on('dragabort', (): void => { + onDragEnd(); + this.draggableShape = null; + aborted = true; + // disable internal drag events of SVG.js + // call chain is (mouseup -> SVG.handler.end -> SVG.handler.drag -> dragend) + window.dispatchEvent(new MouseEvent('mouseup')); + }); + } else { + shape.removeClass('cvat_canvas_shape_draggable'); + + if (this.draggableShape === shape) { + draggableInstance.fire('dragabort'); + } + + draggableInstance.off('dragstart'); + draggableInstance.off('dragmove'); + draggableInstance.off('dragend'); + draggableInstance.off('dragabort'); + (draggableInstance as any).draggable(false); + } + } + + private resizable( + state: any, + shape: SVG.Shape, + onResizeStart: () => void = () => {}, + onResizing: () => void = () => {}, + onResizeEnd: () => void = () => {}, + ): void { + let resizableInstance = shape; + let skeletonSVGTemplate: SVG.G = null; + + if (shape.classes().includes('cvat_canvas_shape_skeleton')) { + // for skeletons we use wrapping rectangle to resize the skeleton itself + resizableInstance = (shape as any).children().find((child: SVG.Element) => child.type === 'rect'); + + const circles = (shape as any).children().filter((child: SVG.Element) => child.type === 'circle'); + const svgElements = Object.fromEntries(circles.map((circle: SVG.Circle) => [circle.attr('data-client-id'), circle])); + + Object.entries(svgElements).forEach(([key, element]) => { + if (state) { + const clientID = +key; + const elementState = state.elements + .find((_element: any) => _element.clientID === clientID); + const text = this.svgTexts[clientID]; + const hideElementText = (): void => { + if (text) { + text.addClass('cvat_canvas_hidden'); + } + }; + + const showElementText = (): void => { + if (text) { + text.removeClass('cvat_canvas_hidden'); + this.updateTextPosition(text); + } + }; + + if (!elementState.lock) { + this.draggable(elementState, element, () => { + this.mode = Mode.DRAG; + hideElementText(); + }, () => { + skeletonSVGTemplate = skeletonSVGTemplate ?? makeSVGFromTemplate(state.label.structure.svg); + setupSkeletonEdges(shape as SVG.G, skeletonSVGTemplate); + }, () => { + this.mode = Mode.IDLE; + showElementText(); + }); + } + } else { + this.draggable(null, element); + } + }); + } + + if (state) { + let resized = false; + let aborted = false; + let start = Date.now(); + let draggedPointIndex: number | null = null; // Track which point is being dragged + + (resizableInstance as any) + .resize({ + snapToGrid: 0.1, + snapToAngle: this.snapToAngleResize, + }) + .on('resizestart', (e: CustomEvent): void => { + onResizeStart(); + resized = false; + start = Date.now(); + this.resizableShape = shape; + const detail = (e.detail.event.detail as any); + draggedPointIndex = detail?.i ?? null; + }) + .on('resizing', (e: CustomEvent): void => { + resized = true; + onResizing(); + + if (this.configuration.snapToPoint && + !this.ctrlPressed && + ['polygon', 'polyline', 'points'].includes(state.shapeType) && + draggedPointIndex !== null && + draggedPointIndex >= 0) { + const snapRadius = this.configuration.snapRadius / this.geometry.scale; + + applySnapToShapePoint( + shape as SVG.Polygon | SVG.PolyLine, + draggedPointIndex, + this.drawnStates, + this.geometry.offset, + snapRadius, + state.clientID, + ); + } + + if (state.shapeType === 'skeleton' && e.target) { + const { instance } = e.target as any; + + // rotate skeleton instead of wrapping bounding box + const { rotation } = resizableInstance.transform(); + shape.rotate(rotation); + + const [x, y] = [instance.x(), instance.y()]; + const prevXtl = +resizableInstance.attr('data-xtl'); + const prevYtl = +resizableInstance.attr('data-ytl'); + const prevXbr = +resizableInstance.attr('data-xbr'); + const prevYbr = +resizableInstance.attr('data-ybr'); + + if (prevXbr - prevXtl < 0.1) return; + if (prevYbr - prevYtl < 0.1) return; + + for (const child of (shape as SVG.G).children()) { + if (child.type === 'circle') { + const childClientID = child.attr('data-client-id'); + if (state.elements.find((el: any) => el.clientID === childClientID).lock || false) { + continue; + } + const offsetX = (child.cx() - prevXtl) / (prevXbr - prevXtl); + const offsetY = (child.cy() - prevYtl) / (prevYbr - prevYtl); + child.center(offsetX * instance.width() + x, offsetY * instance.height() + y); + } + } + + resizableInstance.attr('data-xtl', x); + resizableInstance.attr('data-ytl', y); + resizableInstance.attr('data-xbr', x + instance.width()); + resizableInstance.attr('data-ybr', y + instance.height()); + + resized = true; + skeletonSVGTemplate = skeletonSVGTemplate ?? makeSVGFromTemplate(state.label.structure.svg); + setupSkeletonEdges(shape as SVG.G, skeletonSVGTemplate); + } + }) + .on('resizedone', (): void => { + if (aborted) { + this.resetViewPosition(state.clientID); + return; + } + + onResizeEnd(); + this.resizableShape = null; + + // be sure, that rotation in range [0; 360] + let rotation = getRoundedRotation(shape); + while (rotation < 0) rotation += 360; + rotation %= 360; + + if (resized) { + if (state.shapeType === 'skeleton') { + if (rotation) { + this.onEditDone(state, state.points, rotation); + } else { + const points: number[] = []; + state.elements.forEach((element: any) => { + const elementShape = (shape as SVG.G).children() + .find((child: SVG.Shape) => ( + child.id() === `cvat_canvas_shape_${element.clientID}` + )); + + if (elementShape) { + points.push(...this.translateFromCanvas( + readPointsFromShape(elementShape), + )); + } + }); + this.onEditDone(state, points, 0); + } + } else { + // these points does not take into account possible transformations, applied on the element + // so, if any (like rotation) we need to map them to canvas coordinate space + let points = readPointsFromShape(shape); + if (rotation) { + points = this.translatePointsFromRotatedShape(shape, points); + } + this.onEditDone(state, this.translateFromCanvas(points), rotation); + } + + this.canvas.dispatchEvent( + new CustomEvent('canvas.resizeshape', { + bubbles: false, + cancelable: true, + detail: { + state, + duration: Date.now() - start, + }, + }), + ); + } + }).on('resizeabort', () => { + onResizeEnd(); + aborted = true; + this.resizableShape = null; + // disable internal resize events of SVG.js + // call chain is (mouseup -> SVG.handler.end -> SVG.handler.resize-> resizeend) + window.dispatchEvent(new MouseEvent('mouseup')); + }); + } else { + if (this.resizableShape === shape) { + resizableInstance.fire('resizeabort'); + } + + (shape as any).off('resizestart'); + (shape as any).off('resizing'); + (shape as any).off('resizedone'); + (shape as any).off('resizeabort'); + (shape as any).resize('stop'); + } + } + + private onKeyDown = (e: KeyboardEvent): void => { + if (e.repeat) { + return; + } + + const code = (e.code ?? '').toLowerCase(); + + if (code.includes('shift')) { + this.snapToAngleResize = consts.SNAP_TO_ANGLE_RESIZE_SHIFT; + if (this.activeElement) { + const shape = this.svgShapes[this.activeElement.clientID]; + if (shape && shape?.remember('_selectHandler')?.options?.rotationPoint) { + if (this.drawnStates[this.activeElement.clientID]?.shapeType === 'skeleton') { + const wrappingRect = (shape as any).children().find((child: SVG.Element) => child.type === 'rect'); + if (wrappingRect) { + (wrappingRect as any).resize({ snapToAngle: this.snapToAngleResize }); + } + } else { + (shape as any).resize({ snapToAngle: this.snapToAngleResize }); + } + } + } + } + + if (code.includes('control')) { + this.ctrlPressed = true; + } + }; + + private onKeyUp = (e: KeyboardEvent): void => { + const code = (e.code ?? '').toLowerCase(); + + if (code.includes('shift') && this.activeElement) { + this.snapToAngleResize = consts.SNAP_TO_ANGLE_RESIZE_DEFAULT; + if (this.activeElement) { + const shape = this.svgShapes[this.activeElement.clientID]; + if (shape && shape?.remember('_selectHandler')?.options?.rotationPoint) { + if (this.drawnStates[this.activeElement.clientID]?.shapeType === 'skeleton') { + const wrappingRect = (shape as any).children().find((child: SVG.Element) => child.type === 'rect'); + if (wrappingRect) { + (wrappingRect as any).resize({ snapToAngle: this.snapToAngleResize }); + } + } else { + (shape as any).resize({ snapToAngle: this.snapToAngleResize }); + } + } + } + } + + if (code.includes('control')) { + this.ctrlPressed = false; + } + }; + + private onMouseUp = (event: MouseEvent): void => { + if (event.button === 0 || event.button === 1) { + this.controller.disableDrag(); + } + }; + + public constructor(model: CanvasModel & Master, controller: CanvasController) { + this.controller = controller; + this.geometry = controller.geometry; + this.svgShapes = {}; + this.svgTexts = {}; + this.drawnStates = {}; + this.drawnIssueRegions = {}; + this.activeElement = { + clientID: null, + attributeID: null, + }; + this.highlightedElements = { + elementsIDs: [], + severity: null, + }; + this.configuration = model.configuration; + this.mode = Mode.IDLE; + this.snapToAngleResize = consts.SNAP_TO_ANGLE_RESIZE_DEFAULT; + this.ctrlPressed = false; + this.innerObjectsFlags = { + drawHidden: {}, + editHidden: {}, + sliceHidden: {}, + }; + + this.isImageLoading = true; + this.draggableShape = null; + this.resizableShape = null; + + // Create HTML elements + this.text = window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + this.adoptedText = SVG.adopt((this.text as any) as HTMLElement) as SVG.Container; + this.background = window.document.createElement('canvas'); + this.masksContent = window.document.createElement('canvas'); + this.bitmapUpdateReqId = 0; + this.bitmap = window.document.createElement('canvas'); + // window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + + this.grid = window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + this.gridPath = window.document.createElementNS('http://www.w3.org/2000/svg', 'path'); + this.gridPattern = window.document.createElementNS('http://www.w3.org/2000/svg', 'pattern'); + + this.content = window.document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + this.adoptedContent = SVG.adopt((this.content as any) as HTMLElement) as SVG.Container; + + this.attachmentBoard = window.document.createElement('div'); + + this.canvas = window.document.createElement('div'); + + const gridDefs: SVGDefsElement = window.document.createElementNS('http://www.w3.org/2000/svg', 'defs'); + const gridRect: SVGRectElement = window.document.createElementNS('http://www.w3.org/2000/svg', 'rect'); + + // Setup defs + const contentDefs = this.adoptedContent.defs(); + + this.issueRegionPattern_1 = contentDefs + .pattern(consts.BASE_PATTERN_SIZE, consts.BASE_PATTERN_SIZE, (add): void => { + add.line(0, 0, 0, 10).stroke('red'); + }) + .attr({ + id: 'cvat_issue_region_pattern_1', + patternTransform: 'rotate(45)', + patternUnits: 'userSpaceOnUse', + }); + + this.issueRegionPattern_2 = contentDefs + .pattern(consts.BASE_PATTERN_SIZE, consts.BASE_PATTERN_SIZE, (add): void => { + add.line(0, 0, 0, 10).stroke('yellow'); + }) + .attr({ + id: 'cvat_issue_region_pattern_2', + patternTransform: 'rotate(45)', + patternUnits: 'userSpaceOnUse', + }); + + // Setup grid + this.grid.setAttribute('id', 'cvat_canvas_grid'); + this.grid.setAttribute('version', '2'); + this.gridPath.setAttribute('d', 'M 1000 0 L 0 0 0 1000'); + this.gridPath.setAttribute('fill', 'none'); + this.gridPath.setAttribute('stroke-width', `${consts.BASE_GRID_WIDTH}`); + this.gridPath.setAttribute('opacity', 'inherit'); + this.gridPattern.setAttribute('id', 'cvat_canvas_grid_pattern'); + this.gridPattern.setAttribute('width', '100'); + this.gridPattern.setAttribute('height', '100'); + this.gridPattern.setAttribute('patternUnits', 'userSpaceOnUse'); + gridRect.setAttribute('width', '100%'); + gridRect.setAttribute('height', '100%'); + gridRect.setAttribute('fill', 'url(#cvat_canvas_grid_pattern)'); + + // Setup content + this.text.setAttribute('id', 'cvat_canvas_text_content'); + this.background.setAttribute('id', 'cvat_canvas_background'); + this.masksContent.setAttribute('id', 'cvat_canvas_masks_content'); + this.content.setAttribute('id', 'cvat_canvas_content'); + this.bitmap.setAttribute('id', 'cvat_canvas_bitmap'); + this.bitmap.style.display = 'none'; + + // Setup sticked div + this.attachmentBoard.setAttribute('id', 'cvat_canvas_attachment_board'); + + // Setup wrappers + this.canvas.setAttribute('id', 'cvat_canvas_wrapper'); + + // Unite created HTML elements together + this.grid.appendChild(gridDefs); + this.grid.appendChild(gridRect); + + gridDefs.appendChild(this.gridPattern); + this.gridPattern.appendChild(this.gridPath); + + this.canvas.appendChild(this.text); + this.canvas.appendChild(this.background); + this.canvas.appendChild(this.masksContent); + this.canvas.appendChild(this.bitmap); + this.canvas.appendChild(this.grid); + this.canvas.appendChild(this.content); + this.canvas.appendChild(this.attachmentBoard); + + // Setup API handlers + this.autoborderHandler = new AutoborderHandlerImpl(this.content, () => this.ctrlPressed); + this.drawHandler = new DrawHandlerImpl( + this.onDrawDone, + this.adoptedContent, + this.adoptedText, + this.autoborderHandler, + this.geometry, + this.configuration, + () => this.drawnStates, + () => this.ctrlPressed, + ); + this.masksHandler = new MasksHandlerImpl( + this.onDrawDone, + this.controller.draw.bind(this.controller), + this.onEditStart, + this.onEditDone, + this.drawHandler, + this.masksContent, + ); + this.editHandler = new EditHandlerImpl(this.onEditDone, this.adoptedContent, this.autoborderHandler); + this.mergeHandler = new MergeHandlerImpl( + this.onMergeDone, + this.onFindObject, + this.adoptedContent, + ); + this.splitHandler = new SplitHandlerImpl( + this.onSplitDone, + this.onFindObject, + this.adoptedContent, + ); + this.objectSelector = new ObjectSelectorImpl( + this.onFindObject, + () => this.controller.objects, + this.geometry, + this.adoptedContent, + ); + this.groupHandler = new GroupHandlerImpl(this.onSelectDone, this.objectSelector); + this.sliceHandler = new SliceHandlerImpl( + (clientID) => this.setupInnerFlags(clientID, 'sliceHidden', true), + (clientID) => this.setupInnerFlags(clientID, 'sliceHidden', false), + this.onSliceDone, + this.onMessage, + this.onError, + () => this.controller.objects, + this.geometry, + this.adoptedContent, + this.objectSelector, + ); + this.regionSelector = new RegionSelectorImpl( + this.onRegionSelected, + this.adoptedContent, + this.geometry, + ); + this.zoomHandler = new ZoomHandlerImpl(this.onFocusRegion, this.adoptedContent, this.geometry); + this.interactionHandler = new InteractionHandlerImpl( + this.onInteraction, + this.onMessage, + this.adoptedContent, + this.geometry, + this.configuration, + ); + + // Setup event handlers + this.canvas.addEventListener('dblclick', (e: MouseEvent): void => { + if (this.activeElement.clientID !== null) { + // -1 means auto padding based on the shape size + this.controller.focus(this.activeElement.clientID, -1); + } else { + this.controller.fit(); + } + e.preventDefault(); + }); + + this.canvas.addEventListener('mousedown', (event): void => { + if ([0, 1].includes(event.button)) { + if ( + [Mode.IDLE, Mode.DRAG_CANVAS, Mode.MERGE, Mode.SPLIT] + .includes(this.mode) || event.button === 1 || event.altKey + ) { + this.controller.enableDrag(event.clientX, event.clientY); + } + } + }); + + window.document.addEventListener('mouseup', this.onMouseUp); + window.document.addEventListener('keydown', this.onKeyDown); + window.document.addEventListener('keyup', this.onKeyUp); + + for (const eventName of ['wheel', 'mousedown', 'dblclick', 'contextmenu']) { + this.attachmentBoard.addEventListener(eventName, (event) => { + event.stopPropagation(); + }); + } + + this.canvas.addEventListener('wheel', (event): void => { + if (this.ctrlPressed) { + // we do not use event.ctrlKey to handle pinch zoom using touchpad correctly + // peach zoom automatically generates 'wheel' event with event.ctrlKey equals to true + // even when the ctrl key is not pressed actually + return; + } + + let { deltaY } = event; + // clamp too high values to avoid strong zooming + // high values are usually applicable to mice + // 8 is a good experimental value to avoid strong zooming + const LIMIT_DELTA_Y = 8; + deltaY = clamp(deltaY, -LIMIT_DELTA_Y, LIMIT_DELTA_Y); + + const { offset } = this.controller.geometry; + const point = translateToSVG(this.content, [event.clientX, event.clientY]); + this.controller.zoom(point[0] - offset, point[1] - offset, deltaY); + this.canvas.dispatchEvent( + new CustomEvent('canvas.zoom', { + bubbles: false, + cancelable: true, + }), + ); + event.preventDefault(); + }); + + this.canvas.addEventListener('mousemove', (e): void => { + this.controller.drag(e.clientX, e.clientY); + + if (this.mode !== Mode.IDLE) return; + if (e.ctrlKey || e.altKey) return; + + if (!this.isImageLoading) { + const { offset } = this.controller.geometry; + const [x, y] = translateToSVG(this.content, [e.clientX, e.clientY]); + const event: CustomEvent = new CustomEvent('canvas.moved', { + bubbles: false, + cancelable: true, + detail: { + x: x - offset, + y: y - offset, + states: this.controller.objects, + }, + }); + + this.canvas.dispatchEvent(event); + } + }); + + this.content.oncontextmenu = (): boolean => false; + model.subscribe(this); + } + + public notify(model: CanvasModel & Master, reason: UpdateReasons): void { + this.geometry = this.controller.geometry; + if (reason === UpdateReasons.CONFIG_UPDATED) { + const { activeElement } = this; + this.deactivate(); + const { configuration } = model; + + const updateShapeViews = (states: DrawnState[], parentState?: DrawnState): void => { + for (const drawnState of states) { + const { + fill, stroke, 'fill-opacity': fillOpacity, + } = this.getShapeColorization(drawnState, { parentState }); + const shapeView = window.document.getElementById(`cvat_canvas_shape_${drawnState.clientID}`); + const [objectState] = this.controller.objects + .filter((_state: any) => _state.clientID === drawnState.clientID); + if (shapeView) { + const handler = (shapeView as any).instance.remember('_selectHandler'); + if (handler && handler.nested) { + handler.nested.fill({ color: fill }); + } + + if (drawnState.shapeType === 'mask') { + // if there are masks, we need to redraw them + this.deleteObjects([drawnState]); + this.addObjects([objectState]); + continue; + } + + (shapeView as any).instance + .fill({ color: fill, opacity: fillOpacity }) + .stroke({ color: stroke }); + } + + if (drawnState.elements) { + updateShapeViews(drawnState.elements, drawnState); + } + } + }; + + const withUpdatingShapeViews = configuration.shapeOpacity !== this.configuration.shapeOpacity || + configuration.selectedShapeOpacity !== this.configuration.selectedShapeOpacity || + configuration.outlinedBorders !== this.configuration.outlinedBorders || + configuration.colorBy !== this.configuration.colorBy || + configuration.showConflicts !== this.configuration.showConflicts; + + if (configuration.displayAllText && !this.configuration.displayAllText) { + for (const i in this.drawnStates) { + if (!(i in this.svgTexts)) { + this.svgTexts[i] = this.addText(this.drawnStates[i]); + } + } + } else if (configuration.displayAllText === false && this.configuration.displayAllText) { + for (const clientID in this.drawnStates) { + if (+clientID !== activeElement.clientID) { + this.deleteText(+clientID); + } + } + } + + const recreateText = configuration.textContent !== this.configuration.textContent; + const updateTextPosition = configuration.displayAllText !== this.configuration.displayAllText || + configuration.textFontSize !== this.configuration.textFontSize || + configuration.textPosition !== this.configuration.textPosition || + recreateText; + + if (configuration.smoothImage === true) { + this.background.classList.remove('cvat_canvas_pixelized'); + } else if (configuration.smoothImage === false) { + this.background.classList.add('cvat_canvas_pixelized'); + } + + this.configuration = configuration; + if (withUpdatingShapeViews) { + updateShapeViews(Object.values(this.drawnStates)); + } + + if (recreateText) { + const states = this.controller.objects; + for (const key of Object.keys(this.drawnStates)) { + const clientID = +key; + const [state] = states.filter((_state: any) => _state.clientID === clientID); + if (clientID in this.svgTexts) { + this.deleteText(+clientID); + if (state) { + this.addText(state); + } + } + } + } + + if (updateTextPosition) { + for (const i in this.drawnStates) { + if (i in this.svgTexts) { + this.updateTextPosition(this.svgTexts[i]); + } + } + } + + if (typeof configuration.CSSImageFilter === 'string') { + this.background.style.filter = configuration.CSSImageFilter; + } + this.activate(activeElement); + this.editHandler.configure(this.configuration); + this.drawHandler.configure(this.configuration); + this.masksHandler.configure(this.configuration); + this.autoborderHandler.configure(this.configuration); + this.interactionHandler.configure(this.configuration); + this.sliceHandler.configure(this.configuration); + this.transformCanvas(); + + // remove if exist and not enabled + // this.setupObjects([]); + // this.setupObjects(model.objects); + } else if (reason === UpdateReasons.BITMAP) { + const { imageBitmap } = model; + if (imageBitmap) { + this.bitmap.style.display = ''; + this.redrawBitmap(); + } else { + this.bitmap.style.display = 'none'; + } + } else if (reason === UpdateReasons.IMAGE_CHANGED) { + const { image } = model; + if (image) { + this.isImageLoading = false; + const ctx = this.background.getContext('2d'); + this.background.setAttribute('width', `${image.renderWidth}px`); + this.background.setAttribute('height', `${image.renderHeight}px`); + + if (ctx) { + ctx.drawImage(image.imageData, 0, 0, image.renderWidth, image.renderHeight); + } + + if (model.imageIsDeleted) { + let { width, height } = this.background; + if (image.imageData instanceof ImageData) { + width = image.imageData.width; + height = image.imageData.height; + } + + this.background.classList.add('cvat_canvas_removed_image'); + const canvasContext = this.background.getContext('2d'); + const fontSize = width / 10; + canvasContext.font = `bold ${fontSize}px serif`; + canvasContext.textAlign = 'center'; + canvasContext.lineWidth = fontSize / 20; + canvasContext.strokeStyle = 'white'; + canvasContext.strokeText('IMAGE REMOVED', width / 2, height / 2); + canvasContext.fillStyle = 'black'; + canvasContext.fillText('IMAGE REMOVED', width / 2, height / 2); + } else if (this.background.classList.contains('cvat_canvas_removed_image')) { + this.background.classList.remove('cvat_canvas_removed_image'); + } + + this.moveCanvas(); + this.resizeCanvas(); + this.transformCanvas(); + } else { + this.isImageLoading = true; + } + } else if (reason === UpdateReasons.FITTED_CANVAS) { + // Canvas geometry is going to be changed. Old object positions aren't valid any more + this.setupObjects([]); + this.setupIssueRegions({}); + this.moveCanvas(); + this.resizeCanvas(); + this.canvas.dispatchEvent( + new CustomEvent('canvas.reshape', { + bubbles: false, + cancelable: true, + }), + ); + } else if ([UpdateReasons.IMAGE_ZOOMED, UpdateReasons.IMAGE_FITTED].includes(reason)) { + if (reason === UpdateReasons.IMAGE_FITTED) { + this.canvas.dispatchEvent( + new CustomEvent('canvas.fit', { + bubbles: false, + cancelable: true, + }), + ); + } + + this.moveCanvas(); + this.transformCanvas(); + } else if (reason === UpdateReasons.IMAGE_ROTATED) { + this.transformCanvas(); + } else if (reason === UpdateReasons.IMAGE_MOVED) { + this.moveCanvas(); + } else if (reason === UpdateReasons.OBJECTS_UPDATED) { + this.objectSelector.resetSelected(); + this.setupObjects(this.controller.objects); + if (this.mode === Mode.MERGE) { + this.mergeHandler.repeatSelection(); + } + const event: CustomEvent = new CustomEvent('canvas.setup'); + this.canvas.dispatchEvent(event); + } else if (reason === UpdateReasons.ISSUE_REGIONS_UPDATED) { + this.setupIssueRegions(this.controller.issueRegions); + } else if (reason === UpdateReasons.GRID_UPDATED) { + const size: Size = this.geometry.grid; + this.gridPattern.setAttribute('width', `${size.width}`); + this.gridPattern.setAttribute('height', `${size.height}`); + } else if (reason === UpdateReasons.SHAPE_FOCUSED) { + const padding = this.configuration.focusedObjectPadding ?? 0; + const { clientID } = this.controller.focusData; + const drawnState = this.drawnStates[clientID]; + const object = this.svgShapes[clientID]; + if (drawnState && object) { + const { offset } = this.geometry; + let [x, y, width, height] = [0, 0, 0, 0]; + + if (drawnState.shapeType === 'mask') { + const [xtl, ytl, xbr, ybr] = drawnState.points.slice(-4); + x = xtl + offset; + y = ytl + offset; + width = xbr - xtl + 1; + height = ybr - ytl + 1; + } else { + const bbox: SVG.BBox = object.bbox(); + ({ + x, y, width, height, + } = bbox); + } + + this.onFocusRegion(x - padding, y - padding, width + padding * 2, height + padding * 2); + } + } else if (reason === UpdateReasons.SHAPE_ACTIVATED) { + this.activate(this.controller.activeElement); + } else if (reason === UpdateReasons.SHAPE_HIGHLIGHTED) { + this.highlight(this.controller.highlightedElements); + } else if (reason === UpdateReasons.SELECT_REGION) { + if (this.mode === Mode.SELECT_REGION) { + this.regionSelector.select(true); + this.canvas.style.cursor = 'pointer'; + } else { + this.regionSelector.select(false); + this.canvas.style.cursor = ''; + } + } else if (reason === UpdateReasons.DRAG_CANVAS) { + if (this.mode === Mode.DRAG_CANVAS) { + this.canvas.dispatchEvent( + new CustomEvent('canvas.dragstart', { + bubbles: false, + cancelable: true, + }), + ); + this.canvas.style.cursor = 'move'; + } else { + this.canvas.dispatchEvent( + new CustomEvent('canvas.dragstop', { + bubbles: false, + cancelable: true, + }), + ); + this.canvas.style.cursor = ''; + } + } else if (reason === UpdateReasons.ZOOM_CANVAS) { + if (this.mode === Mode.ZOOM_CANVAS) { + this.canvas.dispatchEvent( + new CustomEvent('canvas.zoomstart', { + bubbles: false, + cancelable: true, + }), + ); + this.canvas.style.cursor = 'zoom-in'; + this.zoomHandler.zoom(); + } else { + this.canvas.dispatchEvent( + new CustomEvent('canvas.zoomstop', { + bubbles: false, + cancelable: true, + }), + ); + this.canvas.style.cursor = ''; + this.zoomHandler.cancel(); + } + } else if (reason === UpdateReasons.DRAW) { + const data: DrawData = this.controller.drawData; + if (data.enabled && [Mode.IDLE, Mode.DRAW].includes(this.mode)) { + if (data.shapeType !== 'mask') { + this.drawHandler.draw(data, this.geometry); + } else { + this.masksHandler.draw(data); + } + + if (this.mode === Mode.IDLE) { + this.canvas.style.cursor = 'crosshair'; + this.mode = Mode.DRAW; + this.canvas.dispatchEvent( + new CustomEvent('canvas.drawstart', { + bubbles: false, + cancelable: true, + detail: { + drawData: data, + }, + }), + ); + + if (typeof data.redraw === 'number') { + this.setupInnerFlags(data.redraw, 'drawHidden', true); + } + } + } else if (this.mode !== Mode.IDLE) { + this.canvas.style.cursor = ''; + this.mode = Mode.IDLE; + if (this.masksHandler.enabled) { + this.masksHandler.draw(data); + } else { + this.drawHandler.draw(data, this.geometry); + } + } + } else if (reason === UpdateReasons.EDIT) { + const data = this.controller.editData; + if (data.enabled && data.state.shapeType === 'mask') { + this.masksHandler.edit(data); + } else if (this.masksHandler.enabled) { + this.masksHandler.edit(data); + } else if (this.editHandler.enabled && this.editHandler.shapeType === 'polyline') { + this.editHandler.edit(data); + } + } else if (reason === UpdateReasons.INTERACT) { + const data: InteractionData = this.controller.interactionData; + if (data.enabled && this.mode === Mode.IDLE) { + this.canvas.style.cursor = 'crosshair'; + this.mode = Mode.INTERACT; + } + + if (!data.enabled && this.mode === Mode.INTERACT) { + this.canvas.style.cursor = ''; + this.mode = Mode.IDLE; + } + this.interactionHandler.interact(data); + } else if (reason === UpdateReasons.MERGE) { + const data: MergeData = this.controller.mergeData; + if (data.enabled) { + this.canvas.style.cursor = 'copy'; + this.mode = Mode.MERGE; + } + this.mergeHandler.merge(data); + } else if (reason === UpdateReasons.SPLIT) { + const data: SplitData = this.controller.splitData; + if (data.enabled) { + this.canvas.style.cursor = 'copy'; + this.mode = Mode.SPLIT; + this.splitHandler.split(data); + } + } else if ([UpdateReasons.JOIN, UpdateReasons.GROUP].includes(reason)) { + let data: GroupData | JoinData = null; + if (reason === UpdateReasons.GROUP) { + data = this.controller.groupData; + this.mode = Mode.GROUP; + this.groupHandler.group(data, {}); + } else { + data = this.controller.joinData; + this.mode = Mode.JOIN; + + this.onMessage([{ + type: 'text', + icon: 'info', + content: 'Click polygons or masks you would like to join together. To unselect click selected shape one more time', + }], 'join'); + + this.groupHandler.group(data, { + shapeType: ['mask', 'polygon'], + objectType: ['shape'], + restrictToFirstSelectedType: true, + }); + } + } else if (reason === UpdateReasons.SLICE) { + const data = this.controller.sliceData; + if (data.enabled && this.mode === Mode.IDLE) { + this.mode = Mode.SLICE; + this.sliceHandler.slice(data); + } + } else if (reason === UpdateReasons.SELECT) { + this.objectSelector.push(this.controller.selected); + if (this.mode === Mode.MERGE) { + this.mergeHandler.select(this.controller.selected); + } else if (this.mode === Mode.SPLIT) { + this.splitHandler.select(this.controller.selected); + } + } else if (reason === UpdateReasons.CANCEL) { + if (this.mode === Mode.DRAW) { + if (this.masksHandler.enabled) { + this.masksHandler.cancel(); + } else { + this.drawHandler.cancel(); + } + } else if (this.mode === Mode.INTERACT) { + this.interactionHandler.cancel(); + } else if (this.mode === Mode.MERGE) { + this.mergeHandler.cancel(); + } else if (this.mode === Mode.SPLIT) { + this.splitHandler.cancel(); + } else if (this.mode === Mode.GROUP || this.mode === Mode.JOIN) { + this.groupHandler.cancel(); + } else if (this.mode === Mode.SLICE) { + this.sliceHandler.cancel(); + } else if (this.mode === Mode.SELECT_REGION) { + this.regionSelector.cancel(); + } else if (this.mode === Mode.EDIT) { + if (this.masksHandler.enabled) { + this.masksHandler.cancel(); + } else { + this.editHandler.cancel(); + } + } else if (this.mode === Mode.DRAG_CANVAS) { + this.canvas.dispatchEvent( + new CustomEvent('canvas.dragstop', { + bubbles: false, + cancelable: true, + }), + ); + } else if (this.mode === Mode.ZOOM_CANVAS) { + this.zoomHandler.cancel(); + this.canvas.dispatchEvent( + new CustomEvent('canvas.zoomstop', { + bubbles: false, + cancelable: true, + }), + ); + } + this.canvas.style.cursor = ''; + this.dispatchCanceledEvent(); + } else if (reason === UpdateReasons.DATA_FAILED) { + this.onError(model.exception, 'data fetching'); + } else if (reason === UpdateReasons.DESTROY) { + this.canvas.dispatchEvent( + new CustomEvent('canvas.destroy', { + bubbles: false, + cancelable: true, + }), + ); + + window.document.removeEventListener('keydown', this.onKeyDown); + window.document.removeEventListener('keyup', this.onKeyUp); + window.document.removeEventListener('mouseup', this.onMouseUp); + this.interactionHandler.destroy(); + } + + if (model.imageBitmap && [UpdateReasons.OBJECTS_UPDATED].includes(reason)) { + this.redrawBitmap(); + } + } + + public html(): HTMLDivElement { + return this.canvas; + } + + public setupConflictRegions(state: any): number[] { + let cx = 0; + let cy = 0; + const shape = this.svgShapes[state.clientID]; + if (!shape) return []; + const box = (shape.node as any).getBBox(); + cx = box.x + (box.width) / 2; + cy = box.y; + return [cx, cy]; + } + + public translateFromSVG(point: number[]): number[] { + return translateFromSVG(this.content, point); + } + + private redrawBitmap(): void { + this.bitmapUpdateReqId++; + const { bitmapUpdateReqId } = this; + const width = +this.background.style.width.slice(0, -2); + const height = +this.background.style.height.slice(0, -2); + this.bitmap.setAttribute('width', `${width}px`); + this.bitmap.setAttribute('height', `${height}px`); + const states = this.controller.objects; + + const ctx = this.bitmap.getContext('2d'); + ctx.imageSmoothingEnabled = false; + if (ctx) { + ctx.clearRect(0, 0, width, height); + for (const state of states) { + if (state.hidden || state.outside) continue; + ctx.fillStyle = 'white'; + if (['rectangle', 'polygon', 'cuboid'].includes(state.shapeType)) { + let points = [...state.points]; + if (state.shapeType === 'rectangle') { + points = rotate2DPoints( + points[0] + (points[2] - points[0]) / 2, + points[1] + (points[3] - points[1]) / 2, + state.rotation, + [ + points[0], // xtl + points[1], // ytl + points[2], // xbr + points[1], // ytl + points[2], // xbr + points[3], // ybr + points[0], // xtl + points[3], // ybr + ], + ); + } else if (state.shapeType === 'cuboid') { + points = [ + points[0], + points[1], + points[4], + points[5], + points[8], + points[9], + points[12], + points[13], + ]; + } + ctx.beginPath(); + ctx.moveTo(points[0], points[1]); + for (let i = 0; i < points.length; i += 2) { + ctx.lineTo(points[i], points[i + 1]); + } + ctx.closePath(); + ctx.fill(); + } + + if (state.shapeType === 'ellipse') { + const [cx, cy, rightX, topY] = state.points; + ctx.beginPath(); + ctx.ellipse(cx, cy, rightX - cx, cy - topY, (state.rotation * Math.PI) / 180.0, 0, 2 * Math.PI); + ctx.closePath(); + ctx.fill(); + } + + if (state.shapeType === 'mask') { + const { points } = state; + const [left, top, right, bottom] = points.slice(-4); + const imageBitmap = RLEToImageData(255, 255, 255, points); + imageDataToDataURL( + imageBitmap, + right - left + 1, + bottom - top + 1, + (dataURL: string) => { + if (bitmapUpdateReqId === this.bitmapUpdateReqId) { + const img = document.createElement('img'); + img.addEventListener('load', () => { + ctx.drawImage(img, left, top); + URL.revokeObjectURL(dataURL); + }, { once: true }); + img.addEventListener('error', () => { + URL.revokeObjectURL(dataURL); + }, { once: true }); + img.src = dataURL; + } else { + URL.revokeObjectURL(dataURL); + } + }, + ); + } + + if (state.shapeType === 'cuboid') { + for (let i = 0; i < 5; i++) { + const points = [ + state.points[(0 + i * 4) % 16], + state.points[(1 + i * 4) % 16], + state.points[(2 + i * 4) % 16], + state.points[(3 + i * 4) % 16], + state.points[(6 + i * 4) % 16], + state.points[(7 + i * 4) % 16], + state.points[(4 + i * 4) % 16], + state.points[(5 + i * 4) % 16], + ]; + ctx.beginPath(); + ctx.moveTo(points[0], points[1]); + for (let j = 0; j < points.length; j += 2) { + ctx.lineTo(points[j], points[j + 1]); + } + ctx.closePath(); + ctx.fill(); + } + } + } + } + } + + private saveState(state: any): DrawnState { + const result = { + clientID: state.clientID, + outside: state.outside, + occluded: state.occluded, + source: state.source, + hidden: state.hidden, + lock: state.lock, + shapeType: state.shapeType, + points: [...state.points], + rotation: state.rotation, + attributes: { ...state.attributes }, + descriptions: [...state.descriptions], + zOrder: state.zOrder, + pinned: state.pinned, + updated: state.updated, + frame: state.frame, + label: state.label, + group: { id: state.group.id, color: state.group.color }, + color: state.color, + elements: state.shapeType === 'skeleton' ? + state.elements.map((element: any) => this.saveState(element)) : null, + visibleSkeletonElements: state.shapeType === 'skeleton' ? + this.getVisibleSkeletonElements(state.clientID) : null, + }; + + return result; + } + + private getShapeColorization(state: any, opts: { + parentState?: any, + } = {}): { fill: string; stroke: string, 'fill-opacity': number } { + const { shapeType } = state; + const parentShapeType = opts.parentState?.shapeType; + const { configuration } = this; + const { colorBy, shapeOpacity, outlinedBorders } = configuration; + let shapeColor = ''; + + if (colorBy === ColorBy.INSTANCE) { + shapeColor = state.color; + } else if (colorBy === ColorBy.GROUP) { + shapeColor = state.group.color; + } else if (colorBy === ColorBy.LABEL) { + shapeColor = state.label.color; + } + if (this.highlightedElements.elementsIDs.length) { + if (this.highlightedElements.elementsIDs.includes(state.clientID)) { + if (this.highlightedElements.severity === HighlightSeverity.ERROR) { + shapeColor = consts.CONFLICT_COLOR; + } else if (this.highlightedElements.severity === HighlightSeverity.WARNING) { + shapeColor = consts.WARNING_COLOR; + } + } else { + shapeColor = consts.SHADED_COLOR; + } + } + const outlinedColor = parentShapeType === 'skeleton' ? 'black' : outlinedBorders || shapeColor; + + return { + fill: shapeColor, + stroke: outlinedColor, + 'fill-opacity': !['polyline', 'points', 'skeleton'].includes(shapeType) || parentShapeType === 'skeleton' ? shapeOpacity : 0, + }; + } + + private getHighlightClassname(): string { + const { severity } = this.highlightedElements; + if (severity === HighlightSeverity.ERROR) { + return 'cvat_canvas_conflicted'; + } + if (severity === HighlightSeverity.WARNING) { + return 'cvat_canvas_warned'; + } + return ''; + } + + private updateObjects(states: any[]): void { + for (const state of states) { + const { clientID } = state; + const drawnState = this.drawnStates[clientID]; + const shape = this.svgShapes[state.clientID]; + const text = this.svgTexts[state.clientID]; + const isInvisible = state.hidden || state.outside || this.isInnerHidden(state.clientID); + + if (drawnState.hidden !== state.hidden || drawnState.outside !== state.outside) { + if (isInvisible) { + (state.shapeType === 'points' ? shape.remember('_selectHandler').nested : shape).addClass( + 'cvat_canvas_hidden', + ); + if (text) { + text.addClass('cvat_canvas_hidden'); + } + } else { + (state.shapeType === 'points' ? shape.remember('_selectHandler').nested : shape).removeClass( + 'cvat_canvas_hidden', + ); + if (text) { + text.removeClass('cvat_canvas_hidden'); + this.updateTextPosition(text); + } + } + } + + if (drawnState.zOrder !== state.zOrder) { + if (state.shapeType === 'points') { + shape.remember('_selectHandler').nested.attr('data-z-order', state.zOrder); + } else { + shape.attr('data-z-order', state.zOrder); + } + } + + if (drawnState.occluded !== state.occluded) { + const instance = state.shapeType === 'points' ? this.svgShapes[clientID].remember('_selectHandler').nested : shape; + if (state.occluded) { + instance.addClass('cvat_canvas_shape_occluded'); + } else { + instance.removeClass('cvat_canvas_shape_occluded'); + } + } + + if (drawnState.pinned !== state.pinned && this.activeElement.clientID !== null) { + const activeElement = { ...this.activeElement }; + this.deactivate(); + this.activate(activeElement); + } + + if (drawnState.rotation) { + // need to rotate it back before changing points + shape.untransform(); + } + + const pointsUpdated = state.points.length !== drawnState.points.length || + state.points.some((p: number, id: number): boolean => p !== drawnState.points[id]); + + if (pointsUpdated) { + if (state.shapeType === 'mask') { + // if masks points were updated, draw from scratch + this.deleteObjects([this.drawnStates[+clientID]]); + this.addObjects([state]); + continue; + } + + const translatedPoints: number[] = this.translateToCanvas(state.points); + + if (state.shapeType === 'rectangle') { + const [xtl, ytl, xbr, ybr] = translatedPoints; + + shape.attr({ + x: xtl, + y: ytl, + width: xbr - xtl, + height: ybr - ytl, + }); + } else if (state.shapeType === 'ellipse') { + const [cx, cy] = translatedPoints; + const [rx, ry] = [translatedPoints[2] - cx, cy - translatedPoints[3]]; + shape.attr({ + cx, cy, rx, ry, + }); + } else { + const stringified = stringifyPoints(translatedPoints); + if (state.shapeType !== 'cuboid') { + (shape as any).clear(); + } + shape.attr('points', stringified); + + if (state.shapeType === 'points' && !isInvisible) { + this.selectize(false, shape); + this.setupPoints(shape as SVG.PolyLine, state); + } + } + } + + if (state.rotation) { + // now, when points changed, need to rotate it to new angle + shape.rotate(state.rotation); + } + + const stateDescriptions = state.descriptions; + const drawnStateDescriptions = drawnState.descriptions; + const rotationUpdated = drawnState.rotation !== state.rotation; + + if ( + drawnState.label.id !== state.label.id || + drawnState.zOrder !== state.zOrder || + pointsUpdated || + rotationUpdated || + drawnStateDescriptions.length !== stateDescriptions.length || + drawnStateDescriptions.some((desc: string, id: number): boolean => desc !== stateDescriptions[id]) + ) { + // remove created text and create it again + if (text) { + text.remove(); + this.updateTextPosition(this.addText(state)); + } + } else { + const attrNames = Object.fromEntries(state.label.attributes.map((attr) => [attr.id, attr.name])); + // check if there are updates in attributes + for (const attrID of Object.keys(state.attributes)) { + if (state.attributes[attrID] !== drawnState.attributes[+attrID]) { + if (text) { + const [span] = text.node.querySelectorAll(`[attrID="${attrID}"]`); + if (span && span.textContent) { + span.textContent = `${attrNames[attrID]}: ${state.attributes[attrID]}`; + } + } + } + } + } + + if ( + drawnState.label.id !== state.label.id || + drawnState.group.id !== state.group.id || + drawnState.group.color !== state.group.color || + drawnState.color !== state.color + ) { + // update shape color if necessary + if (shape) { + if (state.shapeType === 'mask') { + // if masks points were updated, draw from scratch + this.deleteObjects([this.drawnStates[+clientID]]); + this.addObjects([state]); + continue; + } else if (state.shapeType === 'points') { + const colorization = { ...this.getShapeColorization(state) }; + shape.remember('_selectHandler').nested.attr(colorization); + shape.attr(colorization); + } else { + shape.attr({ ...this.getShapeColorization(state) }); + } + } + } + + this.drawnStates[state.clientID] = this.saveState(state); + } + } + + private deleteObjects(states: any[]): void { + for (const state of states) { + if (state.clientID in this.svgTexts) { + this.deleteText(state.clientID); + } + + if (state.shapeType === 'skeleton') { + this.deleteObjects(state.elements); + } + + if (state.clientID in this.svgShapes) { + const shape = this.svgShapes[state.clientID]; + const { node } = shape; + shape.fire('remove'); + shape.off('click'); + shape.off('remove'); + if (node instanceof SVGImageElement) { + URL.revokeObjectURL(node.href.baseVal); + } + shape.remove(); + delete this.svgShapes[state.clientID]; + } + + if (state.clientID in this.drawnStates) { + delete this.drawnStates[state.clientID]; + } + } + } + + private addObjects(states: any[]): void { + const { displayAllText } = this.configuration; + for (const state of states) { + const points: number[] = state.points as number[]; + + // TODO: Use enums after typification cvat-core + if (state.shapeType === 'mask') { + this.svgShapes[state.clientID] = this.addMask(points, state); + } else if (state.shapeType === 'skeleton') { + this.svgShapes[state.clientID] = this.addSkeleton(state); + } else { + const translatedPoints: number[] = this.translateToCanvas(points); + if (state.shapeType === 'rectangle') { + this.svgShapes[state.clientID] = this.addRect(translatedPoints, state); + } else { + const stringified = stringifyPoints(translatedPoints); + + if (state.shapeType === 'polygon') { + this.svgShapes[state.clientID] = this.addPolygon(stringified, state); + } else if (state.shapeType === 'polyline') { + this.svgShapes[state.clientID] = this.addPolyline(stringified, state); + } else if (state.shapeType === 'points') { + this.svgShapes[state.clientID] = this.addPoints(stringified, state); + } else if (state.shapeType === 'ellipse') { + this.svgShapes[state.clientID] = this.addEllipse(stringified, state); + } else if (state.shapeType === 'cuboid') { + this.svgShapes[state.clientID] = this.addCuboid(stringified, state); + } else { + continue; + } + } + } + + this.svgShapes[state.clientID].on('click.canvas', (): void => { + this.canvas.dispatchEvent( + new CustomEvent('canvas.clicked', { + bubbles: false, + cancelable: true, + detail: { + state, + }, + }), + ); + }); + + if (displayAllText) { + this.addText(state); + this.updateTextPosition(this.svgTexts[state.clientID]); + } + + this.drawnStates[state.clientID] = this.saveState(state); + } + } + + private sortObjects(): void { + // TODO: Can be significantly optimized + const states = Array.from(this.content.getElementsByClassName('cvat_canvas_shape')).map((state: SVGElement): [ + SVGElement, + number, + ] => [state, +state.getAttribute('data-z-order')]); + + Array.from(this.content.getElementsByClassName('cvat_canvas_crosshair')) + .forEach((line: SVGLineElement): void => this.content.append(line)); + Array.from(this.content.getElementsByClassName('cvat_interaction_point')).concat( + Array.from(this.content.getElementsByClassName('cvat_interaction_rectangle')), + ).forEach((interactionShape: SVGCircleElement): void => this.content.append(interactionShape)); + + const needSort = states.some((pair): boolean => pair[1] !== states[0][1]); + if (!states.length || !needSort) { + return; + } + + const sorted = states.sort((a, b): number => a[1] - b[1]); + sorted.forEach((pair): void => { + this.content.appendChild(pair[0]); + }); + + this.content.prepend(...sorted.map((pair): SVGElement => pair[0])); + } + + private deactivateAttribute(): void { + const { clientID, attributeID } = this.activeElement; + if (clientID !== null && attributeID !== null) { + const text = this.svgTexts[clientID]; + if (text) { + const [span] = (text.node.querySelectorAll(`[attrID="${attributeID}"]`) as any) as SVGTSpanElement[]; + if (span) { + span.style.fill = ''; + } + } + + this.activeElement = { + ...this.activeElement, + attributeID: null, + }; + } + } + + private deactivateShape(): void { + if (this.activeElement.clientID) { + const { displayAllText } = this.configuration; + const { clientID } = this.activeElement; + const drawnState = this.drawnStates[clientID]; + const shape = this.svgShapes[clientID]; + + if (drawnState.shapeType === 'points') { + this.svgShapes[clientID] + .remember('_selectHandler').nested + .removeClass('cvat_canvas_shape_activated'); + } else { + shape.removeClass('cvat_canvas_shape_activated'); + } + + if (drawnState.shapeType === 'mask') { + shape.attr('opacity', `${Math.sqrt(this.configuration.shapeOpacity)}`); + } else { + shape.attr('fill-opacity', `${this.configuration.shapeOpacity}`); + } + + if (!drawnState.pinned) { + // as resizable for skeletons uses "draggable" inside, it is important first to disable draggable + this.draggable(null, shape); + } + + if (drawnState.shapeType !== 'mask') { + this.resizable(null, shape); + } else { + (shape as any).off('dblclick'); + } + + if (drawnState.shapeType !== 'points') { + this.selectize(false, shape); + } + + if (drawnState.shapeType === 'cuboid') { + (shape as any).attr('projections', false); + } + + // TODO: Hide text only if it is hidden by settings + const text = this.svgTexts[clientID]; + if (text && !displayAllText) { + this.deleteText(clientID); + } + + this.sortObjects(); + + this.activeElement = { + ...this.activeElement, + clientID: null, + }; + } + } + + private deactivate(): void { + this.deactivateAttribute(); + this.deactivateShape(); + } + + private activateAttribute(clientID: number, attributeID: number): void { + const text = this.svgTexts[clientID]; + if (text) { + const [span] = (text.node.querySelectorAll(`[attrID="${attributeID}"]`) as any) as SVGTSpanElement[]; + if (span) { + span.style.fill = 'red'; + } + + this.activeElement = { + ...this.activeElement, + attributeID, + }; + } + } + + private activateShape(clientID: number): void { + const [state] = this.controller.objects.filter((_state: any): boolean => _state.clientID === clientID); + if (state && state.shapeType === 'points') { + this.svgShapes[clientID] + .remember('_selectHandler') + .nested.style('pointer-events', this.stateIsLocked(state) ? 'none' : ''); + } + + if (!state || state.hidden || state.outside) { + return; + } + + const shape = this.svgShapes[clientID]; + if (!this.svgTexts[clientID]) { + this.addText(state); + } + this.updateTextPosition(this.svgTexts[clientID]); + + if (this.stateIsLocked(state)) { + return; + } + + if (state.shapeType === 'points') { + this.svgShapes[clientID] + .remember('_selectHandler').nested + .addClass('cvat_canvas_shape_activated'); + } else { + shape.addClass('cvat_canvas_shape_activated'); + } + + if (state.shapeType === 'mask') { + shape.attr('opacity', `${Math.sqrt(this.configuration.selectedShapeOpacity)}`); + } else { + shape.attr('fill-opacity', `${this.configuration.selectedShapeOpacity}`); + } + + if (state.shapeType === 'points') { + this.content.append(this.svgShapes[clientID].remember('_selectHandler').nested.node); + } else { + this.content.append(shape.node); + } + + const { showProjections } = this.configuration; + if (state.shapeType === 'cuboid' && showProjections) { + (shape as any).attr('projections', true); + } + + if (state.shapeType !== 'points') { + this.selectize(true, shape); + } + + const textList = [ + state.clientID, ...state.elements.map((element: any): number => element.clientID), + ].map((id: number) => this.svgTexts[id]).filter((text: SVG.Text | undefined) => ( + typeof text !== 'undefined' + )); + + const hideText = (): void => { + textList.forEach((text: SVG.Text) => { + text.addClass('cvat_canvas_hidden'); + }); + }; + + const showText = (): void => { + textList.forEach((text: SVG.Text) => { + text.removeClass('cvat_canvas_hidden'); + this.updateTextPosition(text); + }); + }; + + if (state.shapeType !== 'mask') { + const showDirection = (): void => { + if (['polygon', 'polyline'].includes(state.shapeType)) { + this.showDirection(state, shape as SVG.Polygon | SVG.PolyLine); + } + }; + + const hideDirection = (): void => { + if (['polygon', 'polyline'].includes(state.shapeType)) { + this.hideDirection(shape as SVG.Polygon | SVG.PolyLine); + } + }; + + let shapeSizeElement: ShapeSizeElement | null = null; + this.resizable(state, shape, () => { + this.mode = Mode.RESIZE; + hideDirection(); + hideText(); + if (state.shapeType === 'rectangle' || state.shapeType === 'ellipse') { + shapeSizeElement = displayShapeSize(this.adoptedContent, this.adoptedText); + } + }, () => { + if (shapeSizeElement) { + shapeSizeElement.update(shape); + } + }, () => { + this.mode = Mode.IDLE; + if (shapeSizeElement) { + shapeSizeElement.rm(); + shapeSizeElement = null; + } + + showDirection(); + showText(); + }); + + showDirection(); + } else { + (shape as any).on('dblclick', (e: MouseEvent) => { + if (e.shiftKey) { + this.controller.edit({ enabled: true, state }); + e.stopPropagation(); + } + }); + } + + if (!state.pinned) { + this.draggable(state, shape, () => { + this.mode = Mode.DRAG; + hideText(); + }, () => {}, () => { + this.mode = Mode.IDLE; + showText(); + }); + } + + this.canvas.dispatchEvent( + new CustomEvent('canvas.activated', { + bubbles: false, + cancelable: true, + detail: { + state, + }, + }), + ); + } + + private activate(activeElement: ActiveElement): void { + // Check if another element have been already activated + if (this.activeElement.clientID !== null) { + if (this.activeElement.clientID !== activeElement.clientID) { + // Deactivate previous shape and attribute + this.deactivate(); + } else if (this.activeElement.attributeID !== activeElement.attributeID) { + this.deactivateAttribute(); + } + } + + const { clientID, attributeID } = activeElement; + if (clientID !== null && this.activeElement.clientID !== clientID) { + this.activateShape(clientID); + this.activeElement = { + ...this.activeElement, + clientID, + }; + } + + if (clientID !== null && attributeID !== null && this.activeElement.attributeID !== attributeID) { + this.activateAttribute(clientID, attributeID); + } + } + + private highlight(highlightedElements: HighlightedElements): void { + this.highlightedElements.elementsIDs.forEach((clientID) => { + const shapeView = window.document.getElementById(`cvat_canvas_shape_${clientID}`); + if (shapeView) shapeView.classList.remove(this.getHighlightClassname()); + }); + const redrawMasks = (highlightedElements.elementsIDs.length !== 0 || + this.highlightedElements.elementsIDs.length !== 0); + + if (highlightedElements.elementsIDs.length) { + this.highlightedElements = { ...highlightedElements }; + this.canvas.classList.add('cvat-canvas-highlight-enabled'); + this.highlightedElements.elementsIDs.forEach((clientID) => { + const shapeView = window.document.getElementById(`cvat_canvas_shape_${clientID}`); + if (shapeView) shapeView.classList.add(this.getHighlightClassname()); + }); + } else { + this.highlightedElements = { + elementsIDs: [], + severity: null, + }; + this.canvas.classList.remove('cvat-canvas-highlight-enabled'); + } + if (redrawMasks) { + const masks = Object.values(this.drawnStates).filter((state) => state.shapeType === 'mask'); + this.deleteObjects(masks); + this.addObjects(masks); + } + if (this.highlightedElements.elementsIDs.length) { + this.deactivate(); + const clientID = this.highlightedElements.elementsIDs[0]; + this.activate({ clientID, attributeID: null }); + } + } + + // Update text position after corresponding box has been moved, resized, etc. + private updateTextPosition( + text: SVG.Text, + options: { rotation?: { angle: number, cx: number, cy: number } } = {}, + ): void { + const clientID = text.attr('data-client-id'); + if (!Number.isInteger(clientID)) return; + const shape = this.svgShapes[clientID]; + if (!shape) return; + + if (text.node.style.display === 'none') return; // wrong transformation matrix + const { textFontSize } = this.configuration; + let { textPosition } = this.configuration; + if (shape.type === 'circle') { + // force auto for skeleton elements + textPosition = 'auto'; + } + + text.untransform(); + text.style({ 'font-size': `${textFontSize}px` }); + const rotation = options.rotation?.angle || getRoundedRotation(shape); + + // Find the best place for a text + let [clientX, clientY, clientCX, clientCY]: number[] = [0, 0, 0, 0]; + if (textPosition === 'center') { + let cx = 0; + let cy = 0; + if (['rect', 'image'].includes(shape.type)) { + // for rectangle/mask finding a center is simple + cx = +shape.attr('x') + +shape.attr('width') / 2; + cy = +shape.attr('y') + +shape.attr('height') / 2; + } else if (shape.type === 'g') { + ({ cx, cy } = shape.bbox()); + } else if (shape.type === 'ellipse') { + // even simpler for ellipses + cx = +shape.attr('cx'); + cy = +shape.attr('cy'); + } else { + // for polyshapes we use special algorithm + const points = parsePoints(pointsToNumberArray(shape.attr('points'))); + [cx, cy] = polylabel([points.map((point) => [point.x, point.y])]); + } + + [clientX, clientY] = translateFromSVG(this.content, [cx, cy]); + // center is exactly clientX, clientY + clientCX = clientX; + clientCY = clientY; + } else { + let box = (shape.node as any).getBBox(); + + // Translate the whole box to the client coordinate system + const [x1, y1, x2, y2]: number[] = translateFromSVG(this.content, [ + box.x, + box.y, + box.x + box.width, + box.y + box.height, + ]); + + clientCX = x1 + (x2 - x1) / 2; + clientCY = y1 + (y2 - y1) / 2; + + box = { + x: Math.min(x1, x2), + y: Math.min(y1, y2), + width: Math.max(x1, x2) - Math.min(x1, x2), + height: Math.max(y1, y2) - Math.min(y1, y2), + }; + + // first try to put to the top right corner + [clientX, clientY] = [box.x + box.width, box.y]; + if ( + clientX + ((text.node as any) as SVGTextElement) + .getBBox().width + consts.TEXT_MARGIN > this.canvas.offsetWidth + ) { + // if out of visible area, try to put text to top left corner + [clientX, clientY] = [box.x, box.y]; + } + } + + // Translate found coordinates to text SVG + const [x, y, rotX, rotY]: number[] = translateToSVG(this.text, [ + clientX + (textPosition === 'auto' ? consts.TEXT_MARGIN : 0), + clientY + (textPosition === 'auto' ? consts.TEXT_MARGIN : 0), + options.rotation?.cx || clientCX, + options.rotation?.cy || clientCY, + ]); + + const textBBox = ((text.node as any) as SVGTextElement).getBBox(); + // Finally draw a text + if (textPosition === 'center') { + text.move(x - textBBox.width / 2, y - textBBox.height / 2); + } else { + text.move(x, y); + } + + let childOptions = {}; + if (rotation) { + text.rotate(rotation, rotX, rotY); + childOptions = { + rotation: { + angle: rotation, + cx: clientCX, + cy: clientCY, + }, + }; + } + + if (clientID in this.drawnStates && this.drawnStates[clientID].shapeType === 'skeleton') { + this.drawnStates[clientID].elements.forEach((element: DrawnState) => { + if (element.clientID in this.svgTexts) { + this.updateTextPosition(this.svgTexts[element.clientID], childOptions); + } + }); + } + + function applyParentX(parentText: SVGTSpanElement | SVGTextElement): void { + for (let i = 0; i < parentText.children.length; i++) { + if (i === 0) { + // do not align the first child + continue; + } + + const tspan = parentText.children[i]; + tspan.setAttribute('x', parentText.getAttribute('x')); + applyParentX(tspan as SVGTSpanElement); + } + } + + applyParentX(text.node as any as SVGTextElement); + } + + private deleteText(clientID: number): void { + if (clientID in this.svgTexts) { + this.svgTexts[clientID].remove(); + delete this.svgTexts[clientID]; + } + + if (clientID in this.drawnStates && this.drawnStates[clientID].shapeType === 'skeleton') { + this.drawnStates[clientID].elements.forEach((element) => { + this.deleteText(element.clientID); + }); + } + } + + private addText(state: any, options: { textContent?: string, isSkeletonElement?: boolean } = {}): SVG.Text { + const { undefinedAttrValue } = this.configuration; + const content = options.textContent || this.configuration.textContent; + const withID = content.includes('id'); + const withAttr = content.includes('attributes'); + const withLabel = content.includes('label'); + const withSource = content.includes('source'); + const withDescriptions = content.includes('descriptions'); + const withDimensions = content.includes('dimensions'); + const withLayer = content.includes('layer') || content.includes('zOrder'); + + const textFontSize = this.configuration.textFontSize || 12; + const { + label, clientID, attributes, source, descriptions, score, votes, zOrder, + } = state; + const isConsensus = source === 'consensus'; + const withScore = isConsensus && !options.isSkeletonElement; + const withVotes = isConsensus && !options.isSkeletonElement; + + const attrNames = Object.fromEntries(state.label.attributes.map((attr) => [attr.id, attr.name])); + if (state.shapeType === 'skeleton') { + const visibleElementIDs = this.getVisibleSkeletonElementIDs(state.clientID); + state.elements.forEach((element: any) => { + if (visibleElementIDs && !visibleElementIDs.has(element.clientID)) { + return; + } + + if (!(element.clientID in this.svgTexts)) { + this.svgTexts[element.clientID] = this.addText(element, { + textContent: [ + ...(withLabel ? ['label'] : []), + ...(withAttr ? ['attributes'] : []), + // Note: explicitly exclude 'layer', 'score' and 'votes' for skeleton elements + ].join(',') || ' ', + isSkeletonElement: true, + }); + } + }); + } + + this.svgTexts[state.clientID] = this.adoptedText + .text((block): void => { + block.tspan(`${withLabel ? label.name : ''} ` + + `${withID ? clientID : ''} ` + + `${withSource ? `(${source})` : ''}`).style({ + 'text-transform': 'uppercase', + }); + + if (withLayer) { + block + .tspan(`Layer: ${zOrder}`) + .attr({ + dy: '1.25em', + x: 0, + }) + .addClass('cvat_canvas_text_layer'); + } + + if (withDimensions && ['rectangle', 'ellipse'].includes(state.shapeType)) { + let width = state.points[2] - state.points[0]; + let height = state.points[3] - state.points[1]; + + if (state.shapeType === 'ellipse') { + width *= 2; + height *= -2; + } + + block + .tspan(composeShapeDimensions(width, height, state.rotation)) + .attr({ + dy: '1.25em', + x: 0, + }) + .addClass('cvat_canvas_text_dimensions'); + } + if (withDescriptions) { + descriptions.forEach((desc: string, idx: number) => { + block + .tspan(`${desc}`) + .attr({ + dy: idx === 0 ? '1.25em' : '1em', + x: 0, + }) + .addClass('cvat_canvas_text_description'); + }); + } + if (withScore || withVotes) { + const parts = []; + if (withScore) { + parts.push(`Score: ${score.toFixed(2)}`); + } + if (withVotes) { + parts.push(`Votes: ${votes}`); + } + block + .tspan(parts.join(', ')) + .attr({ dy: '1.25em', x: 0 }) + .addClass('cvat_canvas_text_score'); + } + if (withAttr) { + Object.keys(attributes).forEach((attrID: string, idx: number) => { + const values = `${attributes[attrID] === undefinedAttrValue ? + '' : attributes[attrID]}`.split('\n'); + const parent = block.tspan(`${attrNames[attrID]}: `) + .attr({ attrID, dy: idx === 0 ? '1.25em' : '1em', x: 0 }) + .addClass('cvat_canvas_text_attribute'); + values.forEach((attrLine: string, index: number) => { + parent + .tspan(attrLine) + .attr({ + dy: index === 0 ? 0 : '1em', + }); + }); + }); + } + }) + .move(0, 0) + .attr({ 'data-client-id': state.clientID }) + .style({ 'font-size': textFontSize }) + .addClass('cvat_canvas_text'); + + return this.svgTexts[state.clientID]; + } + + private addRect(points: number[], state: any): SVG.Rect { + const [xtl, ytl, xbr, ybr] = points; + const rect = this.adoptedContent + .rect() + .size(xbr - xtl, ybr - ytl) + .attr({ + clientID: state.clientID, + 'color-rendering': 'optimizeQuality', + id: `cvat_canvas_shape_${state.clientID}`, + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'data-z-order': state.zOrder, + ...this.getShapeColorization(state), + }).move(xtl, ytl).addClass('cvat_canvas_shape'); + + if (state.rotation) { + rect.rotate(state.rotation); + } + + if (state.occluded) { + rect.addClass('cvat_canvas_shape_occluded'); + } + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + rect.addClass('cvat_canvas_hidden'); + } + + if (state.isGroundTruth) { + rect.addClass('cvat_canvas_ground_truth'); + } + + return rect; + } + + private addPolygon(points: string, state: any): SVG.Polygon { + const polygon = this.adoptedContent + .polygon(points) + .attr({ + clientID: state.clientID, + 'color-rendering': 'optimizeQuality', + id: `cvat_canvas_shape_${state.clientID}`, + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'data-z-order': state.zOrder, + ...this.getShapeColorization(state), + }).addClass('cvat_canvas_shape'); + + if (state.occluded) { + polygon.addClass('cvat_canvas_shape_occluded'); + } + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + polygon.addClass('cvat_canvas_hidden'); + } + + if (state.isGroundTruth) { + polygon.addClass('cvat_canvas_ground_truth'); + } + + return polygon; + } + + private addPolyline(points: string, state: any): SVG.PolyLine { + const polyline = this.adoptedContent + .polyline(points) + .attr({ + clientID: state.clientID, + 'color-rendering': 'optimizeQuality', + id: `cvat_canvas_shape_${state.clientID}`, + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'data-z-order': state.zOrder, + ...this.getShapeColorization(state), + }).addClass('cvat_canvas_shape'); + + if (state.occluded) { + polyline.addClass('cvat_canvas_shape_occluded'); + } + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + polyline.addClass('cvat_canvas_hidden'); + } + + if (state.isGroundTruth) { + polyline.addClass('cvat_canvas_ground_truth'); + } + + return polyline; + } + + private addCuboid(points: string, state: any): any { + const cube = (this.adoptedContent as any) + .cube(points) + .fill(state.color) + .attr({ + clientID: state.clientID, + 'color-rendering': 'optimizeQuality', + id: `cvat_canvas_shape_${state.clientID}`, + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'data-z-order': state.zOrder, + ...this.getShapeColorization(state), + }).addClass('cvat_canvas_shape'); + + if (state.occluded) { + cube.addClass('cvat_canvas_shape_occluded'); + } + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + cube.addClass('cvat_canvas_hidden'); + } + + if (state.isGroundTruth) { + cube.addClass('cvat_canvas_ground_truth'); + } + + return cube; + } + + private addMask(points: number[], state: any): SVG.Image { + const colorization = this.getShapeColorization(state); + const color = fabric.Color.fromHex(colorization.fill).getSource(); + const [left, top, right, bottom] = points.slice(-4); + const imageBitmap = RLEToImageData(color[0], color[1], color[2], points); + + const image = this.adoptedContent.image().attr({ + clientID: state.clientID, + 'color-rendering': 'optimizeQuality', + id: `cvat_canvas_shape_${state.clientID}`, + 'shape-rendering': 'geometricprecision', + 'data-z-order': state.zOrder, + // apply sqrt function to colorization to enhance displaying the mask on the canvas + opacity: Math.sqrt(colorization['fill-opacity']), + stroke: colorization.stroke, + }).addClass('cvat_canvas_shape'); + image.move(this.geometry.offset + left, this.geometry.offset + top); + + imageDataToDataURL( + imageBitmap, + right - left + 1, + bottom - top + 1, + (dataURL: string): void => { + const destroy = (): void => URL.revokeObjectURL(dataURL); + if (image.parent() !== null) { + image.loaded(destroy); + image.error(destroy); + image.load(dataURL); + } else { + destroy(); + } + }, + ); + + if (state.occluded) { + image.addClass('cvat_canvas_shape_occluded'); + } + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + image.addClass('cvat_canvas_hidden'); + } + + if (state.isGroundTruth) { + image.addClass('cvat_canvas_ground_truth'); + } + + return image; + } + + private addSkeleton(state: any): any { + const skeleton = (this.adoptedContent as any) + .group() + .attr({ + clientID: state.clientID, + 'color-rendering': 'optimizeQuality', + id: `cvat_canvas_shape_${state.clientID}`, + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'data-z-order': state.zOrder, + 'pointer-events': 'all', + ...this.getShapeColorization(state), + }).addClass('cvat_canvas_shape cvat_canvas_shape_skeleton') as SVG.G; + + const SVGElement = makeSVGFromTemplate(state.label.structure.svg); + + let [xtl, ytl, xbr, ybr] = [null, null, null, null]; + const svgElements: Record = {}; + const visibleElementIDs = this.getVisibleSkeletonElementIDs(state.clientID); + const visibleNodeIDs = new Set(); + const templateElements = Array.from(SVGElement.children()).filter((el: SVG.Element) => el.type === 'circle'); + for (let i = 0; i < state.elements.length; i++) { + const element = state.elements[i]; + if (visibleElementIDs && !visibleElementIDs.has(element.clientID)) { + continue; + } + + if (element.shapeType === 'points') { + const points: number[] = element.points as number[]; + const [cx, cy] = this.translateToCanvas(points); + + if (!element.outside) { + xtl = xtl === null ? cx : Math.min(xtl, cx); + ytl = ytl === null ? cy : Math.min(ytl, cy); + xbr = xbr === null ? cx : Math.max(xbr, cx); + ybr = ybr === null ? cy : Math.max(ybr, cy); + } + + const templateElement = templateElements.find((el: SVG.Circle) => el.attr('data-label-id') === element.label.id); + visibleNodeIDs.add(templateElement.attr('data-node-id')); + const circle = skeleton.circle() + .center(cx, cy) + .attr({ + id: `cvat_canvas_shape_${element.clientID}`, + r: this.configuration.controlPointsSize / this.geometry.scale, + 'color-rendering': 'optimizeQuality', + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'data-node-id': templateElement.attr('data-node-id'), + 'data-element-id': templateElement.attr('data-element-id'), + 'data-label-id': templateElement.attr('data-label-id'), + 'data-client-id': element.clientID, + ...this.getShapeColorization(element, { parentState: state }), + }).style({ + cursor: 'default', + }); + this.svgShapes[element.clientID] = circle; + if (element.occluded) { + circle.addClass('cvat_canvas_shape_occluded'); + } + + if (element.hidden || element.outside || this.isInnerHidden(element.clientID)) { + circle.addClass('cvat_canvas_hidden'); + } + + const mouseover = (e: MouseEvent): void => { + const locked = this.drawnStates[state.clientID].lock; + if (!locked && !e.ctrlKey && this.mode === Mode.IDLE) { + circle.attr({ + 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale, + }); + + const [x, y] = translateToSVG(this.content, [e.clientX, e.clientY]); + const event: CustomEvent = new CustomEvent('canvas.moved', { + bubbles: false, + cancelable: true, + detail: { + x: x - this.geometry.offset, + y: y - this.geometry.offset, + activatedElementID: element.clientID, + states: this.controller.objects, + }, + }); + + this.canvas.dispatchEvent(event); + } + }; + + const mousemove = (e: MouseEvent): void => { + if (this.mode === Mode.IDLE) { + // stop propagation to canvas where it calls another canvas.moved + // and does not allow to activate an element + e.stopPropagation(); + } + }; + + const mouseleave = (): void => { + circle.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + }); + }; + + const click = (e: MouseEvent): void => { + e.stopPropagation(); + this.canvas.dispatchEvent( + new CustomEvent('canvas.clicked', { + bubbles: false, + cancelable: true, + detail: { + state: element, + }, + }), + ); + }; + + circle.on('mouseover', mouseover); + circle.on('mouseleave', mouseleave); + circle.on('mousemove', mousemove); + circle.on('click', click); + circle.on('remove', () => { + circle.off('remove'); + circle.off('mouseover', mouseover); + circle.off('mouseleave', mouseleave); + circle.off('mousemove', mousemove); + circle.off('click', click); + }); + + svgElements[element.clientID] = circle; + } + } + + // if all elements were outside, set coordinates to zeros + xtl = xtl || 0; + ytl = ytl || 0; + xbr = xbr || 0; + ybr = ybr || 0; + + // apply bounding box margin + xtl -= consts.SKELETON_RECT_MARGIN; + ytl -= consts.SKELETON_RECT_MARGIN; + xbr += consts.SKELETON_RECT_MARGIN; + ybr += consts.SKELETON_RECT_MARGIN; + + skeleton.on('remove', () => { + Object.values(svgElements).forEach((element) => element.fire('remove')); + skeleton.off('remove'); + }); + + const wrappingRect = skeleton.rect(xbr - xtl, ybr - ytl).move(xtl, ytl).attr({ + fill: 'inherit', + 'fill-opacity': 0, + 'color-rendering': 'optimizeQuality', + 'shape-rendering': 'geometricprecision', + stroke: 'inherit', + 'stroke-width': 'inherit', + 'data-xtl': xtl, + 'data-ytl': ytl, + 'data-xbr': xbr, + 'data-ybr': ybr, + }).addClass('cvat_canvas_skeleton_wrapping_rect'); + + skeleton.node.prepend(wrappingRect.node); + setupSkeletonEdges(skeleton, SVGElement, visibleNodeIDs); + + if (state.occluded) { + skeleton.addClass('cvat_canvas_shape_occluded'); + } + + if (state.isGroundTruth) { + skeleton.addClass('cvat_canvas_ground_truth'); + } + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + skeleton.addClass('cvat_canvas_hidden'); + } + + (skeleton as any).selectize = (enabled: boolean) => { + this.selectize(enabled, wrappingRect); + const handler = wrappingRect.remember('_selectHandler'); + if (enabled && handler) { + this.adoptedContent.node.append(handler.nested.node); + handler.nested.attr('fill', skeleton.attr('fill')); + } + + return skeleton; + }; + + return skeleton; + } + + private setupPoints(basicPolyline: SVG.PolyLine, state: any | DrawnState): any { + this.selectize(true, basicPolyline); + + const group: SVG.G = basicPolyline + .remember('_selectHandler') + .nested.addClass('cvat_canvas_shape') + .attr({ + clientID: state.clientID, + id: `cvat_canvas_shape_${state.clientID}`, + 'data-polyline-id': basicPolyline.attr('id'), + 'data-z-order': state.zOrder, + }); + + group.on('click.canvas', (event: MouseEvent): void => { + // Need to redispatch the event on another element + basicPolyline.fire(new MouseEvent('click', event)); + // redispatch event to canvas to be able merge points clicking them + this.content.dispatchEvent(new MouseEvent('click', event)); + }); + + group.bbox = basicPolyline.bbox.bind(basicPolyline); + group.clone = basicPolyline.clone.bind(basicPolyline); + + return group; + } + + private addEllipse(points: string, state: any): SVG.Rect { + const [cx, cy, rightX, topY] = points.split(/[/,\s]/g).map((coord) => +coord); + const [rx, ry] = [rightX - cx, cy - topY]; + const rect = this.adoptedContent + .ellipse(rx * 2, ry * 2) + .attr({ + clientID: state.clientID, + 'color-rendering': 'optimizeQuality', + id: `cvat_canvas_shape_${state.clientID}`, + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'data-z-order': state.zOrder, + ...this.getShapeColorization(state), + }) + .center(cx, cy) + .addClass('cvat_canvas_shape'); + + if (state.rotation) { + rect.rotate(state.rotation); + } + + if (state.occluded) { + rect.addClass('cvat_canvas_shape_occluded'); + } + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + rect.addClass('cvat_canvas_hidden'); + } + + if (state.isGroundTruth) { + rect.addClass('cvat_canvas_ground_truth'); + } + + return rect; + } + + private addPoints(points: string, state: any): SVG.PolyLine { + const shape = this.adoptedContent + .polyline(points) + .attr({ + 'color-rendering': 'optimizeQuality', + 'pointer-events': 'none', + 'shape-rendering': 'geometricprecision', + 'stroke-width': 0, + ...this.getShapeColorization(state), + }).style({ + opacity: 0, + }); + + const group = this.setupPoints(shape, state); + + if (state.hidden || state.outside || this.isInnerHidden(state.clientID)) { + group.addClass('cvat_canvas_hidden'); + } + + if (state.occluded) { + group.addClass('cvat_canvas_shape_occluded'); + } + + if (state.isGroundTruth) { + group.addClass('cvat_canvas_ground_truth'); + } + + shape.remove = (): SVG.PolyLine => { + this.selectize(false, shape); + shape.constructor.prototype.remove.call(shape); + return shape; + }; + + return shape; + } +} diff --git a/cvat-canvas/src/typescript/consts.ts b/cvat-canvas/src/typescript/consts.ts new file mode 100644 index 0000000..1e39c13 --- /dev/null +++ b/cvat-canvas/src/typescript/consts.ts @@ -0,0 +1,59 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +const BASE_STROKE_WIDTH = 1.25; +const BASE_GRID_WIDTH = 2; +const BASE_POINT_SIZE = 4; +const TEXT_MARGIN = 10; +const SIZE_THRESHOLD = 1; +const POINTS_STROKE_WIDTH = 1; +const POINTS_SELECTED_STROKE_WIDTH = 4; +const MIN_EDGE_LENGTH = 3; +const CUBOID_ACTIVE_EDGE_STROKE_WIDTH = 2.5; +const CUBOID_UNACTIVE_EDGE_STROKE_WIDTH = 1.75; +const UNDEFINED_ATTRIBUTE_VALUE = '__undefined__'; +const ARROW_PATH = 'M13.162 6.284L.682.524a.483.483 0 0 0-.574.134.477.477 0 ' + + '0 0-.012.59L4.2 6.72.096 12.192a.479.479 0 0 0 .585.724l12.48-5.76a.48.48 0 0 0 0-.872z'; +const BASE_PATTERN_SIZE = 5; +const SNAP_TO_ANGLE_RESIZE_DEFAULT = 0.1; +const SNAP_TO_ANGLE_RESIZE_SHIFT = 15; +const MINIMUM_TEXT_FONT_SIZE = 8; +const SKELETON_RECT_MARGIN = 20; + +const DEFAULT_SHAPE_TEXT_SIZE = 12; +const DEFAULT_SHAPE_TEXT_CONTENT = 'id,label,attributes,source,descriptions'; +const DEFAULT_SHAPE_TEXT_POSITION: 'auto' | 'center' = 'auto'; +const DEFAULT_UNDEFINED_ATTR_VALUE = '__undefined__'; + +const CONFLICT_COLOR = '#ff4800'; +const WARNING_COLOR = '#ff7301'; +const SHADED_COLOR = '#808080'; + +export default { + BASE_STROKE_WIDTH, + BASE_GRID_WIDTH, + BASE_POINT_SIZE, + TEXT_MARGIN, + SIZE_THRESHOLD, + POINTS_STROKE_WIDTH, + POINTS_SELECTED_STROKE_WIDTH, + MIN_EDGE_LENGTH, + CUBOID_ACTIVE_EDGE_STROKE_WIDTH, + CUBOID_UNACTIVE_EDGE_STROKE_WIDTH, + UNDEFINED_ATTRIBUTE_VALUE, + ARROW_PATH, + BASE_PATTERN_SIZE, + SNAP_TO_ANGLE_RESIZE_DEFAULT, + SNAP_TO_ANGLE_RESIZE_SHIFT, + DEFAULT_SHAPE_TEXT_SIZE, + DEFAULT_SHAPE_TEXT_CONTENT, + DEFAULT_SHAPE_TEXT_POSITION, + DEFAULT_UNDEFINED_ATTR_VALUE, + MINIMUM_TEXT_FONT_SIZE, + SKELETON_RECT_MARGIN, + CONFLICT_COLOR, + WARNING_COLOR, + SHADED_COLOR, +}; diff --git a/cvat-canvas/src/typescript/crosshair.ts b/cvat-canvas/src/typescript/crosshair.ts new file mode 100644 index 0000000..5b7a55d --- /dev/null +++ b/cvat-canvas/src/typescript/crosshair.ts @@ -0,0 +1,81 @@ +// Copyright (C) 2020-2022 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import consts from './consts'; + +export default class Crosshair { + private x: SVG.Line | null; + private y: SVG.Line | null; + private canvas: SVG.Container | null; + + public constructor() { + this.x = null; + this.y = null; + this.canvas = null; + } + + public show(canvas: SVG.Container, x: number, y: number, scale: number): void { + if (this.canvas && this.canvas !== canvas) { + if (this.x) this.x.remove(); + if (this.y) this.y.remove(); + this.x = null; + this.y = null; + } + + if (this.x && this.y) { + this.move(x, y); + return; + } + + this.canvas = canvas; + this.x = this.canvas + .line(0, y, this.canvas.node.clientWidth, y) + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), + }) + .addClass('cvat_canvas_crosshair'); + + this.y = this.canvas + .line(x, 0, x, this.canvas.node.clientHeight) + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), + }) + .addClass('cvat_canvas_crosshair'); + } + + public hide(): void { + if (this.x) { + this.x.remove(); + this.x = null; + } + + if (this.y) { + this.y.remove(); + this.y = null; + } + + this.canvas = null; + } + + public move(x: number, y: number): void { + if (this.x) { + this.x.attr({ y1: y, y2: y }); + } + + if (this.y) { + this.y.attr({ x1: x, x2: x }); + } + } + + public scale(scale: number): void { + if (this.x) { + this.x.attr('stroke-width', consts.BASE_STROKE_WIDTH / (2 * scale)); + } + + if (this.y) { + this.y.attr('stroke-width', consts.BASE_STROKE_WIDTH / (2 * scale)); + } + } +} diff --git a/cvat-canvas/src/typescript/cuboid.ts b/cvat-canvas/src/typescript/cuboid.ts new file mode 100644 index 0000000..19addc0 --- /dev/null +++ b/cvat-canvas/src/typescript/cuboid.ts @@ -0,0 +1,484 @@ +// Copyright (C) 2021-2022 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import consts from './consts'; +import { Point } from './shared'; + +export enum Orientation { + LEFT = 'left', + RIGHT = 'right', +} + +export function intersection(p1: Point, p2: Point, p3: Point, p4: Point): Point | null { + // Check if none of the lines are of length 0 + const { x: x1, y: y1 } = p1; + const { x: x2, y: y2 } = p2; + const { x: x3, y: y3 } = p3; + const { x: x4, y: y4 } = p4; + + if ((x1 === x2 && y1 === y2) || (x3 === x4 && y3 === y4)) { + return null; + } + + const denominator = ((y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1)); + + // Lines are parallel + if (Math.abs(denominator) < Number.EPSILON) { + return null; + } + + const ua = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator; + + // Return a object with the x and y coordinates of the intersection + return { x: x1 + ua * (x2 - x1), y: y1 + ua * (y2 - y1) }; +} + +export class Equation { + private a: number; + private b: number; + private c: number; + + public constructor(p1: Point, p2: Point) { + this.a = p1.y - p2.y; + this.b = p2.x - p1.x; + this.c = this.b * p1.y + this.a * p1.x; + } + + // get the line equation in actual coordinates + public getY(x: number): number { + return (this.c - this.a * x) / this.b; + } +} + +export class Figure { + private indices: number[]; + private allPoints: Point[]; + + public constructor(indices: number[], points: Point[]) { + this.indices = indices; + this.allPoints = points; + } + + public get points(): Point[] { + const points = []; + for (const index of this.indices) { + points.push(this.allPoints[index]); + } + return points; + } + + // sets the point for a given edge, points must be given in + // array form in the same ordering as the getter + // if you only need to update a subset of the points, + // simply put null for the points you want to keep + public set points(newPoints) { + const oldPoints = this.allPoints; + for (let i = 0; i < newPoints.length; i += 1) { + if (newPoints[i] !== null) { + oldPoints[this.indices[i]] = { x: newPoints[i].x, y: newPoints[i].y }; + } + } + } +} + +export class Edge extends Figure { + public getEquation(): Equation { + return new Equation(this.points[0], this.points[1]); + } +} + +export class CuboidModel { + public points: Point[]; + private fr: Edge; + private fl: Edge; + private dr: Edge; + private dl: Edge; + private ft: Edge; + private rt: Edge; + private lt: Edge; + private dt: Edge; + private fb: Edge; + private rb: Edge; + private lb: Edge; + private db: Edge; + public edgeList: Edge[]; + private front: Figure; + private right: Figure; + private dorsal: Figure; + private left: Figure; + private top: Figure; + private bot: Figure; + public facesList: Figure[]; + public vpl: Point | null; + public vpr: Point | null; + public orientation: Orientation; + + public constructor(points?: Point[]) { + this.points = points; + this.initEdges(); + this.initFaces(); + this.updateVanishingPoints(false); + this.buildBackEdge(false); + this.updatePoints(); + this.updateOrientation(); + } + + public getPoints(): Point[] { + return this.points; + } + + public setPoints(points: (Point | null)[]): void { + points.forEach((point: Point | null, i: number): void => { + if (point !== null) { + this.points[i].x = point.x; + this.points[i].y = point.y; + } + }); + } + + public updateOrientation(): void { + if (this.dl.points[0].x > this.fl.points[0].x) { + this.orientation = Orientation.LEFT; + } else { + this.orientation = Orientation.RIGHT; + } + } + + public updatePoints(): void { + // making sure that the edges are vertical + this.fr.points[0].x = this.fr.points[1].x; + this.fl.points[0].x = this.fl.points[1].x; + this.dr.points[0].x = this.dr.points[1].x; + this.dl.points[0].x = this.dl.points[1].x; + } + + public computeSideEdgeConstraints(edge: any): any { + const midLength = this.fr.points[1].y - this.fr.points[0].y - 1; + + const minY = edge.points[1].y - midLength; + const maxY = edge.points[0].y + midLength; + + const y1 = edge.points[0].y; + const y2 = edge.points[1].y; + + const miny1 = y2 - midLength; + const maxy1 = y2 - consts.MIN_EDGE_LENGTH; + + const miny2 = y1 + consts.MIN_EDGE_LENGTH; + const maxy2 = y1 + midLength; + + return { + constraint: { + minY, + maxY, + }, + y1Range: { + max: maxy1, + min: miny1, + }, + y2Range: { + max: maxy2, + min: miny2, + }, + }; + } + + // boolean value parameter controls which edges should be used to recalculate vanishing points + private updateVanishingPoints(buildright: boolean): void { + let leftEdge = []; + let rightEdge = []; + let midEdge = []; + if (buildright) { + leftEdge = this.fr.points; + rightEdge = this.dl.points; + midEdge = this.fl.points; + } else { + leftEdge = this.fl.points; + rightEdge = this.dr.points; + midEdge = this.fr.points; + } + + this.vpl = intersection(leftEdge[0], midEdge[0], leftEdge[1], midEdge[1]); + this.vpr = intersection(rightEdge[0], midEdge[0], rightEdge[1], midEdge[1]); + if (this.vpl === null) { + // shift the edge slightly to avoid edge case + leftEdge[0].y -= 0.001; + leftEdge[0].x += 0.001; + leftEdge[1].x += 0.001; + this.vpl = intersection(leftEdge[0], midEdge[0], leftEdge[1], midEdge[1]); + } + if (this.vpr === null) { + // shift the edge slightly to avoid edge case + rightEdge[0].y -= 0.001; + rightEdge[0].x -= 0.001; + rightEdge[1].x -= 0.001; + this.vpr = intersection(leftEdge[0], midEdge[0], leftEdge[1], midEdge[1]); + } + } + + private initEdges(): void { + this.fl = new Edge([0, 1], this.points); + this.fr = new Edge([2, 3], this.points); + this.dr = new Edge([4, 5], this.points); + this.dl = new Edge([6, 7], this.points); + + this.ft = new Edge([0, 2], this.points); + this.lt = new Edge([0, 6], this.points); + this.rt = new Edge([2, 4], this.points); + this.dt = new Edge([6, 4], this.points); + + this.fb = new Edge([1, 3], this.points); + this.lb = new Edge([1, 7], this.points); + this.rb = new Edge([3, 5], this.points); + this.db = new Edge([7, 5], this.points); + + this.edgeList = [ + this.fl, + this.fr, + this.dl, + this.dr, + this.ft, + this.lt, + this.rt, + this.dt, + this.fb, + this.lb, + this.rb, + this.db, + ]; + } + + private initFaces(): void { + this.front = new Figure([0, 1, 3, 2], this.points); + this.right = new Figure([2, 3, 5, 4], this.points); + this.dorsal = new Figure([4, 5, 7, 6], this.points); + this.left = new Figure([6, 7, 1, 0], this.points); + this.top = new Figure([0, 2, 4, 6], this.points); + this.bot = new Figure([1, 3, 5, 7], this.points); + + this.facesList = [this.front, this.right, this.dorsal, this.left]; + } + + private buildBackEdge(buildright: boolean): void { + this.updateVanishingPoints(buildright); + let leftPoints = []; + let rightPoints = []; + + let topIndex = 0; + let botIndex = 0; + + if (buildright) { + leftPoints = this.dl.points; + rightPoints = this.fr.points; + topIndex = 4; + botIndex = 5; + } else { + leftPoints = this.dr.points; + rightPoints = this.fl.points; + topIndex = 6; + botIndex = 7; + } + + const vpLeft = this.vpl; + const vpRight = this.vpr; + + let p1 = intersection(vpLeft, leftPoints[0], vpRight, rightPoints[0]); + let p2 = intersection(vpLeft, leftPoints[1], vpRight, rightPoints[1]); + + if (p1 === null) { + p1 = { x: p2.x, y: vpLeft.y }; + } else if (p2 === null) { + p2 = { x: p1.x, y: vpLeft.y }; + } + + this.points[topIndex] = { x: p1.x, y: p1.y }; + this.points[botIndex] = { x: p2.x, y: p2.y }; + + // Making sure that the vertical edges stay vertical + this.updatePoints(); + } +} + +function sortPointsClockwise(points: any[]): any[] { + points.sort((a, b): number => a.y - b.y); + // Get center y + const cy = (points[0].y + points[points.length - 1].y) / 2; + + // Sort from right to left + points.sort((a, b): number => b.x - a.x); + + // Get center x + const cx = (points[0].x + points[points.length - 1].x) / 2; + + // Center point + const center = { + x: cx, + y: cy, + }; + + // Starting angle used to reference other angles + let startAng: number | undefined; + points.forEach((point): void => { + let ang = Math.atan2(point.y - center.y, point.x - center.x); + if (!startAng) { + startAng = ang; + // ensure that all points are clockwise of the start point + } else if (ang < startAng) { + ang += Math.PI * 2; + } + // eslint-disable-next-line no-param-reassign + point.angle = ang; // add the angle to the point + }); + + // first sort clockwise + points.sort((a, b): number => a.angle - b.angle); + return points.reverse(); +} + +function setupCuboidPoints(points: Point[]): any[] { + let left; + let right; + let left2; + let right2; + let p1; + let p2; + let p3; + let p4; + + const height = Math.abs(points[0].x - points[1].x) < Math.abs(points[1].x - points[2].x) ? + Math.abs(points[1].y - points[0].y) : Math.abs(points[1].y - points[2].y); + + // separate into left and right point + // we pick the first and third point because we know assume they will be on + // opposite corners + if (points[0].x < points[2].x) { + [left, , right] = points; + } else { + [right, , left] = points; + } + + // get other 2 points using the given height + if (left.y < right.y) { + left2 = { x: left.x, y: left.y + height }; + right2 = { x: right.x, y: right.y - height }; + } else { + left2 = { x: left.x, y: left.y - height }; + right2 = { x: right.x, y: right.y + height }; + } + + // get the vector for the last point relative to the previous point + const vec = { + x: points[3].x - points[2].x, + y: points[3].y - points[2].y, + }; + + if (left.y < left2.y) { + p1 = left; + p2 = left2; + } else { + p1 = left2; + p2 = left; + } + + if (right.y < right2.y) { + p3 = right; + p4 = right2; + } else { + p3 = right2; + p4 = right; + } + + const p5 = { x: p3.x + vec.x, y: p3.y + vec.y + 0.1 }; + const p6 = { x: p4.x + vec.x, y: p4.y + vec.y - 0.1 }; + const p7 = { x: p1.x + vec.x, y: p1.y + vec.y + 0.1 }; + const p8 = { x: p2.x + vec.x, y: p2.y + vec.y - 0.1 }; + + p1.y += 0.1; + return [p1, p2, p3, p4, p5, p6, p7, p8]; +} + +export function cuboidFrom4Points(flattenedPoints: any[]): any[] { + const points: Point[] = []; + for (let i = 0; i < 4; i++) { + const [x, y] = flattenedPoints.slice(i * 2, i * 2 + 2); + points.push({ x, y }); + } + const unsortedPlanePoints = points.slice(0, 3); + function rotate(array: any[], times: number): void { + let t = times; + while (t--) { + const temp = array.shift(); + array.push(temp); + } + } + + const plane2 = { + p1: points[0], + p2: points[0], + p3: points[0], + p4: points[0], + }; + + // completing the plane + const vector = { + x: points[2].x - points[1].x, + y: points[2].y - points[1].y, + }; + + // sorting the first plane + unsortedPlanePoints.push({ + x: points[0].x + vector.x, + y: points[0].y + vector.y, + }); + const sortedPlanePoints = sortPointsClockwise(unsortedPlanePoints); + let leftIndex = 0; + for (let i = 0; i < 4; i++) { + leftIndex = sortedPlanePoints[i].x < sortedPlanePoints[leftIndex].x ? i : leftIndex; + } + rotate(sortedPlanePoints, leftIndex); + const plane1 = { + p1: sortedPlanePoints[0], + p2: sortedPlanePoints[1], + p3: sortedPlanePoints[2], + p4: sortedPlanePoints[3], + }; + + const vec = { + x: points[3].x - points[2].x, + y: points[3].y - points[2].y, + }; + // determine the orientation + const angle = Math.atan2(vec.y, vec.x); + + // making the other plane + plane2.p1 = { x: plane1.p1.x + vec.x, y: plane1.p1.y + vec.y }; + plane2.p2 = { x: plane1.p2.x + vec.x, y: plane1.p2.y + vec.y }; + plane2.p3 = { x: plane1.p3.x + vec.x, y: plane1.p3.y + vec.y }; + plane2.p4 = { x: plane1.p4.x + vec.x, y: plane1.p4.y + vec.y }; + + let cuboidPoints; + // right + if (Math.abs(angle) < Math.PI / 2 - 0.1) { + cuboidPoints = setupCuboidPoints(points); + // left + } else if (Math.abs(angle) > Math.PI / 2 + 0.1) { + cuboidPoints = setupCuboidPoints(points); + // down + } else if (angle > 0) { + cuboidPoints = [plane1.p1, plane2.p1, plane1.p2, plane2.p2, plane1.p3, plane2.p3, plane1.p4, plane2.p4]; + cuboidPoints[0].y += 0.1; + cuboidPoints[4].y += 0.1; + // up + } else { + cuboidPoints = [plane2.p1, plane1.p1, plane2.p2, plane1.p2, plane2.p3, plane1.p3, plane2.p4, plane1.p4]; + cuboidPoints[0].y += 0.1; + cuboidPoints[4].y += 0.1; + } + + return cuboidPoints.reduce((arr: number[], point: any): number[] => { + arr.push(point.x); + arr.push(point.y); + return arr; + }, []); +} diff --git a/cvat-canvas/src/typescript/drawHandler.ts b/cvat-canvas/src/typescript/drawHandler.ts new file mode 100644 index 0000000..d3c2584 --- /dev/null +++ b/cvat-canvas/src/typescript/drawHandler.ts @@ -0,0 +1,1476 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import 'svg.draw.js'; +import { CIRCLE_STROKE } from './svg.patch'; + +import { AutoborderHandler } from './autoborderHandler'; +import { + translateToSVG, + displayShapeSize, + ShapeSizeElement, + stringifyPoints, + BBox, + Box, + Point, + readPointsFromShape, + clamp, + translateToCanvas, + computeWrappingBox, + makeSVGFromTemplate, + setupSkeletonEdges, + translateFromCanvas, + DrawnState, + applySnapToShapePoint, +} from './shared'; +import Crosshair from './crosshair'; +import consts from './consts'; +import { + DrawData, Geometry, RectDrawingMethod, Configuration, CuboidDrawingMethod, +} from './canvasModel'; + +import { cuboidFrom4Points, intersection } from './cuboid'; + +export interface DrawHandler { + configure(configuration: Configuration): void; + draw(drawData: DrawData, geometry: Geometry): void; + transform(geometry: Geometry): void; + cancel(): void; +} + +interface FinalCoordinates { + points: number[]; + box: Box; +} + +function checkConstraint(shapeType: string, points: number[], box: Box | null = null): boolean { + if (shapeType === 'rectangle') { + const [xtl, ytl, xbr, ybr] = points; + const [width, height] = [xbr - xtl, ybr - ytl]; + return width >= consts.SIZE_THRESHOLD && height >= consts.SIZE_THRESHOLD; + } + + if (shapeType === 'polygon') { + const [width, height] = [box.xbr - box.xtl, box.ybr - box.ytl]; + return (width >= consts.SIZE_THRESHOLD || height > consts.SIZE_THRESHOLD) && points.length >= 3 * 2; + } + + if (shapeType === 'polyline') { + const [width, height] = [box.xbr - box.xtl, box.ybr - box.ytl]; + return (width >= consts.SIZE_THRESHOLD || height >= consts.SIZE_THRESHOLD) && points.length >= 2 * 2; + } + + if (shapeType === 'points') { + return points.length > 2 || (points.length === 2 && points[0] !== 0 && points[1] !== 0); + } + + if (shapeType === 'ellipse') { + const [width, height] = [(points[2] - points[0]) * 2, (points[1] - points[3]) * 2]; + return width >= consts.SIZE_THRESHOLD && height > consts.SIZE_THRESHOLD; + } + + if (shapeType === 'cuboid') { + return points.length === 4 * 2 || points.length === 8 * 2 || + (points.length === 2 * 2 && + (points[2] - points[0]) >= consts.SIZE_THRESHOLD && + (points[3] - points[1]) >= consts.SIZE_THRESHOLD + ); + } + + if (shapeType === 'skeleton') { + const [xtl, ytl, xbr, ybr] = points; + const [width, height] = [xbr - xtl, ybr - ytl]; + return width >= consts.SIZE_THRESHOLD || height >= consts.SIZE_THRESHOLD; + } + + return false; +} + +export class DrawHandlerImpl implements DrawHandler { + // callback is used to notify about creating new shape + private onDrawDoneDefault: ( + data: object | null, + duration?: number, + continueDraw?: boolean, + prevDrawData?: DrawData, + ) => void; + private startTimestamp: number; + private canvas: SVG.Container; + private text: SVG.Container; + private cursorPosition: { + x: number; + y: number; + }; + private crosshair: Crosshair; + private drawData: DrawData | null; + private geometry: Geometry; + private configuration: Configuration; + private autoborderHandler: AutoborderHandler; + private autobordersEnabled: boolean; + private controlPointsSize: number; + private selectedShapeOpacity: number; + private outlinedBorders: string; + private isHidden: boolean; + private getDrawnStates: (() => Record) | null; + private isCtrlKeyDown: (() => boolean) | null; + + // we should use any instead of SVG.Shape because svg plugins cannot change declared interface + // so, methods like draw() just undefined for SVG.Shape, but nevertheless they exist + private drawInstance: any; + private initialized: boolean; + private canceled: boolean; + private pointsGroup: SVG.G | null; + private shapeSizeElement: ShapeSizeElement | null; + + private getFinalEllipseCoordinates(points: number[], fitIntoFrame: boolean): number[] { + const { offset } = this.geometry; + const [cx, cy, rightX, topY] = points.map((coord: number) => coord - offset); + const [rx, ry] = [rightX - cx, cy - topY]; + const frameWidth = this.geometry.image.width; + const frameHeight = this.geometry.image.height; + const [fitCX, fitCY] = fitIntoFrame ? + [clamp(cx, 0, frameWidth), clamp(cy, 0, frameHeight)] : [cx, cy]; + const [fitRX, fitRY] = fitIntoFrame ? + [Math.min(rx, frameWidth - cx, cx), Math.min(ry, frameHeight - cy, cy)] : [rx, ry]; + return [fitCX, fitCY, fitCX + fitRX, fitCY - fitRY]; + } + + private getFinalRectCoordinates(points: number[], fitIntoFrame: boolean): number[] { + const frameWidth = this.geometry.image.width; + const frameHeight = this.geometry.image.height; + const { offset } = this.geometry; + + let [xtl, ytl, xbr, ybr] = points.map((coord: number): number => coord - offset); + + if (fitIntoFrame) { + xtl = Math.min(Math.max(xtl, 0), frameWidth); + xbr = Math.min(Math.max(xbr, 0), frameWidth); + ytl = Math.min(Math.max(ytl, 0), frameHeight); + ybr = Math.min(Math.max(ybr, 0), frameHeight); + } + + return [xtl, ytl, xbr, ybr]; + } + + private getFinalPolyshapeCoordinates(targetPoints: number[], fitIntoFrame: boolean): FinalCoordinates { + const { offset } = this.geometry; + let points = targetPoints.map((coord: number): number => coord - offset); + const box = { + xtl: Number.MAX_SAFE_INTEGER, + ytl: Number.MAX_SAFE_INTEGER, + xbr: Number.MIN_SAFE_INTEGER, + ybr: Number.MIN_SAFE_INTEGER, + }; + + const frameWidth = this.geometry.image.width; + const frameHeight = this.geometry.image.height; + + enum Direction { + Horizontal, + Vertical, + } + + function isBetween(x1: number, x2: number, c: number): boolean { + return c >= Math.min(x1, x2) && c <= Math.max(x1, x2); + } + + const isInsideFrame = (p: Point, direction: Direction): boolean => { + if (direction === Direction.Horizontal) { + return isBetween(0, frameWidth, p.x); + } + return isBetween(0, frameHeight, p.y); + }; + + const findInersection = (p1: Point, p2: Point, p3: Point, p4: Point): number[] => { + const intersectionPoint = intersection(p1, p2, p3, p4); + if ( + intersectionPoint && + isBetween(p1.x, p2.x, intersectionPoint.x) && + isBetween(p1.y, p2.y, intersectionPoint.y) + ) { + return [intersectionPoint.x, intersectionPoint.y]; + } + return []; + }; + + const findIntersectionsWithFrameBorders = (p1: Point, p2: Point, direction: Direction): number[] => { + const resultPoints = []; + const leftLine = [ + { x: 0, y: 0 }, + { x: 0, y: frameHeight }, + ]; + const topLine = [ + { x: frameWidth, y: 0 }, + { x: 0, y: 0 }, + ]; + const rightLine = [ + { x: frameWidth, y: frameHeight }, + { x: frameWidth, y: 0 }, + ]; + const bottomLine = [ + { x: 0, y: frameHeight }, + { x: frameWidth, y: frameHeight }, + ]; + + if (direction === Direction.Horizontal) { + resultPoints.push(...findInersection(p1, p2, leftLine[0], leftLine[1])); + resultPoints.push(...findInersection(p1, p2, rightLine[0], rightLine[1])); + } else { + resultPoints.push(...findInersection(p1, p2, bottomLine[0], bottomLine[1])); + resultPoints.push(...findInersection(p1, p2, topLine[0], topLine[1])); + } + + if (resultPoints.length === 4) { + if ( + (p1.x === p2.x || Math.sign(resultPoints[0] - resultPoints[2]) !== Math.sign(p1.x - p2.x)) && + (p1.y === p2.y || Math.sign(resultPoints[1] - resultPoints[3]) !== Math.sign(p1.y - p2.y)) + ) { + [resultPoints[0], resultPoints[2]] = [resultPoints[2], resultPoints[0]]; + [resultPoints[1], resultPoints[3]] = [resultPoints[3], resultPoints[1]]; + } + } + return resultPoints; + }; + + const crop = (shapePoints: number[], direction: Direction): number[] => { + const resultPoints = []; + const isPolyline = this.drawData.shapeType === 'polyline'; + const isPolygon = this.drawData.shapeType === 'polygon'; + + for (let i = 0; i < shapePoints.length - 1; i += 2) { + const curPoint = { x: shapePoints[i], y: shapePoints[i + 1] }; + if (isInsideFrame(curPoint, direction)) { + resultPoints.push(shapePoints[i], shapePoints[i + 1]); + } + const isLastPoint = i === shapePoints.length - 2; + if (isLastPoint && (isPolyline || (isPolygon && shapePoints.length === 4))) { + break; + } + const nextPoint = isLastPoint ? + { x: shapePoints[0], y: shapePoints[1] } : + { x: shapePoints[i + 2], y: shapePoints[i + 3] }; + const intersectionPoints = findIntersectionsWithFrameBorders(curPoint, nextPoint, direction); + if (intersectionPoints.length !== 0) { + resultPoints.push(...intersectionPoints); + } + } + return resultPoints; + }; + + if (fitIntoFrame) { + points = crop(points, Direction.Horizontal); + points = crop(points, Direction.Vertical); + } + + for (let i = 0; i < points.length - 1; i += 2) { + box.xtl = Math.min(box.xtl, points[i]); + box.ytl = Math.min(box.ytl, points[i + 1]); + box.xbr = Math.max(box.xbr, points[i]); + box.ybr = Math.max(box.ybr, points[i + 1]); + } + + return { + points, + box, + }; + } + + private getFinalCuboidCoordinates(targetPoints: number[]): FinalCoordinates { + const { offset } = this.geometry; + let points = targetPoints; + + const box = { + xtl: Number.MAX_SAFE_INTEGER, + ytl: Number.MAX_SAFE_INTEGER, + xbr: Number.MIN_SAFE_INTEGER, + ybr: Number.MIN_SAFE_INTEGER, + }; + + const frameWidth = this.geometry.image.width; + const frameHeight = this.geometry.image.height; + + const cuboidOffsets = []; + const minCuboidOffset = { + d: Number.MAX_SAFE_INTEGER, + dx: 0, + dy: 0, + }; + + for (let i = 0; i < points.length - 1; i += 2) { + const [x, y] = points.slice(i); + + if (x >= offset && x <= offset + frameWidth && y >= offset && y <= offset + frameHeight) continue; + + let xOffset = 0; + let yOffset = 0; + + if (x < offset) { + xOffset = offset - x; + } else if (x > offset + frameWidth) { + xOffset = offset + frameWidth - x; + } + + if (y < offset) { + yOffset = offset - y; + } else if (y > offset + frameHeight) { + yOffset = offset + frameHeight - y; + } + + cuboidOffsets.push([xOffset, yOffset]); + } + + if (cuboidOffsets.length === points.length / 2) { + cuboidOffsets.forEach((offsetCoords: number[]): void => { + const dx = offsetCoords[0] ** 2; + const dy = offsetCoords[1] ** 2; + if (Math.sqrt(dx + dy) < minCuboidOffset.d) { + minCuboidOffset.d = Math.sqrt(dx + dy); + [minCuboidOffset.dx, minCuboidOffset.dy] = offsetCoords; + } + }); + + points = points.map((coord: number, i: number): number => { + if (i % 2) { + return coord + minCuboidOffset.dy; + } + return coord + minCuboidOffset.dx; + }); + } + + points.forEach((coord: number, i: number): number => { + if (i % 2 === 0) { + box.xtl = Math.min(box.xtl, coord); + box.xbr = Math.max(box.xbr, coord); + } else { + box.ytl = Math.min(box.ytl, coord); + box.ybr = Math.max(box.ybr, coord); + } + + return coord; + }); + + return { + points: points.map((coord: number): number => coord - offset), + box, + }; + } + + private addCrosshair(): void { + const { x, y } = this.cursorPosition; + this.crosshair.show(this.canvas, x, y, this.geometry.scale); + } + + private removeCrosshair(): void { + this.crosshair.hide(); + } + + private onDrawDone(...args: any[]): void { + if (this.drawData.onDrawDone) { + this.drawData.onDrawDone.call(this, ...args); + return; + } + + this.onDrawDoneDefault.call(this, ...args); + } + + private release(): void { + if (!this.initialized) { + // prevents recursive calls + return; + } + + this.autoborderHandler.autoborder(false); + this.initialized = false; + this.canvas.off('mousedown.draw'); + this.canvas.off('mousemove.draw'); + + // Draw plugin in some cases isn't activated + // For example when draw from initialState + // Or when no drawn points, but we call cancel() drawing + // We check if it is activated with remember function + if (this.drawInstance.remember('_paintHandler')) { + if (['polygon', 'polyline', 'points'].includes(this.drawData.shapeType) || + (this.drawData.shapeType === 'cuboid' && + this.drawData.cuboidDrawingMethod === CuboidDrawingMethod.CORNER_POINTS)) { + // Check for unsaved drawn shapes + this.drawInstance.draw('done'); + } + // Clear drawing + this.drawInstance.draw('stop'); + } else { + this.onDrawDone(null); + if (this.drawInstance && this.drawData.shapeType === 'ellipse' && !this.drawData.initialState) { + this.drawInstance.fire('drawstop'); + } + } + + if (this.pointsGroup) { + this.pointsGroup.remove(); + this.pointsGroup = null; + } + + this.drawInstance.off(); + this.drawInstance.remove(); + this.drawInstance = null; + + if (this.shapeSizeElement) { + this.shapeSizeElement.rm(); + this.shapeSizeElement = null; + } + + if (this.crosshair) { + this.removeCrosshair(); + } + } + + private initDrawing(): void { + if (this.drawData.crosshair) { + this.addCrosshair(); + } + } + + private drawBox(): void { + this.drawInstance = this.canvas.rect(); + this.drawInstance + .on('drawstop', (e: Event): void => { + const points = readPointsFromShape((e.target as any as { instance: SVG.Rect }).instance); + const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(points, true); + const { shapeType, redraw: clientID } = this.drawData; + + if (this.canceled) { + return; + } + + this.release(); + if (checkConstraint('rectangle', [xtl, ytl, xbr, ybr])) { + this.onDrawDone({ + clientID, + shapeType, + points: [xtl, ytl, xbr, ybr], + }, + Date.now() - this.startTimestamp); + } else { + this.onDrawDone(null); + } + }) + .on('drawupdate', (): void => { + this.shapeSizeElement.update(this.drawInstance); + }) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }); + } + + private drawEllipse(): void { + this.drawInstance = (this.canvas as any).ellipse() + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }); + + const initialPoint: { + x: number; + y: number; + } = { + x: null, + y: null, + }; + + this.canvas.on('mousedown.draw', (e: MouseEvent): void => { + if (e.button === 0 && !e.altKey) { + if (initialPoint.x === null || initialPoint.y === null) { + const translated = translateToSVG(this.canvas.node as any as SVGSVGElement, [e.clientX, e.clientY]); + [initialPoint.x, initialPoint.y] = translated; + } else { + this.drawInstance.fire('drawstop'); + } + } + }); + + this.canvas.on('mousemove.draw', (e: MouseEvent): void => { + if (initialPoint.x !== null && initialPoint.y !== null) { + const translated = translateToSVG(this.canvas.node as any as SVGSVGElement, [e.clientX, e.clientY]); + const rx = Math.abs(translated[0] - initialPoint.x) / 2; + const ry = Math.abs(translated[1] - initialPoint.y) / 2; + const cx = initialPoint.x + rx * Math.sign(translated[0] - initialPoint.x); + const cy = initialPoint.y + ry * Math.sign(translated[1] - initialPoint.y); + this.drawInstance.center(cx, cy); + this.drawInstance.radius(rx, ry); + this.shapeSizeElement.update(this.drawInstance); + } + }); + + this.drawInstance.on('drawstop', () => { + this.drawInstance.off('drawstop'); + const points = this.getFinalEllipseCoordinates(readPointsFromShape(this.drawInstance), false); + const { shapeType, redraw: clientID } = this.drawData; + + if (this.canceled) { + return; + } + + this.release(); + if (checkConstraint('ellipse', points)) { + this.onDrawDone( + { + clientID, + shapeType, + points, + }, + Date.now() - this.startTimestamp, + ); + } else { + this.onDrawDone(null); + } + }); + } + + private drawBoxBy4Points(): void { + let numberOfPoints = 0; + this.drawInstance = (this.canvas as any) + .polygon() + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': 0, + opacity: 0, + }) + .on('drawstart', (): void => { + // init numberOfPoints as one on drawstart + numberOfPoints = 1; + }) + .on('drawpoint', (e: CustomEvent): void => { + // increase numberOfPoints by one on drawpoint + numberOfPoints += 1; + + // finish if numberOfPoints are exactly four + if (numberOfPoints === 4) { + const bbox = (e.target as SVGPolylineElement).getBBox(); + const points = [bbox.x, bbox.y, bbox.x + bbox.width, bbox.y + bbox.height]; + const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(points, true); + const { shapeType, redraw: clientID } = this.drawData; + this.cancel(); + + if (checkConstraint('rectangle', [xtl, ytl, xbr, ybr])) { + this.onDrawDone({ + shapeType, + clientID, + points: [xtl, ytl, xbr, ybr], + }, + Date.now() - this.startTimestamp); + } + } + }) + .on('undopoint', (): void => { + if (numberOfPoints > 0) { + numberOfPoints -= 1; + } + }); + + this.drawPolyshape(); + } + + private drawPolyshape(): void { + let size = this.drawData.shapeType === 'cuboid' ? 4 : this.drawData.numberOfPoints; + const snapToPoint = (pointIndex: number): void => { + if ( + !this.configuration.snapToPoint || + this.isCtrlKeyDown() || + !['polygon', 'polyline'].includes(this.drawData.shapeType) || + !this.getDrawnStates + ) { + return; + } + + const pointsArray = (this.drawInstance as any).array().valueOf(); + if (!pointsArray.length || pointIndex < 0 || pointIndex >= pointsArray.length) { + return; + } + + applySnapToShapePoint( + this.drawInstance, + pointIndex, + this.getDrawnStates(), + this.geometry.offset, + this.configuration.snapRadius / this.geometry.scale, + this.drawData.redraw, + ); + }; + + const sizeDecrement = (): void => { + if (--size === 0) { + // we need additional settimeout because we cannot invoke draw('done') + // from event listener for drawstart event + // because of implementation of svg.js + setTimeout((): void => this.drawInstance.draw('done')); + } + }; + + this.drawInstance.on('drawstart', () => { + sizeDecrement(); + snapToPoint(0); + }); + this.drawInstance.on('drawpoint', sizeDecrement); + this.drawInstance.on('drawupdate', (): void => { + this.transform(this.geometry); + snapToPoint((this.drawInstance as any).array().valueOf().length - 1); + }); + this.drawInstance.on('undopoint', (): number => size++); + + // Add ability to cancel the latest drawn point + this.canvas.on('mousedown.draw', (e: MouseEvent): void => { + if (e.button === 2) { + e.stopPropagation(); + e.preventDefault(); + this.drawInstance.draw('undo'); + } + }); + + // Add ability to draw shapes by sliding + // We need to remember last drawn point + // to implementation of slide drawing + const lastDrawnPoint: { + x: number; + y: number; + } = { + x: null, + y: null, + }; + + this.canvas.on('mousemove.draw', (e: MouseEvent): void => { + // TODO: Use enumeration after typification cvat-core + if (e.shiftKey && ['polygon', 'polyline'].includes(this.drawData.shapeType)) { + if (lastDrawnPoint.x === null || lastDrawnPoint.y === null) { + this.drawInstance.draw('point', e); + } else { + this.drawInstance.draw('update', e); + const deltaThreshold = 15; + const dx = (e.clientX - lastDrawnPoint.x) ** 2; + const dy = (e.clientY - lastDrawnPoint.y) ** 2; + const delta = Math.sqrt(dx + dy); + if (delta > deltaThreshold) { + this.drawInstance.draw('point', e); + } + } + + e.stopPropagation(); + e.preventDefault(); + } + }); + + // We need to scale points that have been just drawn + this.drawInstance.on('drawstart drawpoint', (e: CustomEvent): void => { + this.transform(this.geometry); + lastDrawnPoint.x = e.detail.event.clientX; + lastDrawnPoint.y = e.detail.event.clientY; + }); + + this.drawInstance.on('drawdone', (e: CustomEvent): void => { + const targetPoints = readPointsFromShape((e.target as any as { instance: SVG.Shape }).instance); + const { shapeType, redraw: clientID, simplifyPoly } = this.drawData; + const { points, box } = shapeType === 'cuboid' ? + this.getFinalCuboidCoordinates(targetPoints) : + this.getFinalPolyshapeCoordinates(targetPoints, true); + + if (this.canceled) { + return; + } + + this.release(); + if (checkConstraint(shapeType, points, box)) { + if (shapeType === 'cuboid') { + this.onDrawDone( + { clientID, shapeType, points: cuboidFrom4Points(points) }, + Date.now() - this.startTimestamp, + ); + return; + } + + this.onDrawDone({ + clientID, shapeType, points, simplifyPoly, + }, Date.now() - this.startTimestamp); + } else { + this.onDrawDone(null); + } + }); + } + + private drawPolygon(): void { + this.drawInstance = (this.canvas as any) + .polygon() + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }); + + this.drawPolyshape(); + if (this.autobordersEnabled) { + this.autoborderHandler.autoborder(true, this.drawInstance, this.drawData.redraw); + } + } + + private drawPolyline(): void { + this.drawInstance = (this.canvas as any) + .polyline() + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': 0, + stroke: this.outlinedBorders, + }); + + this.drawPolyshape(); + if (this.autobordersEnabled) { + this.autoborderHandler.autoborder(true, this.drawInstance, this.drawData.redraw); + } + } + + private drawPoints(): void { + this.drawInstance = (this.canvas as any).polygon().addClass('cvat_canvas_shape_drawing').attr({ + 'stroke-width': 0, + opacity: 0, + }); + + this.drawPolyshape(); + } + + private drawCuboidBy4Points(): void { + this.drawInstance = (this.canvas as any) + .polyline() + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + stroke: this.outlinedBorders, + }); + this.drawPolyshape(); + } + + private drawCuboid(): void { + this.drawInstance = this.canvas.rect(); + this.drawInstance + .on('drawstop', (e: Event): void => { + const points = readPointsFromShape((e.target as any as { instance: SVG.Rect }).instance); + const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(points, true); + const { shapeType, redraw: clientID } = this.drawData; + + if (this.canceled) { + return; + } + + this.release(); + if (checkConstraint('cuboid', [xtl, ytl, xbr, ybr])) { + const d = { x: (xbr - xtl) * 0.1, y: (ybr - ytl) * 0.1 }; + this.onDrawDone({ + shapeType, + points: cuboidFrom4Points([xtl, ybr, xbr, ybr, xbr, ytl, xbr + d.x, ytl - d.y]), + clientID, + }, + Date.now() - this.startTimestamp); + } else { + this.onDrawDone(null); + } + }) + .on('drawupdate', (): void => { + this.shapeSizeElement.update(this.drawInstance); + }) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }); + } + + private drawSkeleton(): void { + this.drawInstance = this.canvas.rect().attr({ + stroke: this.outlinedBorders, + }); + this.pointsGroup = makeSVGFromTemplate(this.drawData.skeletonSVG); + this.canvas.add(this.pointsGroup); + this.pointsGroup.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale); + this.pointsGroup.attr('stroke', this.outlinedBorders); + + let minX = Number.MAX_SAFE_INTEGER; + let minY = Number.MAX_SAFE_INTEGER; + let maxX = 0; + let maxY = 0; + + this.pointsGroup.children().forEach((child: SVG.Element): void => { + const cx = child.cx(); + const cy = child.cy(); + minX = Math.min(cx, minX); + minY = Math.min(cy, minY); + maxX = Math.max(cx, maxX); + maxY = Math.max(cy, maxY); + }); + + this.drawInstance + .on('drawstop', (e: Event): void => { + const points = readPointsFromShape((e.target as any as { instance: SVG.Rect }).instance); + const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(points, true); + const elements: any[] = []; + Array.from(this.pointsGroup.node.children).forEach((child: Element) => { + if (child.tagName === 'circle') { + const cx = +(child.getAttribute('cx') as string) + xtl; + const cy = +(child.getAttribute('cy') as string) + ytl; + const label = +child.getAttribute('data-label-id'); + elements.push({ + shapeType: 'points', + points: [cx, cy], + labelID: label, + }); + } + }); + + const { shapeType, redraw: clientID } = this.drawData; + + if (this.canceled) { + return; + } + + this.release(); + if (checkConstraint('skeleton', [xtl, ytl, xbr, ybr])) { + this.onDrawDone({ + clientID, + shapeType, + elements, + }, + Date.now() - this.startTimestamp); + } else { + this.onDrawDone(null); + } + }) + .on('drawupdate', (): void => { + const x = this.drawInstance.x(); + const y = this.drawInstance.y(); + const width = this.drawInstance.width(); + const height = this.drawInstance.height(); + this.pointsGroup.style({ + transform: `translate(${x}px, ${y}px)`, + }); + + this.pointsGroup.node.replaceChildren(...this.drawData.skeletonSVG.cloneNode(true).childNodes); + Array.from(this.pointsGroup.node.children).forEach((child: Element) => { + const dataType = child.getAttribute('data-type'); + if (child.tagName === 'circle' && dataType && dataType.includes('element')) { + child.setAttribute('r', `${this.controlPointsSize / this.geometry.scale}`); + let cx = +(child.getAttribute('cx') as string); + let cy = +(child.getAttribute('cy') as string); + const cxOffset = (cx - minX) / (maxX - minX); + const cyOffset = (cy - minY) / (maxY - minY); + cx = Number.isNaN(cxOffset) ? 0.5 * width : cxOffset * width; + cy = Number.isNaN(cyOffset) ? 0.5 * height : cyOffset * height; + child.setAttribute('cx', `${cx}`); + child.setAttribute('cy', `${cy}`); + } + }); + + Array.from(this.pointsGroup.node.children).forEach((child: Element) => { + const dataType = child.getAttribute('data-type'); + if (child.tagName === 'line' && dataType && dataType.includes('edge')) { + child.setAttribute('stroke-width', 'inherit'); + child.setAttribute('stroke', 'inherit'); + const dataNodeFrom = child.getAttribute('data-node-from'); + const dataNodeTo = child.getAttribute('data-node-to'); + if (dataNodeFrom && dataNodeTo) { + const from = this.pointsGroup.node.querySelector(`[data-node-id="${dataNodeFrom}"]`); + const to = this.pointsGroup.node.querySelector(`[data-node-id="${dataNodeTo}"]`); + + if (from && to) { + const x1 = from.getAttribute('cx'); + const y1 = from.getAttribute('cy'); + const x2 = to.getAttribute('cx'); + const y2 = to.getAttribute('cy'); + + if (x1 && y1 && x2 && y2) { + child.setAttribute('x1', x1); + child.setAttribute('y1', y1); + child.setAttribute('x2', x2); + child.setAttribute('y2', y2); + } + } + } + let cx = +(child.getAttribute('cx') as string); + let cy = +(child.getAttribute('cy') as string); + const cxOffset = cx / 100; + const cyOffset = cy / 100; + cx = cxOffset * width; + cy = cyOffset * height; + child.setAttribute('cx', `${cx}`); + child.setAttribute('cy', `${cy}`); + } + }); + }) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + }); + } + + private pastePolyshape(): void { + this.drawInstance.on('done', (e: CustomEvent): void => { + const targetPoints = this.drawInstance + .attr('points') + .split(/[,\s]/g) + .map((coord: string): number => +coord); + + const { shapeType } = this.drawData.initialState; + const { points, box } = shapeType === 'cuboid' ? + this.getFinalCuboidCoordinates(targetPoints) : + this.getFinalPolyshapeCoordinates(targetPoints, true); + + if (checkConstraint(shapeType, points, box)) { + this.onDrawDone( + { + shapeType, + objectType: this.drawData.initialState.objectType, + points, + occluded: this.drawData.initialState.occluded, + attributes: { ...this.drawData.initialState.attributes }, + label: this.drawData.initialState.label, + color: this.drawData.initialState.color, + }, + Date.now() - this.startTimestamp, + e.detail.originalEvent.ctrlKey, + this.drawData, + ); + } + + if (!e.detail.originalEvent.ctrlKey) { + this.release(); + } + }); + } + + // Common settings for rectangle and polyshapes + private pasteShape(): void { + const moveShape = (shape: SVG.Shape, x: number, y: number): void => { + const { rotation } = shape.transform(); + shape.untransform(); + shape.center(x, y); + shape.rotate(rotation); + }; + + const { x: initialX, y: initialY } = this.cursorPosition; + moveShape(this.drawInstance, initialX, initialY); + + this.canvas.on('mousemove.draw', (): void => { + const { x, y } = this.cursorPosition; // was computed in another callback + moveShape(this.drawInstance, x, y); + }); + } + + private pasteBox(box: BBox, rotation: number): void { + this.drawInstance = (this.canvas as any) + .rect(box.width, box.height) + .center(box.x, box.y) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }).rotate(rotation); + this.pasteShape(); + + this.drawInstance.on('done', (e: CustomEvent): void => { + const points = readPointsFromShape((e.target as any as { instance: SVG.Rect }).instance); + const [xtl, ytl, xbr, ybr] = this.getFinalRectCoordinates(points, !this.drawData.initialState.rotation); + if (checkConstraint('rectangle', [xtl, ytl, xbr, ybr])) { + this.onDrawDone( + { + shapeType: this.drawData.initialState.shapeType, + objectType: this.drawData.initialState.objectType, + points: [xtl, ytl, xbr, ybr], + occluded: this.drawData.initialState.occluded, + attributes: { ...this.drawData.initialState.attributes }, + label: this.drawData.initialState.label, + color: this.drawData.initialState.color, + rotation: this.drawData.initialState.rotation, + }, + Date.now() - this.startTimestamp, + e.detail.originalEvent.ctrlKey, + this.drawData, + ); + } + + if (!e.detail.originalEvent.ctrlKey) { + this.release(); + } + }); + } + + private pasteEllipse([cx, cy, rx, ry]: number[], rotation: number): void { + this.drawInstance = (this.canvas as any) + .ellipse(rx * 2, ry * 2) + .center(cx, cy) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }).rotate(rotation); + this.pasteShape(); + + this.drawInstance.on('done', (e: CustomEvent): void => { + const points = this.getFinalEllipseCoordinates( + readPointsFromShape((e.target as any as { instance: SVG.Ellipse }).instance), false, + ); + if (checkConstraint('ellipse', points)) { + this.onDrawDone( + { + shapeType: this.drawData.initialState.shapeType, + objectType: this.drawData.initialState.objectType, + points, + occluded: this.drawData.initialState.occluded, + attributes: { ...this.drawData.initialState.attributes }, + label: this.drawData.initialState.label, + color: this.drawData.initialState.color, + rotation: this.drawData.initialState.rotation, + }, + Date.now() - this.startTimestamp, + e.detail.originalEvent.ctrlKey, + this.drawData, + ); + } + + if (!e.detail.originalEvent.ctrlKey) { + this.release(); + } + }); + } + + private pastePolygon(points: string): void { + this.drawInstance = (this.canvas as any) + .polygon(points) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }); + this.pasteShape(); + this.pastePolyshape(); + } + + private pastePolyline(points: string): void { + this.drawInstance = (this.canvas as any) + .polyline(points) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + stroke: this.outlinedBorders, + }); + this.pasteShape(); + this.pastePolyshape(); + } + + private pasteCuboid(points: string): void { + this.drawInstance = (this.canvas as any) + .cube(points) + .addClass('cvat_canvas_shape_drawing') + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + 'face-stroke': this.outlinedBorders, + 'fill-opacity': this.selectedShapeOpacity, + stroke: this.outlinedBorders, + }); + this.pasteShape(); + this.pastePolyshape(); + } + + private pasteSkeleton(box: BBox, elements: any[]): void { + const { offset } = this.geometry; + let [xtl, ytl] = [box.x, box.y]; + + this.pasteBox(box, 0); + this.pointsGroup = makeSVGFromTemplate(this.drawData.skeletonSVG); + this.pointsGroup.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + stroke: this.outlinedBorders, + }); + this.canvas.add(this.pointsGroup); + + this.pointsGroup.children().forEach((child: SVG.Element): void => { + const dataType = child.attr('data-type'); + if (child.node.tagName === 'circle' && dataType && dataType.includes('element')) { + child.attr('r', `${this.controlPointsSize / this.geometry.scale}`); + const labelID = +child.attr('data-label-id'); + const element = elements.find((_element: any): boolean => _element.label.id === labelID); + if (element) { + const points = translateToCanvas(offset, element.points); + child.center(points[0], points[1]); + } + } + }); + + this.drawInstance.off('done').on('done', (e: CustomEvent) => { + const result = { + shapeType: this.drawData.initialState.shapeType, + objectType: this.drawData.initialState.objectType, + elements: this.drawData.initialState.elements.map((element: any) => ({ + shapeType: element.shapeType, + outside: element.outside, + occluded: element.occluded, + label: element.label, + attributes: element.attributes, + points: (() => { + const circle = this.pointsGroup.children() + .find((child: SVG.Element) => child.attr('data-label-id') === element.label.id); + const points = translateFromCanvas(this.geometry.offset, [circle.cx(), circle.cy()]); + return points; + })(), + })), + occluded: this.drawData.initialState.occluded, + attributes: { ...this.drawData.initialState.attributes }, + label: this.drawData.initialState.label, + color: this.drawData.initialState.color, + rotation: this.drawData.initialState.rotation, + }; + + this.onDrawDone( + result, + Date.now() - this.startTimestamp, + e.detail.originalEvent.ctrlKey, + this.drawData, + ); + + if (!e.detail.originalEvent.ctrlKey) { + this.release(); + } + }); + + this.canvas.on('mousemove.draw', (): void => { + const [newXtl, newYtl] = [ + this.drawInstance.x(), this.drawInstance.y(), + this.drawInstance.width(), this.drawInstance.height(), + ]; + const [xDiff, yDiff] = [newXtl - xtl, newYtl - ytl]; + xtl = newXtl; + ytl = newYtl; + this.pointsGroup.children().forEach((child: SVG.Element): void => { + const dataType = child.attr('data-type'); + if (child.node.tagName === 'circle' && dataType && dataType.includes('element')) { + const [cx, cy] = [child.cx(), child.cy()]; + child.center(cx + xDiff, cy + yDiff); + } + }); + this.pointsGroup.untransform(); + setupSkeletonEdges(this.pointsGroup, this.pointsGroup); + }); + } + + private pastePoints(initialPoints: string): void { + const moveShape = (shape: SVG.PolyLine, group: SVG.G, x: number, y: number, scale: number): void => { + const bbox = shape.bbox(); + shape.move(x - bbox.width / 2, y - bbox.height / 2); + + const points = shape.attr('points').split(' '); + const radius = this.controlPointsSize / scale; + + group.children().forEach((child: SVG.Element, idx: number): void => { + const [px, py] = points[idx].split(','); + child.move(px - radius / 2, py - radius / 2); + }); + }; + + const { x: initialX, y: initialY } = this.cursorPosition; + this.pointsGroup = this.canvas.group(); + this.drawInstance = (this.canvas as any).polyline(initialPoints).addClass('cvat_canvas_shape_drawing').style({ + 'stroke-width': 0, + }); + + let numOfPoints = initialPoints.split(' ').length; + while (numOfPoints) { + numOfPoints--; + const radius = this.controlPointsSize / this.geometry.scale; + const stroke = consts.POINTS_STROKE_WIDTH / this.geometry.scale; + this.pointsGroup.circle().fill('white').stroke('black').attr({ + r: radius, + 'stroke-width': stroke, + }); + } + + moveShape(this.drawInstance, this.pointsGroup, initialX, initialY, this.geometry.scale); + + this.canvas.on('mousemove.draw', (): void => { + const { x, y } = this.cursorPosition; // was computer in another callback + moveShape(this.drawInstance, this.pointsGroup, x, y, this.geometry.scale); + }); + + this.pastePolyshape(); + } + + private setupPasteEvents(): void { + this.canvas.on('mousedown.draw', (e: MouseEvent): void => { + if (e.button === 0 && !e.altKey) { + this.drawInstance.fire('done', { originalEvent: e }); + } + }); + } + + private setupDrawEvents(): void { + let initialized = false; + + this.canvas.on('mousedown.draw', (e: MouseEvent): void => { + if (e.button === 0 && !e.altKey) { + if (!initialized) { + this.drawInstance.draw(e, { snapToGrid: 0.1 }); + initialized = true; + } else { + this.drawInstance.draw(e); + } + } + }); + } + + private startDraw(): void { + // TODO: Use enums after typification cvat-core + if (this.drawData.initialState) { + const { offset } = this.geometry; + if (this.drawData.shapeType === 'rectangle') { + const [xtl, ytl, xbr, ybr] = translateToCanvas(offset, this.drawData.initialState.points); + this.pasteBox({ + x: xtl, + y: ytl, + width: xbr - xtl, + height: ybr - ytl, + }, this.drawData.initialState.rotation); + } else if (this.drawData.shapeType === 'ellipse') { + const [cx, cy, rightX, topY] = translateToCanvas(offset, this.drawData.initialState.points); + this.pasteEllipse([cx, cy, rightX - cx, cy - topY], this.drawData.initialState.rotation); + } else if (this.drawData.shapeType === 'skeleton') { + const box = computeWrappingBox( + translateToCanvas(offset, this.drawData.initialState.points), consts.SKELETON_RECT_MARGIN, + ); + this.pasteSkeleton(box, this.drawData.initialState.elements); + } else { + const points = translateToCanvas(offset, this.drawData.initialState.points); + const stringifiedPoints = stringifyPoints(points); + + if (this.drawData.shapeType === 'polygon') { + this.pastePolygon(stringifiedPoints); + } else if (this.drawData.shapeType === 'polyline') { + this.pastePolyline(stringifiedPoints); + } else if (this.drawData.shapeType === 'points') { + this.pastePoints(stringifiedPoints); + } else if (this.drawData.shapeType === 'cuboid') { + this.pasteCuboid(stringifiedPoints); + } + } + this.setupPasteEvents(); + } else { + if (this.drawData.shapeType === 'rectangle') { + if (this.drawData.rectDrawingMethod === RectDrawingMethod.EXTREME_POINTS) { + this.drawBoxBy4Points(); // draw box by extreme clicking + } else { + this.drawBox(); // default box drawing + // draw instance was initialized after drawBox(); + this.shapeSizeElement = displayShapeSize(this.canvas, this.text); + } + } else if (this.drawData.shapeType === 'polygon') { + this.drawPolygon(); + } else if (this.drawData.shapeType === 'polyline') { + this.drawPolyline(); + } else if (this.drawData.shapeType === 'points') { + this.drawPoints(); + } else if (this.drawData.shapeType === 'ellipse') { + this.drawEllipse(); + this.shapeSizeElement = displayShapeSize(this.canvas, this.text); + } else if (this.drawData.shapeType === 'cuboid') { + if (this.drawData.cuboidDrawingMethod === CuboidDrawingMethod.CORNER_POINTS) { + this.drawCuboidBy4Points(); + } else { + this.drawCuboid(); + this.shapeSizeElement = displayShapeSize(this.canvas, this.text); + } + } else if (this.drawData.shapeType === 'skeleton') { + this.drawSkeleton(); + } + + if (this.drawData.shapeType !== 'ellipse') { + this.setupDrawEvents(); + } + } + + this.startTimestamp = Date.now(); + this.initialized = true; + } + + public constructor( + onDrawDone: DrawHandlerImpl['onDrawDoneDefault'], + canvas: SVG.Container, + text: SVG.Container, + autoborderHandler: AutoborderHandler, + geometry: Geometry, + configuration: Configuration, + getDrawnStates: () => Record, + isCtrlKeyDown: () => boolean, + ) { + this.autoborderHandler = autoborderHandler; + this.configuration = configuration; + this.controlPointsSize = configuration.controlPointsSize; + this.selectedShapeOpacity = configuration.selectedShapeOpacity; + this.outlinedBorders = configuration.outlinedBorders || 'black'; + this.autobordersEnabled = false; + this.isHidden = false; + this.startTimestamp = Date.now(); + this.onDrawDoneDefault = onDrawDone; + this.canvas = canvas; + this.text = text; + this.initialized = false; + this.canceled = false; + this.drawData = null; + this.geometry = geometry; + this.crosshair = new Crosshair(); + this.drawInstance = null; + this.pointsGroup = null; + this.getDrawnStates = getDrawnStates; + this.isCtrlKeyDown = isCtrlKeyDown; + this.cursorPosition = { + x: 0, + y: 0, + }; + + this.canvas.on('mousemove.crosshair', (e: MouseEvent): void => { + const [x, y] = translateToSVG((this.canvas.node as any) as SVGSVGElement, [e.clientX, e.clientY]); + this.cursorPosition = { x, y }; + if (this.crosshair) { + this.crosshair.move(x, y); + } + }); + } + + private strokePoint(point: SVG.Element): void { + point.attr('stroke', this.isHidden ? 'none' : CIRCLE_STROKE); + point.fill({ opacity: this.isHidden ? 0 : 1 }); + } + + private updateHidden(value: boolean): void { + this.isHidden = value; + + if (value) { + this.canvas.attr('pointer-events', 'none'); + } else { + this.canvas.attr('pointer-events', 'all'); + } + } + + public configure(configuration: Configuration): void { + this.configuration = configuration; + this.controlPointsSize = configuration.controlPointsSize; + this.selectedShapeOpacity = configuration.selectedShapeOpacity; + this.outlinedBorders = configuration.outlinedBorders || 'black'; + if (this.isHidden !== configuration.hideEditedObject) { + this.updateHidden(configuration.hideEditedObject); + } + + const isFillableRect = this.drawData && + this.drawData.shapeType === 'rectangle' && + (this.drawData.rectDrawingMethod === RectDrawingMethod.CLASSIC || this.drawData.initialState); + const isFillableCuboid = this.drawData && + this.drawData.shapeType === 'cuboid' && + (this.drawData.cuboidDrawingMethod === CuboidDrawingMethod.CLASSIC || this.drawData.initialState); + const isFilalblePolygon = this.drawData && this.drawData.shapeType === 'polygon'; + + if (this.drawInstance && (isFillableRect || isFillableCuboid || isFilalblePolygon)) { + this.drawInstance.fill({ + opacity: configuration.hideEditedObject ? 0 : configuration.selectedShapeOpacity, + }); + } + + if (this.drawInstance && isFilalblePolygon) { + const paintHandler = this.drawInstance.remember('_paintHandler'); + if (paintHandler) { + for (const point of (paintHandler as any).set.members) { + this.strokePoint(point); + } + } + } + + if (this.drawInstance && this.drawInstance.attr('stroke')) { + this.drawInstance.attr('stroke', configuration.hideEditedObject ? 'none' : this.outlinedBorders); + } + + if (this.pointsGroup && this.pointsGroup.attr('stroke')) { + this.pointsGroup.attr('stroke', configuration.hideEditedObject ? 'none' : this.outlinedBorders); + } + + this.autobordersEnabled = configuration.autoborders; + if (this.drawInstance && !this.drawData.initialState) { + if (this.autobordersEnabled) { + this.autoborderHandler.autoborder(true, this.drawInstance, this.drawData.redraw); + } else { + this.autoborderHandler.autoborder(false); + } + } + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + + if (this.shapeSizeElement && this.drawInstance && ['rectangle', 'ellipse'].includes(this.drawData.shapeType)) { + this.shapeSizeElement.update(this.drawInstance); + } + + if (this.crosshair) { + this.crosshair.scale(this.geometry.scale); + } + + if (this.pointsGroup) { + this.pointsGroup.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + }); + + for (const point of this.pointsGroup.children()) { + point.attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / geometry.scale, + r: this.controlPointsSize / geometry.scale, + }); + } + } + + if (this.drawInstance) { + this.drawInstance.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale, + }); + + const paintHandler = this.drawInstance.remember('_paintHandler'); + if (paintHandler) { + for (const point of (paintHandler as any).set.members) { + this.strokePoint(point); + point.attr('stroke-width', `${consts.POINTS_STROKE_WIDTH / geometry.scale}`); + point.attr('r', `${this.controlPointsSize / geometry.scale}`); + } + } + } + } + + public draw(drawData: DrawData, geometry: Geometry): void { + this.geometry = geometry; + + if (drawData.enabled) { + this.canceled = false; + this.drawData = drawData; + this.initDrawing(); + this.startDraw(); + } else { + this.release(); + this.drawData = drawData; + } + } + + public cancel(): void { + this.canceled = true; + this.release(); + } +} diff --git a/cvat-canvas/src/typescript/editHandler.ts b/cvat-canvas/src/typescript/editHandler.ts new file mode 100644 index 0000000..2cd2974 --- /dev/null +++ b/cvat-canvas/src/typescript/editHandler.ts @@ -0,0 +1,479 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import 'svg.select.js'; + +import consts from './consts'; +import { translateFromSVG, pointsToNumberArray } from './shared'; +import { PolyEditData, Geometry, Configuration } from './canvasModel'; +import { AutoborderHandler } from './autoborderHandler'; + +export interface EditHandler { + edit(editData: PolyEditData): void; + transform(geometry: Geometry): void; + configure(configuration: Configuration): void; + cancel(): void; + enabled: boolean; + shapeType: string; +} + +export class EditHandlerImpl implements EditHandler { + private onEditDone: (state: any, points: number[]) => void; + private autoborderHandler: AutoborderHandler; + private geometry: Geometry | null; + private canvas: SVG.Container; + private editData: PolyEditData | null; + private editedShape: SVG.Shape | null; + private editLine: SVG.PolyLine | null; + private clones: SVG.Polygon[]; + private controlPointsSize: number; + private autobordersEnabled: boolean; + private intelligentCutEnabled: boolean; + private outlinedBorders: string; + private isEditing: boolean; + + private setupTrailingPoint(circle: SVG.Circle): void { + circle.on('mouseenter', (): void => { + circle.attr({ + 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale, + }); + }); + + circle.on('mouseleave', (): void => { + circle.attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / this.geometry.scale, + }); + }); + + circle.on('mousedown', (e: MouseEvent): void => { + if (e.button !== 0) return; + this.edit({ enabled: false }); + }); + } + + private startEdit(): void { + // get started coordinates + const [clientX, clientY] = translateFromSVG( + (this.canvas.node as any) as SVGSVGElement, + this.editedShape.attr('points').split(' ')[this.editData.pointID].split(','), + ); + + // generate mouse event + const dummyEvent = new MouseEvent('mousedown', { + bubbles: true, + cancelable: true, + clientX, + clientY, + }); + + // Add ability to edit shapes by sliding + // We need to remember last drawn point + // to implementation of slide drawing + const lastDrawnPoint: { + x: number; + y: number; + } = { + x: null, + y: null, + }; + + this.canvas.on('mousemove.edit', (e: MouseEvent): void => { + if (e.shiftKey && ['polygon', 'polyline'].includes(this.editData.state.shapeType)) { + if (lastDrawnPoint.x === null || lastDrawnPoint.y === null) { + (this.editLine as any).draw('point', e); + } else { + const deltaThreshold = 15; + const dxsqr = (e.clientX - lastDrawnPoint.x) ** 2; + const dysqr = (e.clientY - lastDrawnPoint.y) ** 2; + const delta = Math.sqrt(dxsqr + dysqr); + if (delta > deltaThreshold) { + (this.editLine as any).draw('point', e); + } + } + } + }); + + this.editLine = (this.canvas as any).polyline(); + if (this.editData.state.shapeType === 'polyline') { + (this.editLine as any).on('drawupdate', (e: CustomEvent): void => { + const circle = (e.target as any).instance.remember('_paintHandler').set.last(); + if (circle) this.setupTrailingPoint(circle); + }); + } + + (this.editLine as any) + .addClass('cvat_canvas_shape_drawing') + .style({ + 'pointer-events': 'none', + 'fill-opacity': 0, + }) + .attr({ + 'data-origin-client-id': this.editData.state.clientID, + stroke: this.editedShape.attr('stroke'), + }) + .on('drawstart drawpoint', (e: CustomEvent): void => { + this.transform(this.geometry); + lastDrawnPoint.x = e.detail.event.clientX; + lastDrawnPoint.y = e.detail.event.clientY; + }) + .on('drawupdate', (): void => this.transform(this.geometry)) + .draw(dummyEvent, { snapToGrid: 0.1 }); + + if (this.editData.state.shapeType === 'points') { + this.editLine.attr('stroke-width', 0); + (this.editLine as any).draw('undo'); + } + + this.setupEditEvents(); + if (this.autobordersEnabled) { + this.autoborderHandler.autoborder(true, this.editLine, this.editData.state.clientID); + } + } + + private setupEditEvents(): void { + this.canvas.on('mousedown.edit', (e: MouseEvent): void => { + if (e.button === 0 && !e.altKey) { + (this.editLine as any).draw('point', e); + } else if (e.button === 2 && this.editLine) { + if (this.editData.state.shapeType === 'points' || this.editLine.attr('points').split(' ').length > 2) { + (this.editLine as any).draw('undo'); + } + } + }); + } + + private selectPolygon(shape: SVG.Polygon): void { + const { offset } = this.geometry; + const points = pointsToNumberArray(shape.attr('points')).map((coord: number): number => coord - offset); + + const { state } = this.editData; + this.edit({ + enabled: false, + }); + this.onEditDone(state, points); + } + + private stopEdit(e: MouseEvent): void { + if (!this.editLine) { + return; + } + + // Get stop point and all points + const stopPointID = Array.prototype.indexOf.call((e.target as HTMLElement).parentElement.children, e.target); + const oldPoints = this.editedShape.attr('points').trim().split(' '); + const linePoints = this.editLine.attr('points').trim().split(' '); + + if (this.editLine.attr('points') === '0,0') { + this.cancel(); + return; + } + + // Compute new point array + const [start, stop] = [this.editData.pointID, stopPointID].sort((a, b): number => +a - +b); + + if (this.editData.state.shapeType !== 'polygon') { + let points = null; + const { offset } = this.geometry; + + if (this.editData.state.shapeType === 'polyline') { + if (start !== this.editData.pointID) { + linePoints.reverse(); + } + points = oldPoints + .slice(0, start) + .concat(linePoints) + .concat(oldPoints.slice(stop + 1)); + } else { + points = oldPoints.concat(linePoints.slice(0, -1)); + } + + points = pointsToNumberArray(points.join(' ')).map((coord: number): number => coord - offset); + + const { state } = this.editData; + this.edit({ + enabled: false, + }); + this.onEditDone(state, points); + + return; + } + + const cutIndexes1 = oldPoints.reduce( + (acc: number[], _: string, i: number): number[] => (i >= stop || i <= start ? [...acc, i] : acc), + [], + ); + const cutIndexes2 = oldPoints.reduce( + (acc: number[], _: string, i: number): number[] => (i <= stop && i >= start ? [...acc, i] : acc), + [], + ); + + const curveLength = (indexes: number[]): number => { + const points = indexes + .map((index: number): string => oldPoints[index]) + .map((point: string): string[] => point.split(',')) + .map((point: string[]): number[] => [+point[0], +point[1]]); + let length = 0; + for (let i = 1; i < points.length; i++) { + const dxsqr = (points[i][0] - points[i - 1][0]) ** 2; + const dysqr = (points[i][1] - points[i - 1][1]) ** 2; + length += Math.sqrt(dxsqr + dysqr); + } + + return length; + }; + + const pointsCriteria = cutIndexes1.length > cutIndexes2.length; + const lengthCriteria = curveLength(cutIndexes1) > curveLength(cutIndexes2); + + if (start !== this.editData.pointID) { + linePoints.reverse(); + } + + const firstPart = oldPoints + .slice(0, start) + .concat(linePoints) + .concat(oldPoints.slice(stop + 1)); + const secondPart = oldPoints.slice(start, stop).concat(linePoints.slice(1).reverse()); + + if (firstPart.length < 3 || secondPart.length < 3) { + this.cancel(); + return; + } + + // We do not need these events any more + this.canvas.off('mousedown.edit'); + this.canvas.off('mousemove.edit'); + + (this.editLine as any).draw('stop'); + this.editLine.remove(); + this.editLine = null; + + if (pointsCriteria && lengthCriteria && this.intelligentCutEnabled) { + this.clones.push(this.canvas.polygon(firstPart.join(' '))); + this.selectPolygon(this.clones[0]); + // left indexes1 and + } else if (!pointsCriteria && !lengthCriteria && this.intelligentCutEnabled) { + this.clones.push(this.canvas.polygon(secondPart.join(' '))); + this.selectPolygon(this.clones[0]); + } else { + for (const points of [firstPart, secondPart]) { + this.clones.push( + this.canvas + .polygon(points.join(' ')) + .attr('fill', this.editedShape.attr('fill')) + .attr('fill-opacity', '0.5') + .addClass('cvat_canvas_shape'), + ); + } + + for (const clone of this.clones) { + clone.on('click', (): void => this.selectPolygon(clone)); + clone + .on('mouseenter', (): void => { + clone.addClass('cvat_canvas_shape_splitting'); + }) + .on('mouseleave', (): void => { + clone.removeClass('cvat_canvas_shape_splitting'); + }); + } + } + } + + private setupPoints(enabled: boolean): void { + const stopEdit = this.stopEdit.bind(this); + const getGeometry = (): Geometry => this.geometry; + const fill = this.editedShape.attr('fill') || 'inherit'; + + if (enabled) { + (this.editedShape as any).selectize(true, { + deepSelect: true, + pointSize: (2 * this.controlPointsSize) / getGeometry().scale, + rotationPoint: false, + pointType(cx: number, cy: number): SVG.Circle { + const circle: SVG.Circle = this.nested + .circle(this.options.pointSize) + .stroke('black') + .fill(fill) + .center(cx, cy) + .attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / getGeometry().scale, + }); + + circle.node.addEventListener('mouseenter', (): void => { + circle.attr({ + 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / getGeometry().scale, + }); + + circle.node.addEventListener('click', stopEdit); + circle.addClass('cvat_canvas_selected_point'); + }); + + circle.node.addEventListener('mouseleave', (): void => { + circle.attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / getGeometry().scale, + }); + + circle.node.removeEventListener('click', stopEdit); + circle.removeClass('cvat_canvas_selected_point'); + }); + + return circle; + }, + }); + } else { + (this.editedShape as any).selectize(false, { + deepSelect: true, + }); + } + } + + private release(): void { + this.canvas.off('mousedown.edit'); + this.canvas.off('mousemove.edit'); + this.autoborderHandler.autoborder(false); + this.isEditing = false; + + if (this.editedShape) { + this.setupPoints(false); + this.editedShape.remove(); + this.editedShape = null; + } + + if (this.editLine) { + (this.editLine as any).draw('stop'); + this.editLine.remove(); + this.editLine = null; + } + + if (this.clones.length) { + for (const clone of this.clones) { + clone.remove(); + } + this.clones = []; + } + } + + private initEditing(): void { + this.editedShape = this.canvas + .select(`#cvat_canvas_shape_${this.editData.state.clientID}`).first() + .clone().attr('stroke', this.outlinedBorders); + this.setupPoints(true); + this.startEdit(); + this.isEditing = true; + // draw points for this with selected and start editing till another point is clicked + // click one of two parts to remove (in case of polygon only) + + // else we can start draw polyline + // after we have got shape and points, we are waiting for second point pressed on this shape + } + + private closeEditing(): void { + if (this.isEditing && this.editData.state.shapeType === 'polyline') { + const { offset } = this.geometry; + const head = this.editedShape.attr('points').split(' ').slice(0, this.editData.pointID).join(' '); + const stringifiedPoints = `${head} ${this.editLine.node.getAttribute('points').slice(0, -2)}`; + const points = pointsToNumberArray(stringifiedPoints) + .slice(0, -2) + .map((coord: number): number => coord - offset); + if (points.length >= 2 * 2) { // minimumPoints * 2 + const { state } = this.editData; + this.onEditDone(state, points); + } + } + this.release(); + } + + public constructor( + onEditDone: EditHandlerImpl['onEditDone'], + canvas: SVG.Container, + autoborderHandler: AutoborderHandler, + ) { + this.autoborderHandler = autoborderHandler; + this.autobordersEnabled = false; + this.intelligentCutEnabled = false; + this.controlPointsSize = consts.BASE_POINT_SIZE; + this.outlinedBorders = 'black'; + this.onEditDone = onEditDone; + this.canvas = canvas; + this.editData = null; + this.editedShape = null; + this.editLine = null; + this.geometry = null; + this.clones = []; + this.isEditing = false; + } + + public edit(editData: any): void { + if (editData.enabled) { + if (['polygon', 'polyline', 'points'].includes(editData.state.shapeType)) { + this.editData = editData; + this.initEditing(); + } else { + this.cancel(); + } + } else { + this.closeEditing(); + this.editData = editData; + } + } + + public cancel(): void { + this.release(); + this.onEditDone(null, null); + } + + get enabled(): boolean { + return this.isEditing; + } + + get shapeType(): string { + return this.editData.state.shapeType; + } + + public configure(configuration: Configuration): void { + this.autobordersEnabled = configuration.autoborders; + this.outlinedBorders = configuration.outlinedBorders || 'black'; + + if (this.editedShape) { + this.editedShape.attr('stroke', this.outlinedBorders); + } + + if (this.editLine) { + this.editLine.attr('stroke', this.outlinedBorders); + if (this.autobordersEnabled) { + this.autoborderHandler.autoborder(true, this.editLine, this.editData.state.clientID); + } else { + this.autoborderHandler.autoborder(false); + } + } + this.controlPointsSize = configuration.controlPointsSize || consts.BASE_POINT_SIZE; + this.intelligentCutEnabled = configuration.intelligentPolygonCrop; + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + + if (this.editedShape) { + this.editedShape.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale, + }); + } + + if (this.editLine) { + if (this.editData.state.shapeType !== 'points') { + this.editLine.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale, + }); + } + + const paintHandler = this.editLine.remember('_paintHandler'); + for (const point of paintHandler.set.members) { + point.attr('stroke-width', `${consts.POINTS_STROKE_WIDTH / geometry.scale}`); + point.attr('r', `${this.controlPointsSize / geometry.scale}`); + } + } + } +} diff --git a/cvat-canvas/src/typescript/groupHandler.ts b/cvat-canvas/src/typescript/groupHandler.ts new file mode 100644 index 0000000..ffa1a26 --- /dev/null +++ b/cvat-canvas/src/typescript/groupHandler.ts @@ -0,0 +1,75 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { GroupData } from './canvasModel'; +import { ObjectSelector, SelectionFilter } from './objectSelector'; + +export interface GroupHandler { + group(groupData: GroupData, selectionFilter: SelectionFilter): void; + select(state: any): void; + cancel(): void; +} + +export class GroupHandlerImpl implements GroupHandler { + private onSelectDone: (objects?: any[], duration?: number) => void; + private selector: ObjectSelector; + private initialized: boolean; + private statesToBeGrouped: any[]; + private startTimestamp: number; + + private release(): void { + this.selector.disable(); + this.initialized = false; + } + + private initGrouping(selectionFilter: SelectionFilter): void { + this.statesToBeGrouped = []; + this.selector.enable((selected) => { + this.statesToBeGrouped = selected; + }, selectionFilter); + this.initialized = true; + this.startTimestamp = Date.now(); + } + + private closeGrouping(): void { + if (this.initialized) { + const { statesToBeGrouped } = this; + this.release(); + if (statesToBeGrouped.length) { + this.onSelectDone(statesToBeGrouped, Date.now() - this.startTimestamp); + } else { + this.onSelectDone(); + } + } + } + + public constructor( + onSelectDone: GroupHandlerImpl['onSelectDone'], + selector: ObjectSelector, + ) { + this.onSelectDone = onSelectDone; + this.selector = selector; + this.statesToBeGrouped = []; + this.initialized = false; + this.startTimestamp = Date.now(); + } + + public group(groupData: GroupData, selectionFilter: SelectionFilter): void { + if (groupData.enabled) { + this.initGrouping(selectionFilter); + } else { + this.closeGrouping(); + } + } + + public select(objectState: any): void { + this.selector.push(objectState); + } + + public cancel(): void { + this.release(); + this.onSelectDone(); + } +} diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts new file mode 100644 index 0000000..5d7e2b6 --- /dev/null +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -0,0 +1,631 @@ +// Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import _ from 'lodash'; +import * as SVG from 'svg.js'; +import consts from './consts'; +import Crosshair from './crosshair'; +import { + stringifyPoints, translateToCanvas, RLEToImageData, + imageDataToDataURL, translateFromCanvas, translateToSVG, + clamp, +} from './shared'; +import { + InteractionData, InteractionResult, Geometry, + Configuration, CanvasHint, +} from './canvasModel'; + +export interface InteractionHandler { + transform(geometry: Geometry): void; + interact(interactData: InteractionData): void; + configure(config: Configuration): void; + destroy(): void; + cancel(): void; +} + +type SupportedShapes = SVG.Rect | SVG.Circle; + +const DELETE_BUTTON_OFFSET = 8; + +function getTopRightPosition(shape: SVG.Rect | SVG.Circle): { x: number; y: number } { + if (shape instanceof SVG.Rect) { + return { + x: shape.x() + shape.width(), + y: shape.y(), + }; + } + + if (shape instanceof SVG.Circle) { + return { + x: shape.cx() + shape.attr('r'), + y: shape.cy() - shape.attr('r'), + }; + } + + throw new Error('Unsupported shape type'); +} + +function deleteButtonPath(r: number): string { + const p = [3, 7].map((val) => (val / 10) * r * 2); + return `M ${p[0]} ${p[0]} L ${p[1]} ${p[1]} M ${p[1]} ${p[0]} L ${p[0]} ${p[1]}`; +} + +export class InteractionHandlerImpl implements InteractionHandler { + private settings: Omit, 'hint' | 'regionOfInterest'>; + private enabled: boolean; + private command: 'draw_box' | 'draw_points' | 'put_shapes' | 'refine' | 'idle'; + private currentRectangle: SVG.Rect | null; + private rectanglePrompts: SVG.Rect[]; + private pointPrompts: SVG.Circle[]; + private allPrompts: SupportedShapes[]; + private deletionButtons: Map; + private intermediateShapes: (SVG.Image | SVG.Polygon)[]; + private onInteraction: (interactionResult: InteractionResult[], finished?: boolean) => void; + private onMessage: (messages: CanvasHint[] | null, topic: string) => void; + private geometry: Geometry; + private container: SVG.Container; + private configuration: Configuration; + private effectiveStrokeWidth: number; + private effectivePointSize: number; + private effectiveShapeOpacity: number; + private lastMousePosition: { x: number; y: number }; + private regionOfInterest: InteractionData['settings']['regionOfInterest'] | null; + private crosshair: Crosshair; + + public constructor( + onInteraction: InteractionHandlerImpl['onInteraction'], + onMessage: InteractionHandlerImpl['onMessage'], + adoptedContent: SVG.Container, + geometry: Geometry, + configuration: Configuration, + ) { + this.onInteraction = onInteraction; + this.onMessage = onMessage; + this.enabled = false; + this.command = 'idle'; + this.crosshair = new Crosshair(); + this.regionOfInterest = null; + this.settings = { + crosshair: false, + points_type: 'any', + removalStrategy: 'any', + appendCursorPositionAsPoint: false, + }; + this.container = adoptedContent; + this.geometry = geometry; + this.configuration = configuration; + this.currentRectangle = null; + this.rectanglePrompts = []; + this.pointPrompts = []; + this.allPrompts = []; + this.intermediateShapes = []; + this.deletionButtons = new Map(); + this.effectiveStrokeWidth = consts.BASE_STROKE_WIDTH / this.geometry.scale; + this.effectivePointSize = (configuration.controlPointsSize ?? consts.BASE_POINT_SIZE) / this.geometry.scale; + this.effectiveShapeOpacity = configuration.selectedShapeOpacity ?? 0.5; + this.lastMousePosition = { x: 0, y: 0 }; + this.container.on('mousedown.interaction', this.onMouseDown); + this.container.on('mousemove.interaction', this.onMouseMove); + } + + private clearCurrentRectangle(): void { + if (this.currentRectangle) { + this.currentRectangle.off('drawstop.interaction'); + this.currentRectangle.remove(); + this.currentRectangle = null; + } + } + + private clearPromptsAndButtons(): void { + this.rectanglePrompts.forEach((rect) => { + rect.remove(); + }); + + this.pointPrompts.forEach((point) => { + point.off('mouseenter.interaction'); + point.off('mouseleave.interaction'); + point.off('mousedown.interaction'); + point.remove(); + }); + + this.deletionButtons.forEach((deleteBtn) => { + deleteBtn.off('mouseover.interaction'); + deleteBtn.off('mouseout.interaction'); + deleteBtn.off('mousedown.interaction'); + deleteBtn.remove(); + }); + + this.deletionButtons.clear(); + this.rectanglePrompts = []; + this.pointPrompts = []; + this.allPrompts = []; + } + + private clearIntermediateShapes(): void { + this.intermediateShapes.forEach((shape) => { + if (shape.node instanceof SVGImageElement) { + URL.revokeObjectURL(shape.node.href.baseVal); + } + shape.remove(); + }); + this.intermediateShapes = []; + } + + private release(): void { + this.enabled = false; + this.command = 'idle'; + this.regionOfInterest = null; + this.onMessage(null, 'interaction'); + this.clearCurrentRectangle(); + this.clearPromptsAndButtons(); + this.clearIntermediateShapes(); + this.crosshair.hide(); + } + + private isWithinInteractionBounds(x: number, y: number): boolean { + const { offset, image } = this.geometry; + const { regionOfInterest } = this; + const [imageX, imageY] = [x - offset, y - offset]; + + if (regionOfInterest) { + return ( + imageX >= regionOfInterest[0] && + imageX <= regionOfInterest[2] && + imageY >= regionOfInterest[1] && + imageY <= regionOfInterest[3] + ); + } + + return imageX >= 0 && imageX < image.width && imageY >= 0 && imageY < image.height; + } + + private drawBox(hint?: string): void { + if (this.command === 'draw_box') { + return; + } + + this.command = 'draw_box'; + const initNewDrawingBox = (): void => { + this.currentRectangle = this.container.rect() + .fill('rgba(0, 0, 0, 0)') + .stroke({ color: '#000000', width: this.effectiveStrokeWidth }) + .opacity(this.effectiveShapeOpacity) + .addClass('cvat_interaction_rectangle'); + + this.currentRectangle.on('drawstop.interaction', () => { + const rectangle = this.currentRectangle.clone() as SVG.Rect; + this.clearCurrentRectangle(); + + const { offset, image: { width: imWidth, height: imHeight } } = this.geometry; + const [x, y, width, height] = [rectangle.x(), rectangle.y(), rectangle.width(), rectangle.height()]; + const right = offset + imWidth; + const bottom = offset + imHeight; + + const { regionOfInterest } = this; + const minX = regionOfInterest ? offset + regionOfInterest[0] : offset; + const minY = regionOfInterest ? offset + regionOfInterest[1] : offset; + const maxX = regionOfInterest ? offset + regionOfInterest[2] : right; + const maxY = regionOfInterest ? offset + regionOfInterest[3] : bottom; + const xtl = clamp(x, minX, maxX); + const ytl = clamp(y, minY, maxY); + const xbr = clamp(x + width, minX, maxX); + const ybr = clamp(y + height, minY, maxY); + + if (xbr - xtl < 1 || ybr - ytl < 1) { + rectangle.remove(); + initNewDrawingBox(); + return; + } + + rectangle.x(xtl); + rectangle.y(ytl); + rectangle.width(xbr - xtl); + rectangle.height(ybr - ytl); + + this.rectanglePrompts.push(rectangle); + this.allPrompts.push(rectangle); + this.drawDeleteButton(rectangle); + + initNewDrawingBox(); + this.notify(); + }); + }; + + this.onMessage([{ + type: 'text', + icon: 'info', + content: hint ?? 'Draw rectangle prompts', + }, { + type: 'list', + content: [ + 'Hold to drag the image', + ], + className: 'cvat-canvas-notification-list-shortcuts', + }], 'interaction'); + + initNewDrawingBox(); + } + + private drawPoints(hint?: string): void { + const { points_type: pointsType } = this.settings; + + this.command = 'draw_points'; + const textPrompts = []; + if (pointsType === 'any') { + textPrompts.push( + 'Click to add a positive point', + 'Click to add a negative point', + ); + } + + this.onMessage([{ + type: 'text', + icon: 'info', + content: hint ?? 'Draw point prompts', + }, { + type: 'list', + content: [ + ...textPrompts, + 'Hold to drag the image', + ], + className: 'cvat-canvas-notification-list-shortcuts', + }], 'interaction'); + } + + private putShapes(shapes: InteractionData['payload']['shapes']): void { + this.clearIntermediateShapes(); + + for (const shape of shapes) { + const { points, shapeType } = shape; + if (shapeType === 'polygon') { + const isInvalidShape = points.length < 3 * 2; + const polygon = this.container + .polygon(stringifyPoints(translateToCanvas(this.geometry.offset, points))) + .attr({ + 'color-rendering': 'optimizeQuality', + 'shape-rendering': 'geometricprecision', + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + stroke: isInvalidShape ? 'red' : 'black', + }) + .fill({ opacity: this.effectiveShapeOpacity, color: 'white' }) + .addClass('cvat_canvas_interact_intermediate_shape'); + this.container.node.prepend(polygon.node); + this.intermediateShapes.push(polygon); + } else if (shapeType === 'mask') { + const left = points[points.length - 4]; + const top = points[points.length - 3]; + const right = points[points.length - 2]; + const bottom = points[points.length - 1]; + const imageBitmap = RLEToImageData(255, 255, 255, points); + const image = this.container.image().attr({ + 'color-rendering': 'optimizeQuality', + 'shape-rendering': 'geometricprecision', + 'pointer-events': 'none', + opacity: 0.5, + }).addClass('cvat_canvas_interact_intermediate_shape'); + image.move(this.geometry.offset + left, this.geometry.offset + top); + this.container.node.prepend(image.node); + this.intermediateShapes.push(image); + + imageDataToDataURL( + imageBitmap, + right - left + 1, + bottom - top + 1, + (dataURL: string) => { + const destroy = (): void => URL.revokeObjectURL(dataURL); + if (image.parent() !== null) { + // still in DOM + image.loaded(destroy); + image.error(destroy); + image.load(dataURL); + } else { + destroy(); + } + }, + ); + } + } + } + + private refine(): void { + // TODO: implement + } + + private notify(finished = false, extras: InteractionResult[] = []): void { + const transformed = this.allPrompts.map((shape) => { + if (shape instanceof SVG.Rect) { + const [x, y] = [shape.x(), shape.y()]; + const [width, height] = [shape.width(), shape.height()]; + return { + points: translateFromCanvas(this.geometry.offset, [x, y, x + width, y + height]), + shapeType: 'rectangle', + type: 'positive' as const, + }; + } + + if (shape instanceof SVG.Circle) { + return { + points: translateFromCanvas(this.geometry.offset, [shape.cx(), shape.cy()]), + shapeType: 'points', + type: shape.attr('stroke') === 'green' ? 'positive' as const : 'negative' as const, + }; + } + + throw new Error('Unknown shape type'); + }); + + this.onInteraction(transformed.concat(extras), finished); + } + + private deleteShape(shape: SupportedShapes): void { + const shapeIndex = this.allPrompts.indexOf(shape); + if (shapeIndex > -1) { + this.allPrompts.splice(shapeIndex, 1); + } + + if (shape instanceof SVG.Circle) { + const pointIndex = this.pointPrompts.findIndex((p) => p === shape); + if (pointIndex !== -1) { + this.pointPrompts.splice(pointIndex, 1); + } + } else if (shape instanceof SVG.Rect) { + const rectIndex = this.rectanglePrompts.findIndex((r) => r === shape); + if (rectIndex !== -1) { + this.rectanglePrompts.splice(rectIndex, 1); + } + } + + // Remove delete button from map and canvas + const deleteBtn = this.deletionButtons.get(shape); + if (deleteBtn) { + deleteBtn.remove(); + this.deletionButtons.delete(shape); + } + + shape.remove(); + this.notify(); + } + + private drawDeleteButton(shape: SVG.Rect | SVG.Circle): void { + const { scale } = this.geometry; + const r = consts.BASE_POINT_SIZE / scale; + const { x, y } = getTopRightPosition(shape); + + const deleteButtonGroup = this.container.group().addClass('cvat_interaction_delete_button') as SVG.G; + const circleBg = deleteButtonGroup.circle(r * 2) + .fill('#ff3333') + .stroke({ color: '#ffffff', width: this.effectiveStrokeWidth }) + .center(x + DELETE_BUTTON_OFFSET / scale, y - DELETE_BUTTON_OFFSET / scale); + + const p = [3, 7].map((val) => (val / 10) * r * 2); + deleteButtonGroup.path(`M ${p[0]} ${p[0]} L ${p[1]} ${p[1]} M ${p[1]} ${p[0]} L ${p[0]} ${p[1]}`) + .stroke({ color: '#ffffff', width: this.effectiveStrokeWidth }) + .center(x + DELETE_BUTTON_OFFSET / scale, y - DELETE_BUTTON_OFFSET / scale).fill('none'); + + deleteButtonGroup.on('mouseover.interaction', () => { + circleBg.stroke({ color: '#ffffff', width: this.effectiveStrokeWidth * 1.5 }); + }); + + deleteButtonGroup.on('mouseout.interaction', () => { + circleBg.stroke({ color: '#ffffff', width: this.effectiveStrokeWidth }); + }); + + deleteButtonGroup.on('mousedown.interaction', (e: any) => { + e.stopPropagation(); + this.deleteShape(shape); + }); + + this.deletionButtons.set(shape, deleteButtonGroup); + } + + private onMouseMove = (e: MouseEvent): void => { + const [x, y] = translateToSVG( + this.container.node as unknown as SVGSVGElement, + [e.clientX, e.clientY], + ); + this.lastMousePosition = { x, y }; + this.crosshair.move(x, y); + + if ( + this.command === 'draw_points' && + this.settings.appendCursorPositionAsPoint && + this.pointPrompts.length && + this.isWithinInteractionBounds(x, y) + ) { + const lastPoint = this.pointPrompts[this.pointPrompts.length - 1]; + const [cx, cy] = [lastPoint.cx(), lastPoint.cy()]; + const threshold = 5; + if (Math.hypot(cx - x, cy - y) > threshold) { + this.notify(false, [{ + points: translateFromCanvas(this.geometry.offset, [x, y]), + shapeType: 'points', + type: 'positive' as const, + }]); + } + } + }; + + private onMouseDown = (e: MouseEvent): void => { + if (!(this.enabled && [0, 2].includes(e.button))) { + return; + } + + const [x, y] = translateToSVG( + this.container.node as unknown as SVGSVGElement, + [e.clientX, e.clientY], + ); + if (this.command === 'draw_box') { + (this.currentRectangle as any).draw(e); + } else if (this.command === 'draw_points') { + if (!this.isWithinInteractionBounds(x, y)) { + return; + } + + let color = 'green'; + if (this.settings.points_type === 'any') { + color = e.button === 0 ? 'green' : 'red'; + } else if (this.settings.points_type === 'positive') { + color = 'green'; + } else if (this.settings.points_type === 'negative') { + color = 'red'; + } + + const point = this.container.circle(this.effectivePointSize * 2) + .fill('white') + .stroke(color) + .addClass('cvat_interaction_point') + .center(x, y); + + const pointCanBeRemoved = (): boolean => ( + this.settings.removalStrategy === 'any' || ( + this.settings.removalStrategy === 'last' && + this.allPrompts.indexOf(point) === this.allPrompts.length - 1 + ) + ); + + point.on('mouseenter.interaction', () => { + if (pointCanBeRemoved()) { + point.addClass('cvat_canvas_removable_interaction_point'); + point.attr({ 'stroke-width': this.effectiveStrokeWidth * 1.5, r: this.effectivePointSize * 1.1 }); + } + }); + + point.on('mouseleave.interaction', (): void => { + point.removeClass('cvat_canvas_removable_interaction_point'); + point.attr({ 'stroke-width': this.effectiveStrokeWidth, r: this.effectivePointSize }); + }); + + point.on('mousedown.interaction', (_e: MouseEvent): void => { + _e.preventDefault(); + _e.stopPropagation(); + if (pointCanBeRemoved()) { + this.deleteShape(point); + } + }); + + this.pointPrompts.push(point); + this.allPrompts.push(point); + this.drawDeleteButton(point); + this.notify(); + } + }; + + public transform(geometry: Geometry): void { + this.geometry = geometry; + this.effectiveStrokeWidth = consts.BASE_STROKE_WIDTH / this.geometry.scale; + this.effectivePointSize = ( + this.configuration.controlPointsSize ?? consts.BASE_POINT_SIZE + ) / this.geometry.scale; + + if (this.currentRectangle) { + this.currentRectangle.stroke({ width: this.effectiveStrokeWidth }); + this.currentRectangle.opacity(this.effectiveShapeOpacity); + } + + this.allPrompts.forEach((shape) => { + shape.stroke({ width: this.effectiveStrokeWidth }); + if (shape instanceof SVG.Rect) { + shape.opacity(this.effectiveShapeOpacity); + } else if (shape instanceof SVG.Circle) { + shape.attr('r', this.effectivePointSize); + } + }); + + this.deletionButtons.forEach((deleteButtonGroup, shape) => { + const { scale } = this.geometry; + const group = deleteButtonGroup; + + const { x, y } = getTopRightPosition(shape); + const r = consts.BASE_POINT_SIZE / scale; + group.children().forEach((child) => { + if (child instanceof SVG.Circle) { + child.attr('r', r); + child.stroke({ color: '#ffffff', width: this.effectiveStrokeWidth }); + } else if (child instanceof SVG.Path) { + (child as SVG.Path).plot(deleteButtonPath(r)); + child.stroke({ color: '#ffffff', width: this.effectiveStrokeWidth }); + } + child.center(x + DELETE_BUTTON_OFFSET / scale, y - DELETE_BUTTON_OFFSET / scale); + }); + }); + + this.intermediateShapes.forEach((shape) => { + shape.fill({ opacity: this.effectiveShapeOpacity }); + }); + } + + public interact(interactData: InteractionData): void { + if (Object.hasOwn(interactData, 'settings')) { + this.settings = { + ...this.settings, + ..._.omit(interactData.settings, ['hint', 'regionOfInterest']), + }; + } + + if (interactData.enabled) { + this.enabled = true; + } else if (this.enabled) { + this.notify(true); + this.onInteraction(null); + this.release(); + } + + if (this.enabled) { + if (this.settings.crosshair) { + this.crosshair.show( + this.container, + this.lastMousePosition.x, + this.lastMousePosition.y, + this.geometry.scale, + ); + } else { + this.crosshair.hide(); + } + + if (this.settings.removalStrategy === 'last') { + this.deletionButtons.forEach((deleteBtn) => deleteBtn.hide()); + const lastShape = this.allPrompts?.[this.allPrompts.length - 1]; + if (lastShape) { + this.deletionButtons.get(lastShape)?.show(); + } + } else { + this.deletionButtons.forEach((deleteBtn) => deleteBtn.show()); + } + + const { command } = interactData; + if (['draw_box', 'draw_points'].includes(command)) { + const regionOfInterest = interactData.settings?.regionOfInterest; + this.regionOfInterest = regionOfInterest ? [...regionOfInterest] : null; + if (command === 'draw_box') { + this.drawBox(interactData.settings?.hint); + } else { + this.clearCurrentRectangle(); + this.drawPoints(interactData.settings?.hint); + } + } else if (command === 'put_shapes') { + this.putShapes(interactData.payload.shapes); + } else if (command === 'refine') { + this.refine(); + } + } + } + + public configure(config: Configuration): void { + this.configuration = config; + this.effectivePointSize = (config.controlPointsSize ?? consts.BASE_POINT_SIZE) / this.geometry.scale; + this.effectiveShapeOpacity = config.selectedShapeOpacity ?? 0.5; + } + + public destroy(): void { + this.container.off('mousedown.interaction', this.onMouseDown); + this.container.off('mousemove.interaction', this.onMouseMove); + this.release(); + } + + public cancel(): void { + this.onInteraction(null); + this.release(); + } +} diff --git a/cvat-canvas/src/typescript/masksHandler.ts b/cvat-canvas/src/typescript/masksHandler.ts new file mode 100644 index 0000000..67382ef --- /dev/null +++ b/cvat-canvas/src/typescript/masksHandler.ts @@ -0,0 +1,752 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { fabric } from 'fabric'; +import debounce from 'lodash/debounce'; + +import { + DrawData, MasksEditData, Geometry, Configuration, BrushTool, ColorBy, Position, +} from './canvasModel'; +import consts from './consts'; +import { DrawHandler } from './drawHandler'; +import { + PropType, computeWrappingBox, imageDataToRLE, RLEToImageData, imageDataToDataURL, +} from './shared'; + +interface WrappingBBox { + left: number; + top: number; + right: number; + bottom: number; +} + +export interface MasksHandler { + draw(drawData: DrawData): void; + edit(state: MasksEditData): void; + configure(configuration: Configuration): void; + transform(geometry: Geometry): void; + cancel(): void; + enabled: boolean; +} + +export class MasksHandlerImpl implements MasksHandler { + private onDrawDone: ( + data: object | null, + duration?: number, + continueDraw?: boolean, + prevDrawData?: DrawData, + ) => void; + private onDrawRepeat: (data: DrawData) => void; + private onEditStart: (state: any) => void; + private onEditDone: (state: any, points: number[]) => void; + private vectorDrawHandler: DrawHandler; + + private redraw: number | null; + private isDrawing: boolean; + private isEditing: boolean; + private isInsertion: boolean; + private isMouseDown: boolean; + private isBrushSizeChanging: boolean; + private resizeBrushToolLatestX: number; + private brushMarker: fabric.Rect | fabric.Circle | null; + private drawablePolygon: null | fabric.Polygon; + private isPolygonDrawing: boolean; + private drawnObjects: (fabric.Polygon | fabric.Circle | fabric.Rect | fabric.Line | fabric.Image)[]; + + private tool: DrawData['brushTool'] | null; + private drawData: DrawData | null; + private canvas: fabric.Canvas; + + private editData: MasksEditData | null; + + private colorBy: ColorBy; + private latestMousePos: Position; + private startTimestamp: number; + private geometry: Geometry; + private drawingOpacity: number; + private isHidden: boolean; + + private keepDrawnPolygon(): void { + const canvasWrapper = this.canvas.getElement().parentElement; + canvasWrapper.style.pointerEvents = ''; + canvasWrapper.style.zIndex = ''; + this.isPolygonDrawing = false; + this.vectorDrawHandler.draw({ enabled: false }, this.geometry); + } + + private removeBrushMarker(): void { + if (this.brushMarker) { + this.canvas.remove(this.brushMarker); + this.brushMarker = null; + this.canvas.renderAll(); + } + } + + private setupBrushMarker(): void { + if (['brush', 'eraser'].includes(this.tool.type)) { + const common = { + evented: false, + selectable: false, + opacity: 0.75, + left: this.latestMousePos.x - this.tool.size / 2, + top: this.latestMousePos.y - this.tool.size / 2, + strokeWidth: 1, + stroke: 'white', + }; + this.brushMarker = this.tool.form === 'circle' ? new fabric.Circle({ + ...common, + radius: Math.round(this.tool.size / 2), + }) : new fabric.Rect({ + ...common, + width: this.tool.size, + height: this.tool.size, + }); + + this.canvas.defaultCursor = 'none'; + this.canvas.add(this.brushMarker); + } else { + this.canvas.defaultCursor = 'inherit'; + } + } + + private releaseCanvasWrapperCSS(): void { + const canvasWrapper = this.canvas.getElement().parentElement; + canvasWrapper.style.pointerEvents = ''; + canvasWrapper.style.zIndex = ''; + canvasWrapper.style.display = ''; + } + + private releasePaste(): void { + this.releaseCanvasWrapperCSS(); + this.canvas.clear(); + this.canvas.renderAll(); + this.isInsertion = false; + this.drawnObjects = this.createDrawnObjectsArray(); + this.onDrawDone(null); + } + + private releaseDraw(): void { + this.removeBrushMarker(); + this.releaseCanvasWrapperCSS(); + if (this.isPolygonDrawing) { + this.isPolygonDrawing = false; + this.vectorDrawHandler.cancel(); + } + this.canvas.clear(); + this.canvas.renderAll(); + this.isDrawing = false; + this.isInsertion = false; + this.redraw = null; + this.drawnObjects = this.createDrawnObjectsArray(); + } + + private releaseEdit(): void { + this.removeBrushMarker(); + this.releaseCanvasWrapperCSS(); + if (this.isPolygonDrawing) { + this.isPolygonDrawing = false; + this.vectorDrawHandler.cancel(); + } + this.canvas.clear(); + this.canvas.renderAll(); + this.isEditing = false; + this.drawnObjects = this.createDrawnObjectsArray(); + this.onEditDone(null, null); + } + + private getStateColor(state: any): string { + if (this.colorBy === ColorBy.INSTANCE) { + return state.color; + } + + if (this.colorBy === ColorBy.LABEL) { + return state.label.color; + } + + return state.group.color; + } + + private getDrawnObjectsWrappingBox(): WrappingBBox { + type BoundingRect = ReturnType>; + type TwoCornerBox = Pick & { right: number; bottom: number }; + const { width, height } = this.geometry.image; + const wrappingBbox = this.drawnObjects + .map((obj) => { + if (obj instanceof fabric.Polygon) { + const bbox = computeWrappingBox(obj.points + .reduce(((acc, val) => { + acc.push(val.x, val.y); + return acc; + }), [])); + + return { + left: bbox.xtl, + top: bbox.ytl, + width: bbox.width, + height: bbox.height, + }; + } + + if (obj instanceof fabric.Image) { + return { + left: obj.left, + top: obj.top, + width: obj.width, + height: obj.height, + }; + } + + return obj.getBoundingRect(); + }) + .reduce((acc: TwoCornerBox, rect: BoundingRect) => { + acc.top = Math.floor(Math.max(0, Math.min(rect.top, acc.top))); + acc.left = Math.floor(Math.max(0, Math.min(rect.left, acc.left))); + acc.bottom = Math.floor(Math.min(height - 1, Math.max(rect.top + rect.height, acc.bottom))); + acc.right = Math.floor(Math.min(width - 1, Math.max(rect.left + rect.width, acc.right))); + return acc; + }, { + left: Number.MAX_SAFE_INTEGER, + top: Number.MAX_SAFE_INTEGER, + right: Number.MIN_SAFE_INTEGER, + bottom: Number.MIN_SAFE_INTEGER, + }); + + return wrappingBbox; + } + + private imageDataFromCanvas(wrappingBBox: WrappingBBox): Uint8ClampedArray { + const imageData = this.canvas.toCanvasElement() + .getContext('2d').getImageData( + wrappingBBox.left, + wrappingBBox.top, + wrappingBBox.right - wrappingBBox.left + 1, + wrappingBBox.bottom - wrappingBBox.top + 1, + ).data; + return imageData; + } + + private updateHidden(value: boolean): void { + this.isHidden = value; + + // Need to update style of upper canvas explicitly because update of default cursor is not applied immediately + // https://github.com/fabricjs/fabric.js/issues/1456 + const newOpacity = value ? '0' : ''; + const newCursor = value ? 'inherit' : 'none'; + this.canvas.getElement().parentElement.style.opacity = newOpacity; + const upperCanvas = this.canvas.getElement().parentElement.querySelector('.upper-canvas') as HTMLElement; + if (upperCanvas) { + upperCanvas.style.cursor = newCursor; + } + this.canvas.defaultCursor = newCursor; + } + + private updateBrushTools(brushTool?: BrushTool, opts: Partial = {}): void { + if (this.isPolygonDrawing) { + // tool was switched from polygon to brush for example + this.keepDrawnPolygon(); + } + + this.removeBrushMarker(); + if (brushTool) { + if (brushTool.color && this.tool?.color !== brushTool.color) { + const color = fabric.Color.fromHex(brushTool.color); + for (const object of this.drawnObjects) { + if (object instanceof fabric.Line) { + const alpha = +object.stroke.split(',')[3].slice(0, -1); + color.setAlpha(alpha); + object.set({ stroke: color.toRgba() }); + } else if ( + object instanceof fabric.Rect || + object instanceof fabric.Polygon || + object instanceof fabric.Circle + ) { + const alpha = +(object.fill as string).split(',')[3].slice(0, -1); + color.setAlpha(alpha); + (object as fabric.Object).set({ fill: color.toRgba() }); + } + } + this.canvas.renderAll(); + } + + this.tool = { ...brushTool, ...opts }; + if (this.isDrawing || this.isEditing) { + this.setupBrushMarker(); + } + + this.updateBlockedTools(); + } + + if (this.tool?.type?.startsWith('polygon-')) { + this.isPolygonDrawing = true; + this.vectorDrawHandler.draw({ + enabled: true, + shapeType: 'polygon', + onDrawDone: (data: { points: number[] } | null) => { + if (!data) return; + const points = data.points.reduce((acc: fabric.Point[], _: number, idx: number) => { + if (idx % 2) { + acc.push(new fabric.Point(data.points[idx - 1], data.points[idx])); + } + + return acc; + }, []); + + const color = fabric.Color.fromHex(this.tool.color); + color.setAlpha(this.tool.type === 'polygon-minus' ? 1 : this.drawingOpacity); + const polygon = new fabric.Polygon(points, { + fill: color.toRgba(), + selectable: false, + objectCaching: false, + absolutePositioned: true, + globalCompositeOperation: this.tool.type === 'polygon-minus' ? 'destination-out' : 'xor', + }); + + this.canvas.add(polygon); + this.drawnObjects.push(polygon); + this.canvas.renderAll(); + }, + }, this.geometry); + + const canvasWrapper = this.canvas.getElement().parentElement as HTMLDivElement; + canvasWrapper.style.pointerEvents = 'none'; + canvasWrapper.style.zIndex = '0'; + } + } + + private updateBlockedTools(): void { + if (this.drawnObjects.length === 0) { + this.tool.onBlockUpdated({ + eraser: true, + 'polygon-minus': true, + }); + return; + } + const wrappingBbox = this.getDrawnObjectsWrappingBox(); + if (this.brushMarker) { + this.canvas.remove(this.brushMarker); + } + const imageData = this.imageDataFromCanvas(wrappingBbox); + if (this.brushMarker) { + this.canvas.add(this.brushMarker); + } + const rle = imageDataToRLE(imageData); + const isEmptyMask = rle.length < 2; + this.tool.onBlockUpdated({ + eraser: isEmptyMask, + 'polygon-minus': isEmptyMask, + }); + } + + private createDrawnObjectsArray(): MasksHandlerImpl['drawnObjects'] { + const drawnObjects = []; + const updateBlockedToolsDebounced = debounce(this.updateBlockedTools.bind(this), 250); + return new Proxy(drawnObjects, { + set(target, property, value) { + // eslint-disable-next-line no-param-reassign + target[property] = value; + updateBlockedToolsDebounced(); + return true; + }, + }); + } + + public constructor( + onDrawDone: MasksHandlerImpl['onDrawDone'], + onDrawRepeat: MasksHandlerImpl['onDrawRepeat'], + onEditStart: MasksHandlerImpl['onEditStart'], + onEditDone: MasksHandlerImpl['onEditDone'], + vectorDrawHandler: DrawHandler, + canvas: HTMLCanvasElement, + ) { + this.redraw = null; + this.isDrawing = false; + this.isEditing = false; + this.isMouseDown = false; + this.isBrushSizeChanging = false; + this.isPolygonDrawing = false; + this.drawData = null; + this.editData = null; + this.drawingOpacity = 0.5; + this.brushMarker = null; + this.isHidden = false; + this.colorBy = ColorBy.LABEL; + this.onDrawDone = onDrawDone; + this.onDrawRepeat = onDrawRepeat; + this.onEditDone = onEditDone; + this.onEditStart = onEditStart; + this.vectorDrawHandler = vectorDrawHandler; + this.canvas = new fabric.Canvas(canvas, { + containerClass: 'cvat_masks_canvas_wrapper', + fireRightClick: true, + selection: false, + defaultCursor: 'inherit', + }); + this.canvas.imageSmoothingEnabled = false; + this.drawnObjects = this.createDrawnObjectsArray(); + + this.canvas.getElement().parentElement.addEventListener('contextmenu', (e: MouseEvent) => e.preventDefault()); + this.latestMousePos = { x: -1, y: -1 }; + window.document.addEventListener('mouseup', () => { + this.isMouseDown = false; + this.isBrushSizeChanging = false; + }); + + this.canvas.on('mouse:down', (options: fabric.IEvent) => { + const { isDrawing, isEditing, isInsertion } = this; + this.isMouseDown = (isDrawing || isEditing) && options.e.button === 0 && !options.e.altKey; + this.isBrushSizeChanging = (isDrawing || isEditing) && options.e.button === 2 && options.e.altKey; + + if (isInsertion) { + const continueInserting = options.e.ctrlKey; + const wrappingBbox = this.getDrawnObjectsWrappingBox(); + const imageData = this.imageDataFromCanvas(wrappingBbox); + const rle = imageDataToRLE(imageData); + rle.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom); + + this.onDrawDone({ + occluded: this.drawData.initialState.occluded, + attributes: { ...this.drawData.initialState.attributes }, + color: this.drawData.initialState.color, + objectType: this.drawData.initialState.objectType, + shapeType: this.drawData.shapeType, + points: rle, + label: this.drawData.initialState.label, + }, Date.now() - this.startTimestamp, continueInserting, this.drawData); + + if (!continueInserting) { + this.releasePaste(); + } + } else { + this.canvas.fire('mouse:move', options); + } + }); + + this.canvas.on('mouse:move', (e: fabric.IEvent) => { + const { image: { width: imageWidth, height: imageHeight } } = this.geometry; + const { angle } = this.geometry; + let [x, y] = [e.pointer.x, e.pointer.y]; + if (angle === 180) { + [x, y] = [imageWidth - x, imageHeight - y]; + } else if (angle === 270) { + [x, y] = [imageWidth - (y / imageHeight) * imageWidth, (x / imageWidth) * imageHeight]; + } else if (angle === 90) { + [x, y] = [(y / imageHeight) * imageWidth, imageHeight - (x / imageWidth) * imageHeight]; + } + + const position = { x, y }; + const { + tool, isMouseDown, isInsertion, isBrushSizeChanging, + } = this; + + if (isInsertion) { + const [object] = this.drawnObjects; + if (object && object instanceof fabric.Image) { + object.left = position.x - object.width / 2; + object.top = position.y - object.height / 2; + this.canvas.renderAll(); + } + } + + if (isBrushSizeChanging && ['brush', 'eraser'].includes(tool?.type)) { + const xDiff = e.pointer.x - this.resizeBrushToolLatestX; + let onUpdateConfiguration = null; + if (this.isDrawing) { + onUpdateConfiguration = this.drawData.onUpdateConfiguration; + } else if (this.isEditing) { + onUpdateConfiguration = this.editData.onUpdateConfiguration; + } + if (onUpdateConfiguration) { + onUpdateConfiguration({ + brushTool: { + size: Math.trunc(Math.max(1, this.tool.size + xDiff)), + }, + }); + } + + this.resizeBrushToolLatestX = e.pointer.x; + e.e.stopPropagation(); + return; + } + + if (this.brushMarker) { + this.brushMarker.left = position.x - tool.size / 2; + this.brushMarker.top = position.y - tool.size / 2; + this.canvas.bringToFront(this.brushMarker); + this.canvas.renderAll(); + } + + if (isMouseDown && !this.isHidden && !isBrushSizeChanging && ['brush', 'eraser'].includes(tool?.type)) { + const color = fabric.Color.fromHex(tool.color); + color.setAlpha(tool.type === 'eraser' ? 1 : 0.5); + + const commonProperties = { + selectable: false, + evented: false, + globalCompositeOperation: tool.type === 'eraser' ? 'destination-out' : 'xor', + }; + + const shapeProperties = { + ...commonProperties, + fill: color.toRgba(), + left: position.x - tool.size / 2, + top: position.y - tool.size / 2, + }; + + let shape: fabric.Circle | fabric.Rect | null = null; + if (tool.form === 'circle') { + shape = new fabric.Circle({ + ...shapeProperties, + radius: Math.round(tool.size / 2), + }); + } else if (tool.form === 'square') { + shape = new fabric.Rect({ + ...shapeProperties, + width: tool.size, + height: tool.size, + }); + } + + this.canvas.add(shape); + if (['brush', 'eraser'].includes(tool?.type)) { + this.drawnObjects.push(shape); + } + + // add line to smooth the mask + if (this.latestMousePos.x !== -1 && this.latestMousePos.y !== -1) { + const dx = position.x - this.latestMousePos.x; + const dy = position.y - this.latestMousePos.y; + if (Math.sqrt(dx ** 2 + dy ** 2) > tool.size / 2) { + const line = new fabric.Line([ + this.latestMousePos.x - tool.size / 2, + this.latestMousePos.y - tool.size / 2, + position.x - tool.size / 2, + position.y - tool.size / 2, + ], { + ...commonProperties, + stroke: color.toRgba(), + strokeWidth: tool.size, + strokeLineCap: tool.form === 'circle' ? 'round' : 'square', + }); + + this.canvas.add(line); + if (['brush', 'eraser'].includes(tool?.type)) { + this.drawnObjects.push(line); + } + } + } + this.canvas.renderAll(); + } else if (tool?.type.startsWith('polygon-') && this.drawablePolygon) { + // update the polygon position + const points = this.drawablePolygon.get('points'); + if (points.length) { + points[points.length - 1].setX(e.e.offsetX); + points[points.length - 1].setY(e.e.offsetY); + } + this.canvas.renderAll(); + } + + this.latestMousePos.x = position.x; + this.latestMousePos.y = position.y; + this.resizeBrushToolLatestX = position.x; + }); + } + + public configure(configuration: Configuration): void { + this.colorBy = configuration.colorBy; + + if (this.isHidden !== configuration.hideEditedObject) { + this.updateHidden(configuration.hideEditedObject); + } + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + const { + scale, angle, image: { width, height }, top, left, + } = geometry; + + const topCanvas = this.canvas.getElement().parentElement as HTMLDivElement; + if (this.canvas.width !== width || this.canvas.height !== height) { + this.canvas.setHeight(height); + this.canvas.setWidth(width); + this.canvas.setDimensions({ width, height }); + } + + topCanvas.style.top = `${top}px`; + topCanvas.style.left = `${left}px`; + topCanvas.style.transform = `scale(${scale}) rotate(${angle}deg)`; + + if (this.drawablePolygon) { + this.drawablePolygon.set('strokeWidth', consts.BASE_STROKE_WIDTH / scale); + this.canvas.renderAll(); + } + } + + public draw(drawData: DrawData): void { + if (drawData.enabled && drawData.shapeType === 'mask') { + if (!this.isInsertion && drawData.initialState?.shapeType === 'mask') { + // initialize inserting pipeline if not started + const { points } = drawData.initialState; + const color = fabric.Color.fromHex(this.getStateColor(drawData.initialState)).getSource(); + const [left, top, right, bottom] = points.slice(-4); + const imageBitmap = RLEToImageData(color[0], color[1], color[2], points); + imageDataToDataURL( + imageBitmap, + right - left + 1, + bottom - top + 1, + (dataURL: string) => { + fabric.Image.fromURL(dataURL, (image: fabric.Image) => { + URL.revokeObjectURL(dataURL); + image.selectable = false; + image.evented = false; + image.globalCompositeOperation = 'xor'; + image.opacity = 0.5; + this.canvas.add(image); + /* + when we paste a mask, we do not need additional logic implemented + in MasksHandlerImpl::createDrawnObjectsArray.push using JS Proxy + because we will not work with any drawing tools here, and it will cause the issue + because this.tools may be undefined here + when it is used inside the push custom implementation + */ + this.drawnObjects = [image]; + this.canvas.renderAll(); + }, { left, top }); + }, + ); + + this.isInsertion = true; + } else { + this.updateBrushTools(drawData.brushTool); + if (!this.isDrawing) { + // initialize drawing pipeline if not started + this.isDrawing = true; + this.redraw = drawData.redraw || null; + } + } + + this.canvas.getElement().parentElement.style.display = 'block'; + this.startTimestamp = Date.now(); + } + + if (!drawData.enabled && this.isInsertion) { + this.releasePaste(); + } else if (!drawData.enabled && this.isDrawing) { + try { + if (this.drawnObjects.length) { + const wrappingBbox = this.getDrawnObjectsWrappingBox(); + this.removeBrushMarker(); // remove brush marker from final mask + const imageData = this.imageDataFromCanvas(wrappingBbox); + const rle = imageDataToRLE(imageData); + rle.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom); + + const isEmptyMask = rle.length < 6; + if (isEmptyMask) { + this.onDrawDone(null); + } else { + this.onDrawDone({ + shapeType: this.drawData.shapeType, + points: rle, + ...(Number.isInteger(this.redraw) ? { clientID: this.redraw } : {}), + }, Date.now() - this.startTimestamp, drawData.continue, this.drawData); + } + } else { + this.onDrawDone(null); + } + } finally { + this.releaseDraw(); + } + + if (drawData.continue) { + const newDrawData = { + ...this.drawData, + brushTool: { ...this.tool }, + ...drawData, + enabled: true, + shapeType: 'mask', + }; + + this.onDrawRepeat({ enabled: true, shapeType: 'mask' }); + this.onDrawRepeat(newDrawData); + return; + } + } + + this.drawData = drawData; + } + + public edit(editData: MasksEditData): void { + if (editData.enabled && editData.state.shapeType === 'mask') { + if (!this.isEditing) { + // start editing pipeline if not started yet + this.canvas.getElement().parentElement.style.display = 'block'; + const { points } = editData.state; + const color = fabric.Color.fromHex(this.getStateColor(editData.state)).getSource(); + const [left, top, right, bottom] = points.slice(-4); + const imageBitmap = RLEToImageData(color[0], color[1], color[2], points); + imageDataToDataURL( + imageBitmap, + right - left + 1, + bottom - top + 1, + (dataURL: string) => { + fabric.Image.fromURL(dataURL, (image: fabric.Image) => { + URL.revokeObjectURL(dataURL); + image.selectable = false; + image.evented = false; + image.globalCompositeOperation = 'xor'; + image.opacity = 0.5; + this.canvas.add(image); + this.drawnObjects.push(image); + this.canvas.renderAll(); + }, { left, top }); + }, + ); + + this.isEditing = true; + this.startTimestamp = Date.now(); + this.onEditStart(editData.state); + } + } + + this.updateBrushTools( + editData.brushTool, + editData.state ? { color: this.getStateColor(editData.state) } : {}, + ); + + if (!editData.enabled && this.isEditing) { + try { + if (this.drawnObjects.length) { + const wrappingBbox = this.getDrawnObjectsWrappingBox(); + this.removeBrushMarker(); // remove brush marker from final mask + const imageData = this.imageDataFromCanvas(wrappingBbox); + const rle = imageDataToRLE(imageData); + rle.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom); + const isEmptyMask = rle.length < 6; + if (isEmptyMask) { + this.onEditDone(null, null); + } else { + this.onEditDone(this.editData.state, rle); + } + } + } finally { + this.releaseEdit(); + } + } + this.editData = editData; + } + + get enabled(): boolean { + return this.isDrawing || this.isEditing || this.isInsertion; + } + + public cancel(): void { + if (this.isDrawing || this.isInsertion) { + this.releaseDraw(); + } + + if (this.isEditing) { + this.releaseEdit(); + } + } +} diff --git a/cvat-canvas/src/typescript/master.ts b/cvat-canvas/src/typescript/master.ts new file mode 100644 index 0000000..92fa053 --- /dev/null +++ b/cvat-canvas/src/typescript/master.ts @@ -0,0 +1,30 @@ +// Copyright (C) 2019-2022 Intel Corporation +// +// SPDX-License-Identifier: MIT + +export interface Master { + subscribe(listener: Listener): void; + notify(reason: string): void; +} + +export interface Listener { + notify(master: Master, reason: string): void; +} + +export class MasterImpl implements Master { + private listeners: Listener[]; + + public constructor() { + this.listeners = []; + } + + public subscribe(listener: Listener): void { + this.listeners.push(listener); + } + + public notify(reason: string): void { + for (const listener of this.listeners) { + listener.notify(this, reason); + } + } +} diff --git a/cvat-canvas/src/typescript/mergeHandler.ts b/cvat-canvas/src/typescript/mergeHandler.ts new file mode 100644 index 0000000..0fd39ed --- /dev/null +++ b/cvat-canvas/src/typescript/mergeHandler.ts @@ -0,0 +1,156 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import { MergeData } from './canvasModel'; + +export interface MergeHandler { + merge(mergeData: MergeData): void; + select(state: any): void; + cancel(): void; + repeatSelection(): void; +} + +export class MergeHandlerImpl implements MergeHandler { + // callback is used to notify about merging end + private onMergeDone: (objects: any[] | null, duration?: number) => void; + private onFindObject: (event: MouseEvent) => void; + private startTimestamp: number; + private canvas: SVG.Container; + private initialized: boolean; + private statesToBeMerged: any[]; // are being merged + private highlightedShapes: Record; + private constraints: { + labelID: number; + shapeType: string; + } | null; + + private addConstraints(): void { + const shape = this.statesToBeMerged[0]; + this.constraints = { + labelID: shape.label.id, + shapeType: shape.shapeType, + }; + } + + private removeConstraints(): void { + this.constraints = null; + } + + private checkConstraints(state: any): boolean { + return ( + !this.constraints || + (state.label.id === this.constraints.labelID && state.shapeType === this.constraints.shapeType) + ); + } + + private release(): void { + this.removeConstraints(); + this.canvas.node.removeEventListener('click', this.onFindObject); + for (const state of this.statesToBeMerged) { + const shape = this.highlightedShapes[state.clientID]; + shape.removeClass('cvat_canvas_shape_merging'); + } + this.statesToBeMerged = []; + this.highlightedShapes = {}; + this.initialized = false; + } + + private initMerging(): void { + this.canvas.node.addEventListener('click', this.onFindObject); + this.startTimestamp = Date.now(); + this.initialized = true; + } + + private closeMerging(): void { + if (this.initialized) { + const { statesToBeMerged } = this; + this.release(); + + if (statesToBeMerged.length > 1) { + this.onMergeDone(statesToBeMerged, Date.now() - this.startTimestamp); + } else { + this.onMergeDone(null); + // here is a cycle + // onMergeDone => controller => model => view => closeMerging + // one call of closeMerging is unuseful, but it's okey + } + } + } + + public constructor( + onMergeDone: MergeHandlerImpl['onMergeDone'], + onFindObject: MergeHandlerImpl['onFindObject'], + canvas: SVG.Container, + ) { + this.onMergeDone = onMergeDone; + this.onFindObject = onFindObject; + this.startTimestamp = Date.now(); + this.canvas = canvas; + this.statesToBeMerged = []; + this.highlightedShapes = {}; + this.constraints = null; + this.initialized = false; + } + + public merge(mergeData: MergeData): void { + if (mergeData.enabled) { + this.initMerging(); + } else { + this.closeMerging(); + } + } + + public select(objectState: any): void { + if (objectState.shapeType === 'mask') { + // masks can not be merged + return; + } + const stateIndexes = this.statesToBeMerged.map((state): number => state.clientID); + const stateFrames = this.statesToBeMerged.map((state): number => state.frame); + const includes = stateIndexes.indexOf(objectState.clientID); + if (includes !== -1) { + const shape = this.highlightedShapes[objectState.clientID]; + this.statesToBeMerged.splice(includes, 1); + if (shape) { + delete this.highlightedShapes[objectState.clientID]; + shape.removeClass('cvat_canvas_shape_merging'); + } + + if (!this.statesToBeMerged.length) { + this.removeConstraints(); + } + } else { + const shape = this.canvas.select(`#cvat_canvas_shape_${objectState.clientID}`).first(); + if (shape && this.checkConstraints(objectState) && !stateFrames.includes(objectState.frame)) { + this.statesToBeMerged.push(objectState); + this.highlightedShapes[objectState.clientID] = shape; + shape.addClass('cvat_canvas_shape_merging'); + + if (this.statesToBeMerged.length === 1) { + this.addConstraints(); + } + } + } + } + + public repeatSelection(): void { + for (const objectState of this.statesToBeMerged) { + const shape = this.canvas.select(`#cvat_canvas_shape_${objectState.clientID}`).first(); + if (shape) { + this.highlightedShapes[objectState.clientID] = shape; + shape.addClass('cvat_canvas_shape_merging'); + } + } + } + + public cancel(): void { + this.release(); + this.onMergeDone(null); + // here is a cycle + // onMergeDone => controller => model => view => closeMerging + // one call of closeMerging is unuseful, but it's okey + } +} diff --git a/cvat-canvas/src/typescript/objectSelector.ts b/cvat-canvas/src/typescript/objectSelector.ts new file mode 100644 index 0000000..dc10009 --- /dev/null +++ b/cvat-canvas/src/typescript/objectSelector.ts @@ -0,0 +1,304 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import { RLEToImageData, imageDataToDataURL, translateToSVG } from './shared'; +import { Geometry } from './canvasModel'; +import consts from './consts'; + +export interface SelectionFilter { + objectType?: string[]; + shapeType?: string[]; + maxCount?: number; + restrictToFirstSelectedType?: boolean; +} + +export interface ObjectSelector { + enable( + callback: (selected: ObjectState[]) => void, + filter?: SelectionFilter, + ): void; + transform(geometry: Geometry): void; + push(state: ObjectState): void; + disable(): void; + resetSelected(): void; +} + +export type ObjectState = any; + +export class ObjectSelectorImpl implements ObjectSelector { + private selectionFilter: SelectionFilter | null; + private canvas: SVG.Container; + private selectionRect: SVG.Rect; + private geometry: Geometry; + private isEnabled: boolean; + private mouseDownPosition: { x: number; y: number; }; + private selectedObjects: Record; + private resetAppearance: Record void>; + private findObjectOnClick: (event: MouseEvent) => void; + private getStates: () => ObjectState[]; + private onSelectCallback: (selected: ObjectState[]) => void; + + public constructor( + findObjectOnClick: (event: MouseEvent) => void, + getStates: () => ObjectState[], + geometry: Geometry, + canvas: SVG.Container, + ) { + this.findObjectOnClick = findObjectOnClick; + this.getStates = getStates; + this.geometry = geometry; + this.canvas = canvas; + this.selectionRect = null; + this.isEnabled = false; + this.selectedObjects = {}; + this.resetAppearance = {}; + this.mouseDownPosition = { x: 0, y: 0 }; + this.selectionFilter = null; + } + + private getSelectionBox(event: MouseEvent): { + xtl: number; + ytl: number; + xbr: number; + ybr: number; + } { + const point = translateToSVG((this.canvas.node as any) as SVGSVGElement, [event.clientX, event.clientY]); + return { + xtl: Math.min(this.mouseDownPosition.x, point[0]), + ytl: Math.min(this.mouseDownPosition.y, point[1]), + xbr: Math.max(this.mouseDownPosition.x, point[0]), + ybr: Math.max(this.mouseDownPosition.y, point[1]), + }; + } + + private filterObjects(states: ObjectState[]): ObjectState[] { + let count = Object.keys(this.selectedObjects).length; + const maxCount = this.selectionFilter.maxCount || Number.MAX_SAFE_INTEGER; + const filtered = []; + + let effectiveShapeTypes = this.selectionFilter.shapeType; + if ( + this.selectionFilter.restrictToFirstSelectedType && + count > 0 && + effectiveShapeTypes && + effectiveShapeTypes.length > 1 + ) { + const firstSelected = Object.values(this.selectedObjects)[0]; + effectiveShapeTypes = [firstSelected.shapeType]; + } + + for (const state of states) { + const { objectType, shapeType } = state; + const objectTypes = this.selectionFilter.objectType || [objectType]; + const shapeTypes = effectiveShapeTypes || [shapeType]; + if (objectTypes.includes(objectType) && shapeTypes.includes(shapeType)) { + if (count < maxCount) { + filtered.push(state); + count++; + } + } + } + + return filtered; + } + + private onMouseDown = (event: MouseEvent): void => { + const point = translateToSVG((this.canvas.node as any) as SVGSVGElement, [event.clientX, event.clientY]); + this.mouseDownPosition = { x: point[0], y: point[1] }; + this.selectionRect = this.canvas.rect().addClass('cvat_canvas_selection_box'); + this.selectionRect.attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale }); + this.selectionRect.attr({ ...this.mouseDownPosition }); + }; + + private onMouseUp = (event: MouseEvent): void => { + if (this.selectionRect) { + this.selectionRect.remove(); + this.selectionRect = null; + + const states = this.getStates(); + const box = this.getSelectionBox(event); + const shapes = (this.canvas.select('.cvat_canvas_shape') as any).members.filter( + (shape: SVG.Shape): boolean => !shape.hasClass('cvat_canvas_hidden'), + ); + + let newStates = []; + for (const shape of shapes) { + const bbox = shape.bbox(); + const clientID = shape.attr('clientID'); + if ( + bbox.x > box.xtl && + bbox.y > box.ytl && + bbox.x + bbox.width < box.xbr && + bbox.y + bbox.height < box.ybr && + !(clientID in this.selectedObjects) + ) { + const objectState = states.find((state: ObjectState): boolean => state.clientID === clientID); + if (objectState) { + newStates.push(objectState); + } + } + } + + newStates = this.filterObjects(newStates); + if (newStates.length) { + newStates.forEach((_state) => { + this.selectedObjects[_state.clientID] = _state; + }); + this.onSelectCallback(Object.values(this.selectedObjects)); + } + } + }; + + private onMouseMove = (event: MouseEvent): void => { + if (this.selectionRect) { + const box = this.getSelectionBox(event); + this.selectionRect.attr({ + x: box.xtl, + y: box.ytl, + width: box.xbr - box.xtl, + height: box.ybr - box.ytl, + }); + } + }; + + private resetAllAppearances(): void { + for (const clientID of Object.keys(this.resetAppearance)) { + this.resetAppearance[clientID](); + } + this.resetAppearance = {}; + } + + public enable(callback: (selected: ObjectState[]) => void, filter?: SelectionFilter): void { + if (!this.isEnabled) { + window.document.addEventListener('mouseup', this.onMouseUp); + this.canvas.node.addEventListener('mousedown', this.onMouseDown); + this.canvas.node.addEventListener('mousemove', this.onMouseMove); + this.canvas.node.addEventListener('click', this.findObjectOnClick); + + this.selectedObjects = {}; + this.onSelectCallback = (_selected: ObjectState[]): void => { + const appendToSelection = (objectState: ObjectState): (() => void) => { + const { clientID } = objectState; + const shape = this.canvas.select(`#cvat_canvas_shape_${clientID}`).first(); + if (shape) { + shape.addClass('cvat_canvas_shape_selection'); + if (objectState.shapeType === 'mask') { + const { points } = objectState; + const colorRGB = [252, 251, 252]; + const [left, top, right, bottom] = points.slice(-4); + const imageBitmap = RLEToImageData(colorRGB[0], colorRGB[1], colorRGB[2], points); + + const bbox = shape.bbox(); + const image = this.canvas.image().attr({ + 'color-rendering': 'optimizeQuality', + 'shape-rendering': 'geometricprecision', + 'data-z-order': Number.MAX_SAFE_INTEGER, + 'grouping-copy-for': clientID, + }).move(bbox.x, bbox.y); + + imageDataToDataURL( + imageBitmap, + right - left + 1, + bottom - top + 1, + (dataURL: string) => { + const destroy = (): void => URL.revokeObjectURL(dataURL); + if (image.parent() !== null) { + // still in DOM + image.loaded(destroy); + image.error(destroy); + image.load(dataURL); + } else { + destroy(); + } + }, + ); + + image.style('filter', 'drop-shadow(2px 4px 6px black)'); // for better visibility + image.attr('opacity', 0.5); + + return () => { + if (image.node instanceof SVGImageElement) { + URL.revokeObjectURL(image.node.href.baseVal); + } + image.remove(); + shape.removeClass('cvat_canvas_shape_selection'); + }; + } + + return () => shape.removeClass('cvat_canvas_shape_selection'); + } + + return () => {}; + }; + + for (const state of _selected) { + if (!Object.hasOwn(this.resetAppearance, state.clientID)) { + this.resetAppearance[state.clientID] = appendToSelection(state); + } + } + + for (const clientID of Object.keys(this.resetAppearance)) { + if (!_selected.some((state) => state.clientID === +clientID)) { + this.resetAppearance[clientID](); + delete this.resetAppearance[clientID]; + } + } + + callback(_selected); + }; + + this.selectionFilter = filter; + this.isEnabled = true; + } + } + + public disable(): void { + window.document.removeEventListener('mouseup', this.onMouseUp); + this.canvas.node.removeEventListener('mousedown', this.onMouseDown); + this.canvas.node.removeEventListener('mousemove', this.onMouseMove); + this.canvas.node.removeEventListener('click', this.findObjectOnClick); + + this.selectionRect?.remove(); + this.selectionRect = null; + + this.resetAllAppearances(); + this.onSelectCallback = null; + this.isEnabled = false; + } + + public push(state: ObjectState): void { + if (this.isEnabled) { + if (!Object.hasOwn(this.selectedObjects, state.clientID)) { + const filtered = this.filterObjects([state]); + if (filtered.length) { + filtered.forEach((_state) => { + this.selectedObjects[_state.clientID] = _state; + }); + this.onSelectCallback(Object.values(this.selectedObjects)); + } + } else { + delete this.selectedObjects[state.clientID]; + this.onSelectCallback(Object.values(this.selectedObjects)); + } + } + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + if (this.selectionRect) { + this.selectionRect.attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale }); + } + } + + public resetSelected(): void { + if (this.isEnabled) { + this.selectedObjects = {}; + this.resetAllAppearances(); + if (this.onSelectCallback) { + this.onSelectCallback([]); + } + } + } +} diff --git a/cvat-canvas/src/typescript/regionSelector.ts b/cvat-canvas/src/typescript/regionSelector.ts new file mode 100644 index 0000000..aafee5d --- /dev/null +++ b/cvat-canvas/src/typescript/regionSelector.ts @@ -0,0 +1,134 @@ +// Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; + +import consts from './consts'; +import { translateToSVG } from './shared'; +import { Geometry } from './canvasModel'; + +export interface RegionSelector { + select(enabled: boolean): void; + cancel(): void; + transform(geometry: Geometry): void; +} + +export class RegionSelectorImpl implements RegionSelector { + private onRegionSelected: (points?: number[]) => void; + private geometry: Geometry; + private canvas: SVG.Container; + private selectionRect: SVG.Rect | null; + private startSelectionPoint: { + x: number; + y: number; + }; + + private getSelectionBox(event: MouseEvent): { xtl: number; ytl: number; xbr: number; ybr: number } { + const point = translateToSVG((this.canvas.node as any) as SVGSVGElement, [event.clientX, event.clientY]); + const stopSelectionPoint = { + x: point[0], + y: point[1], + }; + + return { + xtl: Math.min(this.startSelectionPoint.x, stopSelectionPoint.x), + ytl: Math.min(this.startSelectionPoint.y, stopSelectionPoint.y), + xbr: Math.max(this.startSelectionPoint.x, stopSelectionPoint.x), + ybr: Math.max(this.startSelectionPoint.y, stopSelectionPoint.y), + }; + } + + private onMouseMove = (event: MouseEvent): void => { + if (this.selectionRect) { + const box = this.getSelectionBox(event); + + this.selectionRect.attr({ + x: box.xtl, + y: box.ytl, + width: box.xbr - box.xtl, + height: box.ybr - box.ytl, + }); + } + }; + + private onMouseDown = (event: MouseEvent): void => { + if (!this.selectionRect && !event.altKey) { + const point = translateToSVG((this.canvas.node as any) as SVGSVGElement, [event.clientX, event.clientY]); + this.startSelectionPoint = { + x: point[0], + y: point[1], + }; + + this.selectionRect = this.canvas + .rect() + .attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + }) + .addClass('cvat_canvas_shape_region_selection'); + this.selectionRect.attr({ ...this.startSelectionPoint, width: 1, height: 1 }); + } + }; + + private onMouseUp = (): void => { + const { offset } = this.geometry; + if (this.selectionRect) { + const { + w, h, x, y, x2, y2, + } = this.selectionRect.bbox(); + this.selectionRect.remove(); + this.selectionRect = null; + if (w <= 1 && h <= 1) { + this.onRegionSelected([x - offset, y - offset]); + } else { + this.onRegionSelected([x - offset, y - offset, x2 - offset, y2 - offset]); + } + } + }; + + private startSelection(): void { + this.canvas.node.addEventListener('mousemove', this.onMouseMove); + this.canvas.node.addEventListener('mousedown', this.onMouseDown); + this.canvas.node.addEventListener('mouseup', this.onMouseUp); + } + + private stopSelection(): void { + this.canvas.node.removeEventListener('mousemove', this.onMouseMove); + this.canvas.node.removeEventListener('mousedown', this.onMouseDown); + this.canvas.node.removeEventListener('mouseup', this.onMouseUp); + } + + private release(): void { + this.stopSelection(); + } + + public constructor(onRegionSelected: RegionSelectorImpl['onRegionSelected'], canvas: SVG.Container, geometry: Geometry) { + this.onRegionSelected = onRegionSelected; + this.geometry = geometry; + this.canvas = canvas; + this.selectionRect = null; + } + + public select(enabled: boolean): void { + if (enabled) { + this.startSelection(); + } else { + this.release(); + } + } + + public cancel(): void { + this.release(); + this.onRegionSelected(); + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + if (this.selectionRect) { + this.selectionRect.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale, + }); + } + } +} diff --git a/cvat-canvas/src/typescript/shared.ts b/cvat-canvas/src/typescript/shared.ts new file mode 100644 index 0000000..a495df1 --- /dev/null +++ b/cvat-canvas/src/typescript/shared.ts @@ -0,0 +1,807 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import * as martinez from 'martinez-polygon-clipping'; +import { LRUCache } from 'lru-cache'; +import consts from './consts'; + +export interface ShapeSizeElement { + sizeElement: any; + update(shape: SVG.Shape): void; + rm(): void; +} + +export interface Box { + xtl: number; + ytl: number; + xbr: number; + ybr: number; +} + +export interface BBox { + width: number; + height: number; + x: number; + y: number; +} + +export interface Point { + x: number; + y: number; +} + +interface Vector2D { + i: number; + j: number; +} + +export interface DrawnState { + clientID: number; + outside?: boolean; + occluded?: boolean; + hidden?: boolean; + lock: boolean; + source: 'AUTO' | 'SEMI-AUTO' | 'MANUAL' | 'FILE' | 'CONSENSUS'; + shapeType: string; + points?: number[]; + rotation: number; + attributes: Record; + descriptions: string[]; + zOrder?: number; + pinned?: boolean; + updated: number; + frame: number; + label: any; + group: any; + color: string; + elements: DrawnState[] | null; + visibleSkeletonElements?: number[] | null; +} + +// Translate point array from the canvas coordinate system +// to the coordinate system of a client +export function translateFromSVG(svg: SVGSVGElement, points: ArrayLike): number[] { + const output = []; + const transformationMatrix = svg.getScreenCTM() as DOMMatrix; + let pt = svg.createSVGPoint(); + for (let i = 0; i < points.length - 1; i += 2) { + pt.x = points[i]; + pt.y = points[i + 1]; + pt = pt.matrixTransform(transformationMatrix); + output.push(pt.x, pt.y); + } + + return output; +} + +// Translate point array from the coordinate system of a client +// to the canvas coordinate system +export function translateToSVG(svg: SVGSVGElement, points: ArrayLike): number[] { + const output = []; + const transformationMatrix = (svg.getScreenCTM() as DOMMatrix).inverse(); + let pt = svg.createSVGPoint(); + for (let i = 0; i < points.length; i += 2) { + pt.x = points[i]; + pt.y = points[i + 1]; + pt = pt.matrixTransform(transformationMatrix); + output.push(pt.x, pt.y); + } + + return output; +} + +export function composeShapeDimensions(width: number, height: number, rotation: number | null): string { + const text = `${width.toFixed(1)}x${height.toFixed(1)}px`; + let adjustableRotation = rotation; + if (adjustableRotation) { + // make sure, that rotation is in range [0; 360] + while (adjustableRotation < 0) { + adjustableRotation += 360; + } + adjustableRotation %= 360; + return `${text} ${adjustableRotation.toFixed(1)}\u00B0`; + } + + return text; +} + +export function getRoundedRotation(shape: SVG.Shape, defaultValue: number = 0): number { + const rotation = shape.transform().rotation ?? defaultValue; + // Due to floating point arithmeic, rotation value may be updated incorrectly + // even when no rotation actually happened. + // E.g. in one call it may be 16.000000000000014 + // On the next call it may be 16.00000000000003 + // As it may lead to other issues, we round this value up to 5 digits after "." + return +rotation.toFixed(5); +} + +export function displayShapeSize(shapesContainer: SVG.Container, textContainer: SVG.Container): ShapeSizeElement { + const shapeSize: ShapeSizeElement = { + sizeElement: textContainer + .text('') + .font({ + weight: 'bolder', + }) + .fill('white') + .addClass('cvat_canvas_text'), + update(shape: SVG.Shape): void { + const rotation = shape.type === 'rect' || shape.type === 'ellipse' ? + getRoundedRotation(shape) : null; + const text = composeShapeDimensions(shape.width(), shape.height(), rotation); + const [x, y, cx, cy]: number[] = translateToSVG( + (textContainer.node as any) as SVGSVGElement, + translateFromSVG((shapesContainer.node as any) as SVGSVGElement, [ + shape.x(), + shape.y(), + shape.cx(), + shape.cy(), + ]), + ).map((coord: number): number => Math.round(coord)); + this.sizeElement + .clear() + .plain(text) + .move(x + consts.TEXT_MARGIN, y + consts.TEXT_MARGIN) + .rotate(rotation ?? 0, cx, cy); + }, + rm(): void { + if (this.sizeElement) { + this.sizeElement.remove(); + this.sizeElement = null; + } + }, + }; + + return shapeSize; +} + +export function rotate2DPoints(cx: number, cy: number, angle: number, points: ArrayLike): number[] { + const rad = (Math.PI / 180) * angle; + const cos = Math.cos(rad); + const sin = Math.sin(rad); + const result = []; + for (let i = 0; i < points.length; i += 2) { + const x = points[i]; + const y = points[i + 1]; + result.push( + (x - cx) * cos - (y - cy) * sin + cx, + (y - cy) * cos + (x - cx) * sin + cy, + ); + } + + return result; +} + +export function pointsToNumberArray(points: string | Point[]): number[] { + if (Array.isArray(points)) { + return points.reduce((acc: number[], point: Point): number[] => { + acc.push(point.x, point.y); + return acc; + }, []); + } + + return points + .trim() + .split(/[,\s]+/g) + .map((coord: string): number => +coord); +} + +export function parsePoints(source: string | number[]): Point[] { + if (Array.isArray(source)) { + return source.reduce((acc: Point[], _: number, index: number): Point[] => { + if (index % 2) { + acc.push({ + x: source[index - 1], + y: source[index], + }); + } + + return acc; + }, []); + } + + return source + .trim() + .split(/\s/) + .map( + (point: string): Point => { + const [x, y] = point.split(',').map((coord: string): number => +coord); + return { x, y }; + }, + ); +} + +export function readPointsFromShape(shape: SVG.Shape): number[] { + let points = null; + if (shape.type === 'ellipse') { + const [rx, ry] = [+shape.attr('rx'), +shape.attr('ry')]; + const [cx, cy] = [shape.cx(), shape.cy()]; + points = `${cx},${cy} ${cx + rx},${cy - ry}`; + } else if (shape.type === 'rect') { + points = `${shape.attr('x')},${shape.attr('y')} ` + + `${shape.attr('x') + shape.attr('width')},${shape.attr('y') + shape.attr('height')}`; + } else if (shape.type === 'circle') { + points = `${shape.cx()},${shape.cy()}`; + } else { + points = shape.attr('points'); + } + + return pointsToNumberArray(points); +} + +export function stringifyPoints(points: ArrayLike): string; +export function stringifyPoints(points: Point[]): string; +export function stringifyPoints(points: Point[] | ArrayLike): string { + if (typeof points[0] === 'number') { + const tmp = []; + for (let i = 0; i < points.length; i += 2) { + tmp.push(`${points[i]},${points[i + 1]}`); + } + return tmp.join(' '); + } + return (points as Point[]).map((point: Point): string => `${point.x},${point.y}`).join(' '); +} + +export function clamp(x: number, min: number, max: number): number { + return Math.min(Math.max(x, min), max); +} + +export function scalarProduct(a: Vector2D, b: Vector2D): number { + return a.i * b.i + a.j * b.j; +} + +export function vectorLength(vector: Vector2D): number { + const sqrI = vector.i ** 2; + const sqrJ = vector.j ** 2; + return Math.sqrt(sqrI + sqrJ); +} + +export function translateToCanvas(offset: number, points: ArrayLike): number[] { + const result: number[] = []; + for (let i = 0; i < points.length; i++) { + result.push(points[i] + offset); + } + return result; +} + +export function translateFromCanvas(offset: number, points: ArrayLike): number[] { + const result: number[] = []; + for (let i = 0; i < points.length; i++) { + result.push(points[i] - offset); + } + return result; +} + +export function computeWrappingBox(points: ArrayLike, margin = 0): Box & BBox { + let xtl = Number.MAX_SAFE_INTEGER; + let ytl = Number.MAX_SAFE_INTEGER; + let xbr = Number.MIN_SAFE_INTEGER; + let ybr = Number.MIN_SAFE_INTEGER; + + for (let i = 0; i < points.length; i += 2) { + const [x, y] = [points[i], points[i + 1]]; + xtl = Math.min(xtl, x); + ytl = Math.min(ytl, y); + xbr = Math.max(xbr, x); + ybr = Math.max(ybr, y); + } + + const box = { + xtl: xtl - margin, + ytl: ytl - margin, + xbr: xbr + margin, + ybr: ybr + margin, + }; + + return { + ...box, + x: box.xtl, + y: box.ytl, + width: box.xbr - box.xtl, + height: box.ybr - box.ytl, + }; +} + +export function getSkeletonEdgeCoordinates(edge: SVG.Line): { + x1: number, y1: number, x2: number, y2: number +} { + let x1 = 0; + let y1 = 0; + let x2 = 0; + let y2 = 0; + + const parent = edge.parent() as any as SVG.G; + if (parent.type !== 'g') { + throw new Error('Edge parent must be a group'); + } + + const dataNodeFrom = edge.attr('data-node-from'); + const dataNodeTo = edge.attr('data-node-to'); + const nodeFrom = parent.children() + .find((element: SVG.Element): boolean => element.attr('data-node-id') === dataNodeFrom); + const nodeTo = parent.children() + .find((element: SVG.Element): boolean => element.attr('data-node-id') === dataNodeTo); + + if (!nodeFrom || !nodeTo) { + throw new Error(`Edge's nodeFrom ${dataNodeFrom} or nodeTo ${dataNodeTo} do not to refer to any node`); + } + + x1 = nodeFrom.cx(); + y1 = nodeFrom.cy(); + x2 = nodeTo.cx(); + y2 = nodeTo.cy(); + + if (nodeFrom.hasClass('cvat_canvas_hidden') || nodeTo.hasClass('cvat_canvas_hidden')) { + edge.addClass('cvat_canvas_hidden'); + } else { + edge.removeClass('cvat_canvas_hidden'); + } + + if (nodeFrom.hasClass('cvat_canvas_shape_occluded') || nodeTo.hasClass('cvat_canvas_shape_occluded')) { + edge.addClass('cvat_canvas_shape_occluded'); + } + + if ([x1, y1, x2, y2].some((coord: number): boolean => typeof coord !== 'number')) { + throw new Error(`Edge coordinates must be numbers, got [${x1}, ${y1}, ${x2}, ${y2}]`); + } + + return { + x1, y1, x2, y2, + }; +} + +export function makeSVGFromTemplate(template: SVGSVGElement): SVG.G { + const SVGElement = new SVG.G(); + SVGElement.node.replaceChildren(...template.cloneNode(true).childNodes); + return SVGElement; +} + +export function setupSkeletonEdges( + skeleton: SVG.G, + referenceSVG: SVG.G, + visibleNodeIDs?: Set, +): void { + for (const child of referenceSVG.children()) { + // search for all edges on template + const dataType = child.attr('data-type'); + if (child.type === 'line' && dataType === 'edge') { + const dataNodeFrom = child.attr('data-node-from'); + const dataNodeTo = child.attr('data-node-to'); + if (!Number.isInteger(dataNodeFrom) || !Number.isInteger(dataNodeTo)) { + throw new Error(`Edge nodeFrom and nodeTo must be numbers, got ${dataNodeFrom}, ${dataNodeTo}`); + } + if (visibleNodeIDs && (!visibleNodeIDs.has(dataNodeFrom) || !visibleNodeIDs.has(dataNodeTo))) { + continue; + } + + // try to find the same edge on the skeleton + let edge = skeleton.children().find((_child: SVG.Element) => ( + _child.attr('data-node-from') === dataNodeFrom && _child.attr('data-node-to') === dataNodeTo + )) as SVG.Line; + + // if not found, lets create it + if (!edge) { + edge = skeleton.line(0, 0, 0, 0).attr({ + 'data-node-from': dataNodeFrom, + 'data-node-to': dataNodeTo, + 'stroke-width': 'inherit', + }).addClass('cvat_canvas_skeleton_edge') as SVG.Line; + } + + skeleton.node.prepend(edge.node); + const points = getSkeletonEdgeCoordinates(edge); + edge.attr({ ...points, 'stroke-width': 'inherit' }); + } + } +} + +export function imageDataToDataURL( + imageBitmap: Uint8ClampedArray, + width: number, + height: number, + handleResult: (dataURL: string) => void, +): void { + const canvas = new OffscreenCanvas(width, height); + canvas.getContext('2d').putImageData( + new ImageData(imageBitmap, width, height), 0, 0, + ); + canvas.convertToBlob({ type: 'image/png' }).then((blob) => { + const dataURL = URL.createObjectURL(blob); + handleResult(dataURL); + }); +} + +export function imageDataToRLE(imageData: Uint8ClampedArray): number[] { + const rle = []; + + let prev = 0; + let summ = 0; + for (let i = 3; i < imageData.length; i += 4) { + const alpha = imageData[i] > 0 ? 1 : 0; + if (prev !== alpha) { + rle.push(summ); + prev = alpha; + summ = 1; + } else { + summ++; + } + } + + rle.push(summ); + return rle; +} + +export function RLEToImageData( + r: number, + g: number, + b: number, + encoded: ArrayLike, +): Uint8ClampedArray { + function rle2Mask(rle: ArrayLike, width: number, height: number): Uint8ClampedArray { + const decoded = new Uint8ClampedArray(width * height * 4).fill(0); + const { length } = rle; + let decodedIdx = 0; + let value = 0; + let i = 0; + + while (i < length - 4) { + let count = rle[i]; + + while (count > 0) { + decoded[decodedIdx + 0] = r; + decoded[decodedIdx + 1] = g; + decoded[decodedIdx + 2] = b; + decoded[decodedIdx + 3] = value * 255; + decodedIdx += 4; + count--; + } + i++; + value = Math.abs(value - 1); + } + + return decoded; + } + + const left = encoded[encoded.length - 4]; + const top = encoded[encoded.length - 3]; + const right = encoded[encoded.length - 2]; + const bottom = encoded[encoded.length - 1]; + return rle2Mask(encoded, right - left + 1, bottom - top + 1); +} + +export function findIntersection(seg1: Segment, seg2: Segment): [number, number] | null { + const determinant2D = (a: number, b: number, c: number, d: number): number => a * d - b * c; + const numberIsBetween = (a: number, b: number, c: number): boolean => Math.min(a, b) <= c && c <= Math.max(a, b); + const projectionIntersected = (a: number, b: number, c: number, d: number): boolean => { + let [p1, p2] = [a, b]; + let [p3, p4] = [c, d]; + + if (p1 > p2) { + [p1, p2] = [p2, p1]; + } + + if (p3 > p4) { + [p3, p4] = [p4, p3]; + } + + return Math.max(p1, p3) <= Math.min(p2, p4); + }; + + const [[x1, y1], [x2, y2]] = seg1; + const [[x3, y3], [x4, y4]] = seg2; + const A1 = y1 - y2; + const A2 = y3 - y4; + const B1 = x2 - x1; + const B2 = x4 - x3; + const C1 = -A1 * x1 - B1 * y1; + const C2 = -A2 * x3 - B2 * y3; + const determinant = determinant2D(A1, B1, A2, B2); + if (determinant === 0) { + if ( + determinant2D(A1, C1, A2, C2) === 0 && + determinant2D(B1, C1, B2, C2) === 0 && + projectionIntersected(x1, x2, x3, x4) && + projectionIntersected(y1, y2, y3, y4) + ) { + // lines match + return [NaN, NaN]; + } + + // lines are parallel + return null; + } + + const x = -determinant2D(C1, B1, C2, B2) / determinant; + const y = -determinant2D(A1, C1, A2, C2) / determinant; + if (numberIsBetween(x1, x2, x) && + numberIsBetween(y1, y2, y) && + numberIsBetween(x3, x4, x) && + numberIsBetween(y3, y4, y) + ) { + return [x, y]; + } + + return null; +} + +export function findClosestPointOnSegment( + segment: [[number, number], [number, number]], + point: [number, number], +): [number, number] { + const numberIsBetween = (a: number, b: number, c: number): boolean => Math.min(a, b) <= c && c <= Math.max(a, b); + const [[x1, y1], [x2, y2]] = segment; + const [x3, y3] = point; + + const x = (x1 * x1 * x3 - 2 * x1 * x2 * x3 + x2 * x2 * x3 + x2 * + (y1 - y2) * (y1 - y3) - x1 * (y1 - y2) * (y2 - y3)) / + ((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2)); + const y = (x2 * x2 * y1 + x1 * x1 * y2 + x2 * x3 * (y2 - y1) - x1 * + (x3 * (y2 - y1) + x2 * (y1 + y2)) + (y1 - y2) * (y1 - y2) * y3) / + ((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2)); + + if (numberIsBetween(x1, x2, x) && numberIsBetween(y1, y2, y)) { + return [x, y]; + } + + // perpendicular point is not on the segment + // shortest distance is distance to one of edge points + const d1 = Math.sqrt((x - x1) ** 2 + (y - y1) ** 2); + const d2 = Math.sqrt((x - x2) ** 2 + (y - y2) ** 2); + + if (d1 < d2) { + return [x1, y1]; + } + + return [x2, y2]; +} + +export function segmentsFromPoints(points: number[], circuit = false): Segment[] { + return points.reduce((acc, val, idx, arr) => { + if (idx % 2 !== 0) { + if (idx === arr.length - 1) { + if (circuit) { + acc.push([[arr[idx - 1], val], [arr[0], arr[1]]]); + } + } else { + acc.push([[arr[idx - 1], val], [arr[idx + 1], arr[idx + 2]]]); + } + } + return acc; + }, []); +} + +export function toReversed(array: Array): Array { + // actually toReversed already exists in ESMA specification + // but not all CVAT customers uses a browser fresh enough to use it + // instead of using a library with polyfills I will prefer just to rewrite it with reduceRight + return array.reduceRight>((acc, val: T) => { + acc.push(val); + return acc; + }, []); +} + +export function validateUnionResult(result: martinez.Geometry): void { + // martinez.union result format (MultiPolygon): + // [ + // [ // first polygon + // [[x, y], ...], // exterior ring + // [[x, y], ...], // hole 1 (if exists) + // ], + // [ // second polygon (if disjoint) + // [[x, y], ...], + // ] + // ] + + // Check for holes in any polygon (not supported by CVAT) + for (const polygon of result) { + if (polygon.length > 1) { + throw new Error( + 'Cannot join these polygons: the operation would create a shape with holes, ' + + 'which is not supported by CVAT. Please select different polygons or use the mask tool.', + ); + } + } +} + +export function processPolygonUnionResult(result: martinez.Geometry): { shapeType: string; points: number[] }[] { + const results: { shapeType: string; points: number[] }[] = []; + + for (const polygon of result) { + const exterior = polygon[0] as martinez.Ring; + + // Convert to flat CVAT array format (remove closing point) + const exteriorPoints: number[] = []; + for (let i = 0; i < exterior.length - 1; i += 1) { + exteriorPoints.push(exterior[i][0], exterior[i][1]); + } + + results.push({ + shapeType: 'polygon', + points: exteriorPoints, + }); + } + + return results; +} + +export function isPolygonSelfIntersecting(points: number[]): boolean { + if (points.length < 6) { + // Need at least 3 points (6 coordinates) for a polygon + return false; + } + + const segments: Segment[] = []; + for (let i = 0; i < points.length - 2; i += 2) { + segments.push([ + [points[i], points[i + 1]], + [points[i + 2], points[i + 3]], + ]); + } + // Close the polygon: last point to first point + segments.push([ + [points[points.length - 2], points[points.length - 1]], + [points[0], points[1]], + ]); + + // Check each segment against all non-adjacent segments + for (let i = 0; i < segments.length; i += 1) { + for (let j = i + 2; j < segments.length; j += 1) { + // Skip adjacent segments and the closing edge against the first edge + if (i === 0 && j === segments.length - 1) { + continue; + } + + const intersection = findIntersection(segments[i], segments[j]); + if (intersection !== null) { + const [x, y] = intersection; + if (!Number.isNaN(x) && !Number.isNaN(y)) { + const seg1Start = segments[i][0]; + const seg1End = segments[i][1]; + const seg2Start = segments[j][0]; + const seg2End = segments[j][1]; + + const EPSILON = 1e-6; + const isAtEndpoint = ( + (Math.abs(x - seg1Start[0]) < EPSILON && Math.abs(y - seg1Start[1]) < EPSILON) || + (Math.abs(x - seg1End[0]) < EPSILON && Math.abs(y - seg1End[1]) < EPSILON) || + (Math.abs(x - seg2Start[0]) < EPSILON && Math.abs(y - seg2Start[1]) < EPSILON) || + (Math.abs(x - seg2End[0]) < EPSILON && Math.abs(y - seg2End[1]) < EPSILON) + ); + + if (!isAtEndpoint) { + return true; + } + } + } + } + } + + return false; +} + +export type Segment = [[number, number], [number, number]]; +export type PropType = T[Prop]; + +interface SnapPoint { + x: number; + y: number; + clientID: number; +} + +const snapPointsCache = new LRUCache>({ + max: 2000, + updateAgeOnGet: true, + updateAgeOnHas: true, +}); + +function extractSnapPointsFromState(drawnState: DrawnState): Readonly { + if (!drawnState.points) { + return []; + } + + if (!['polygon', 'polyline', 'points', 'rectangle'].includes(drawnState.shapeType)) { + return []; + } + + const cacheKey = `${drawnState.clientID}-${drawnState.updated}`; + const cached = snapPointsCache.get(cacheKey); + if (cached) { + return cached; + } + + let result: Readonly; + if (drawnState.shapeType === 'polygon' || drawnState.shapeType === 'polyline' || drawnState.shapeType === 'points') { + result = drawnState.points; + } else if (drawnState.shapeType === 'rectangle') { + const [xtl, ytl, xbr, ybr] = drawnState.points; + const corners = [xtl, ytl, xbr, ytl, xbr, ybr, xtl, ybr]; + + if (drawnState.rotation && drawnState.rotation !== 0) { + const cx = (xtl + xbr) / 2; + const cy = (ytl + ybr) / 2; + result = rotate2DPoints(cx, cy, drawnState.rotation, corners); + } else { + result = corners; + } + } else { + result = []; + } + + snapPointsCache.set(cacheKey, result); + + return result; +} + +function findNearestSnapPoint( + x: number, + y: number, + allStates: Record, + offset: number, + snapRadius: number, + excludeClientID: number | null = null, +): SnapPoint | null { + const imageX = x - offset; + const imageY = y - offset; + + let nearestPoint: SnapPoint | null = null; + let minDistance = snapRadius; + + for (const clientIDStr of Object.keys(allStates)) { + const clientID = +clientIDStr; + + if (clientID === excludeClientID) { + continue; + } + + const drawnState = allStates[clientID]; + const points = extractSnapPointsFromState(drawnState); + + for (let i = 0; i < points.length; i += 2) { + const px = points[i]; + const py = points[i + 1]; + + const dx = imageX - px; + const dy = imageY - py; + const distance = Math.hypot(dx, dy); + + if (distance <= minDistance) { + minDistance = distance; + nearestPoint = { x: px + offset, y: py + offset, clientID }; + } + } + } + + return nearestPoint; +} + +export function applySnapToShapePoint( + shape: SVG.Polygon | SVG.PolyLine, + pointIndex: number, + allStates: Record, + offset: number, + snapRadius: number, + excludeClientID: number | null = null, +): void { + const pointsArray: number[][] = (shape as any).array().valueOf(); + if (pointIndex < 0 || pointIndex >= pointsArray.length) { + return; + } + + const [currentX, currentY] = pointsArray[pointIndex]; + + const snapTarget = findNearestSnapPoint( + currentX, + currentY, + allStates, + offset, + snapRadius, + excludeClientID, + ); + + if (snapTarget) { + pointsArray[pointIndex] = [snapTarget.x, snapTarget.y]; + shape.plot(pointsArray); + } +} diff --git a/cvat-canvas/src/typescript/sliceHandler.ts b/cvat-canvas/src/typescript/sliceHandler.ts new file mode 100644 index 0000000..eb07ed1 --- /dev/null +++ b/cvat-canvas/src/typescript/sliceHandler.ts @@ -0,0 +1,606 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import { + stringifyPoints, translateToCanvas, translateFromCanvas, translateToSVG, + findIntersection, imageDataToRLE, Segment, findClosestPointOnSegment, segmentsFromPoints, + toReversed, +} from './shared'; +import { + Geometry, SliceData, Configuration, CanvasHint, +} from './canvasModel'; +import consts from './consts'; +import { ObjectSelector } from './objectSelector'; + +export interface SliceHandler { + slice(sliceData: any): void; + transform(geometry: Geometry): void; + configure(config: Configuration): void; + cancel(): void; +} + +type EnhancedSliceData = { + enabled: boolean; + contour: number[]; + state: any; + shapeType: 'mask' | 'polygon'; +}; + +function drawOverOffscreenCanvas(context: OffscreenCanvasRenderingContext2D, image: CanvasImageSource): void { + context.fillStyle = 'black'; + context.globalCompositeOperation = 'source-over'; + context.drawImage(image, 0, 0); +} + +function applyOffscreenCanvasMask(context: OffscreenCanvasRenderingContext2D, polygon: number[]): void { + const currentCompositeOperation = context.globalCompositeOperation; + context.globalCompositeOperation = 'destination-in'; + context.beginPath(); + context.moveTo(polygon[0], polygon[1]); + polygon.forEach((_, idx) => { + if (idx > 1 && !(idx % 2)) { + context.lineTo(polygon[idx], polygon[idx + 1]); + } + }); + context.closePath(); + context.fill(); + context.globalCompositeOperation = currentCompositeOperation; +} + +function indexGenerator(length: number, from: number, to: number, direction: 'forward' | 'backward'): number[] { + const result = []; + const value = direction === 'forward' ? 1 : -1; + + if (from < 0 || from >= length || to < 0 || to >= length) { + throw new Error('Incorrect index generator input'); + } + + let i = from; + while (i !== to) { + result.push(i); + i += value; + + if (i >= length) { + i = 0; + } + + if (i < 0) { + i = length - 1; + } + } + result.push(i); + return result; +} + +function getAllIntersections(segment: Segment, segments: Segment[]): Record { + const intersections: Record = {}; + for (let i = 0; i < segments.length; i++) { + const checkedSegment = segments[i]; + const intersection = findIntersection(checkedSegment, segment); + if (intersection !== null) { + intersections[i] = intersection; + } + } + + return intersections; +} + +export class SliceHandlerImpl implements SliceHandler { + private canvas: SVG.Container; + private startTimestamp: number; + private controlPointSize: number; + private outlinedBorders: string; + private enabled: boolean; + private shapeContour: SVG.PolyLine | null; + private slicingLine: SVG.PolyLine | null; + private slicingPoints: SVG.Circle[]; + private hideObject: (clientID: number) => void; + private showObject: (clientID: number) => void; + private onSliceDone: (state?: any, results?: number[][], duration?: number) => void; + private onMessage: (messages: CanvasHint[] | null, topic: string) => void; + private onError: (exception: unknown) => void; + private getObjects: () => any[]; + private geometry: Geometry; + private objectSelector: ObjectSelector; + private hiddenClientIDs: number[]; + + public constructor( + hideObject: SliceHandlerImpl['hideObject'], + showObject: SliceHandlerImpl['showObject'], + onSliceDone: SliceHandlerImpl['onSliceDone'], + onMessage: SliceHandlerImpl['onMessage'], + onError: SliceHandlerImpl['onError'], + getObjects: () => any[], + geometry: Geometry, + canvas: SVG.Container, + objectSelector: ObjectSelector, + ) { + this.hideObject = hideObject; + this.showObject = showObject; + this.onSliceDone = onSliceDone; + this.onMessage = onMessage; + this.onError = onError; + this.getObjects = getObjects; + this.geometry = geometry; + this.canvas = canvas; + this.enabled = false; + this.startTimestamp = Date.now(); + this.controlPointSize = consts.BASE_POINT_SIZE; + this.outlinedBorders = 'black'; + this.shapeContour = null; + this.slicingPoints = []; + this.slicingLine = null; + this.objectSelector = objectSelector; + this.hiddenClientIDs = []; + } + + private showInitialMessage(): void { + this.onMessage([{ + type: 'text', + icon: 'info', + content: 'Set initial point on the shape contour', + }, { + type: 'list', + content: [ + 'Slicing line must not intersect itself', + 'Slicing line must not intersect contour more than twice', + ], + className: 'cvat-canvas-notification-list-warning', + }], 'slice'); + } + + private initialize(sliceData: EnhancedSliceData): void { + this.showInitialMessage(); + const { clientID } = sliceData.state; + this.hiddenClientIDs = (this.canvas.select('.cvat_canvas_shape') as any).members + .map((shape) => +shape.attr('clientID')).filter((_clientID: number) => _clientID !== clientID); + this.hiddenClientIDs.forEach((clientIDs) => { + this.hideObject(clientIDs); + }); + + const translatedContour = translateToCanvas(this.geometry.offset, sliceData.contour); + this.shapeContour = this.canvas.polygon(stringifyPoints(translatedContour)); + this.shapeContour.attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale }); + this.shapeContour.attr('stroke', this.outlinedBorders); + this.shapeContour.addClass('cvat_canvas_sliced_contour'); + + const contourSegments = segmentsFromPoints(translatedContour, true); + let points: [number, number][] = []; + let firstIntersectedSegmentIdx: number | null = null; + + const filterIntersections = ( + segment: Segment, + intersections: ReturnType, + ): ReturnType => { + for (const key of Object.keys(intersections)) { + const point = intersections[key]; + const d1 = Math.sqrt((point[0] - segment[0][0]) ** 2 + (point[1] - segment[0][1]) ** 2); + const d2 = Math.sqrt((point[0] - segment[0][0]) ** 2 + (point[1] - segment[0][1]) ** 2); + + // if intersection is too close to edge points + // it is an intersection in a point, ignore it + if (d1 < 2e-3 || d2 < 2e-3) { + // eslint-disable-next-line no-param-reassign + delete intersections[key]; + } + } + return intersections; + }; + + const initialClick = (event: MouseEvent): void => { + const [x, y] = translateToSVG(this.canvas.node as any as SVGSVGElement, [event.clientX, event.clientY]); + let shortestDistance = Number.MAX_SAFE_INTEGER; + let closestPoint: [number, number] = [x, y]; + let segmentIdx = -1; + contourSegments.forEach((segment, idx) => { + const point = findClosestPointOnSegment(segment, [x, y]); + const distance = Math.sqrt((x - point[0]) ** 2 + (y - point[1]) ** 2); + if (distance < shortestDistance) { + closestPoint = point; + shortestDistance = distance; + segmentIdx = idx; + } + }); + + const THRESHOLD = 20 / this.geometry.scale; + if (shortestDistance <= THRESHOLD) { + points.push([...closestPoint], [...closestPoint]); + firstIntersectedSegmentIdx = segmentIdx; + this.slicingLine = this.canvas.polyline(stringifyPoints(points.flat())); + this.slicingLine.addClass('cvat_canvas_slicing_line'); + this.slicingLine.attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale }); + this.slicingLine.attr('stroke', this.outlinedBorders); + const circle = this.canvas + .circle((this.controlPointSize * 2) / this.geometry.scale) + .center(closestPoint[0], closestPoint[1]); + circle.attr('fill', 'white'); + circle.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale); + this.slicingPoints.push(circle); + + this.onMessage([{ + type: 'text', + icon: 'info', + content: 'Set more points within the shape contour, if necessary. Intersect contour at another point to slice', + }, { + type: 'list', + content: [ + 'Hold to enable slip mode', + 'Do to cancel the latest point', + ], + className: 'cvat-canvas-notification-list-shortcuts', + }], 'slice'); + } + }; + + const click = (event: MouseEvent): void => { + const [prevX, prevY] = points[points.length - 2]; + const [x, y] = translateToSVG(this.canvas.node as any as SVGSVGElement, [event.clientX, event.clientY]); + points[points.length - 1] = [x, y]; + + // check slicing line does not intersect itself + const segment = [[prevX, prevY], [x, y]] as Segment; + const slicingLineSegments = segmentsFromPoints(points.slice(0, -1).flat()); + const selfIntersections = filterIntersections( + segment, + getAllIntersections(segment, slicingLineSegments), + ); + + if (Object.keys(selfIntersections).length) { + // not allowed + return; + } + + // find all intersections with contour + const intersections = filterIntersections( + [[prevX, prevY], [x, y]], + getAllIntersections([[prevX, prevY], [x, y]], contourSegments), + ); + + const numberOfIntersections = Object.keys(intersections).length; + if (numberOfIntersections !== 1) { + // not allowed + return; + } + + // found two intersections, finish algorithm + const intermediatePoints: [number, number][] = points.slice(1, -1); + const secondIntersectedSegmentIdx = +Object.keys(intersections)[0]; + const firstIntersectionPoint = points[0]; + const secondIntersectionPoint = intersections[secondIntersectedSegmentIdx]; + + let contour1 = []; + let contour2 = []; + if (firstIntersectedSegmentIdx === secondIntersectedSegmentIdx) { + // the same segment. Results in this case are: + contour1 = [ + ...firstIntersectionPoint, // first intersection + ...intermediatePoints.flat(), // intermediate points + ...secondIntersectionPoint, // last intersection + ]; + + contour2 = [...contour1]; + const otherPoints = Array(contourSegments.length).fill(0).map((_, idx) => { + if (firstIntersectedSegmentIdx + idx < contourSegments.length) { + return firstIntersectedSegmentIdx + idx; + } + + return firstIntersectedSegmentIdx + idx - contourSegments.length; + }).map((idx) => contourSegments[idx][1]); + + const p1 = firstIntersectionPoint; + const p2 = secondIntersectionPoint; + const p = otherPoints[0]; + const d1 = Math.sqrt((p1[0] - p[0]) ** 2 + (p1[1] - p[1]) ** 2); + const d2 = Math.sqrt((p2[0] - p[0]) ** 2 + (p2[1] - p[1]) ** 2); + + if (d2 > d1) { + contour2.push(...toReversed<[number, number]>(otherPoints).flat()); + } else { + contour2.push(...otherPoints.flat()); + } + } else { + const firstSegmentIdx = Math.min(firstIntersectedSegmentIdx, secondIntersectedSegmentIdx); + const secondSegmentIdx = Math.max(firstIntersectedSegmentIdx, secondIntersectedSegmentIdx); + const firstSegmentPoint = firstIntersectedSegmentIdx < secondIntersectedSegmentIdx ? + firstIntersectionPoint : secondIntersectionPoint; + const secondSegmentPoint = firstIntersectedSegmentIdx < secondIntersectedSegmentIdx ? + secondIntersectionPoint : firstIntersectionPoint; + + // intersected different segments. Results in this case are: + contour1 = [ + ...firstSegmentPoint, // first intersection + // intermediate points (reversed if intersections order was swopped) + ...(firstSegmentIdx === firstIntersectedSegmentIdx ? + intermediatePoints : toReversed<[number, number]>(intermediatePoints) + ).flat(), + // second intersection + ...secondSegmentPoint, + // all the following contours points N, N+1, .. until (including) the first intersected segment + ...indexGenerator(contourSegments.length, secondSegmentIdx, firstSegmentIdx, 'forward') + .map((idx) => contourSegments[idx][1]).slice(0, -1).flat(), + ]; + + contour2 = [ + ...firstSegmentPoint, // first intersection + // intermediate points (reversed if intersections order was swopped) + ...(firstSegmentIdx === firstIntersectedSegmentIdx ? + intermediatePoints : toReversed<[number, number]>(intermediatePoints) + ).flat(), + ...secondSegmentPoint, + // all the previous contours points N, N-1, .. until (including) the first intersected segment + ...indexGenerator(contourSegments.length, secondSegmentIdx, firstSegmentIdx, 'backward') + .map((idx) => contourSegments[idx][0]).slice(0, -1).flat(), + ]; + } + + if (sliceData.shapeType === 'mask') { + const shape = this.canvas + .select(`#cvat_canvas_shape_${clientID}`).get(0).node; + const width = +shape.getAttribute('width'); + const height = +shape.getAttribute('height'); + const left = +shape.getAttribute('x'); + const top = +shape.getAttribute('y'); + + const polygon1 = contour1.map((val, idx) => { + if (idx % 2) return val - top; + return val - left; + }); + + const polygon2 = contour2.map((val, idx) => { + if (idx % 2) return val - top; + return val - left; + }); + + const offscreenCanvas = new OffscreenCanvas(width, height); + const context = offscreenCanvas.getContext('2d'); + drawOverOffscreenCanvas(context, shape as any as SVGImageElement); + applyOffscreenCanvasMask(context, polygon1); + const firstShape = imageDataToRLE(context.getImageData(0, 0, width, height).data); + // @ts-ignore error TS2339 https://github.com/microsoft/TypeScript/issues/55162 + context.reset(); + drawOverOffscreenCanvas(context, shape as any as SVGImageElement); + applyOffscreenCanvasMask(context, polygon2); + const secondShape = imageDataToRLE(context.getImageData(0, 0, width, height).data); + this.onSliceDone(sliceData.state, [firstShape, secondShape], Date.now() - this.startTimestamp); + } else if (sliceData.shapeType === 'polygon') { + this.onSliceDone( + sliceData.state, + [ + translateFromCanvas(this.geometry.offset, contour1), + translateFromCanvas(this.geometry.offset, contour2), + ], Date.now() - this.startTimestamp, + ); + } else { + this.slice({ enabled: false }); + } + }; + + const handleCanvasMousedown = (event: MouseEvent): void => { + if (event.altKey) { + return; + } + + if (event.button === 0 && !points.length) { + initialClick(event); + } else if (event.button === 0 && event.target !== this.shapeContour.node) { + click(event); + } else if (event.button === 2) { + if (points.length > 2) { + points.splice(-2, 1); + this.slicingLine.plot(stringifyPoints(points.flat())); + } else if (points.length) { + this.slicingPoints.forEach((circle) => { + circle.remove(); + }); + this.showInitialMessage(); + this.slicingLine.remove(); + points = []; + firstIntersectedSegmentIdx = null; + this.slicingPoints = []; + this.slicingLine = null; + } + } + }; + + const handleShapeMousedown = (event: MouseEvent, slipping = false): void => { + if (points.length && event.button === 0 && !event.altKey) { + const [x, y] = translateToSVG(this.canvas.node as any as SVGSVGElement, [event.clientX, event.clientY]); + points[points.length - 1] = [x, y]; + this.slicingLine.plot(stringifyPoints(points.flat())); + + const [prevX, prevY] = points[points.length - 2]; + const segment = [[prevX, prevY], [x, y]] as Segment; + + const slicingLineSegments = segmentsFromPoints(points.slice(0, -1).flat()); + const selfIntersections = filterIntersections( + segment, + getAllIntersections(segment, slicingLineSegments), + ); + + if (Object.keys(selfIntersections).length !== 0) { + return; + } + + // find all intersections with contour + const contourIntersection = filterIntersections( + [[prevX, prevY], [x, y]], + getAllIntersections([[prevX, prevY], [x, y]], contourSegments), + ); + + const numberOfIntersections = Object.keys(contourIntersection).length; + if (!slipping && numberOfIntersections !== 0) { + // shape was clicked with intersections (via out of contour trajectory) + // not allowed + return; + } + + if (numberOfIntersections === 0 && event.target === this.shapeContour.node) { + // mousemove over the shape, left new point + click(event); + } else if (numberOfIntersections === 1 && points.length > 2) { + // maybe out of contour, maybe within + // require at least one more intermediate points in this case + click(event); + } else { + return; + } + + if (this.enabled) { + // check if slicing is still enabled + // because click() may finish slicing from inside + // e.g. when click out of contour with enabled shift + points.push([x, y]); + this.slicingLine.plot(stringifyPoints(points.flat())); + } + } + }; + + const handleCanvasMousemove = (event: MouseEvent): void => { + if (points.length) { + const [x, y] = translateToSVG(this.canvas.node as any as SVGSVGElement, [event.clientX, event.clientY]); + const [prevX, prevY] = points[points.length - 2]; + points[points.length - 1] = [x, y]; + + if (event.shiftKey) { + const d = Math.sqrt((prevX - x) ** 2 + (prevY - y) ** 2); + const threshold = 10 / this.geometry.scale; + if (d > threshold) { + handleShapeMousedown(event, true); + } + } else { + this.slicingLine.plot(stringifyPoints(points.flat())); + } + } + }; + + this.shapeContour.on('mousedown.slice', handleShapeMousedown); + this.canvas.on('mousedown.slice', handleCanvasMousedown); + this.canvas.on('mousemove.slice', handleCanvasMousemove); + } + + private release(): void { + this.objectSelector.disable(); + this.hiddenClientIDs.forEach((clientIDs) => { + this.showObject(clientIDs); + }); + + if (this.slicingLine) { + this.slicingLine.remove(); + this.slicingLine = null; + } + + if (this.shapeContour) { + this.shapeContour.off('mousedown.slice'); + this.shapeContour.remove(); + this.shapeContour = null; + } + + this.slicingPoints.forEach((circle) => { + circle.remove(); + }); + this.slicingPoints = []; + + this.canvas.off('mousedown.slice'); + this.canvas.off('mousemove.slice'); + this.enabled = false; + this.onSliceDone(); + this.onMessage(null, 'slice'); + } + + public slice(sliceData: SliceData): void { + const initializeWithContour = (state: any): void => { + const { shapeType, points } = state; + + if (state.shapeType === 'polygon') { + this.initialize({ + enabled: true, + contour: points, + state, + shapeType, + }); + } else { + this.startTimestamp = Date.now(); + const { startTimestamp } = this; + + this.onMessage([{ + type: 'text', + content: 'Getting shape contour', + icon: 'loading', + }], 'force'); + + sliceData.getContour(state).then((contour) => { + if (this.startTimestamp === startTimestamp && this.enabled) { + // checking if a user does not left mode / reinit it + this.initialize({ + enabled: true, + contour: contour.flat(), + state, + shapeType: state.shapeType, + }); + } + }).catch((error: unknown) => { + this.release(); + this.onError(error); + }); + } + }; + + if (sliceData.enabled && !this.enabled && sliceData.getContour) { + this.enabled = true; + if (sliceData.clientID) { + const state = this.getObjects().find((_state) => _state.clientID === sliceData.clientID); + if (state && state.objectType === 'shape' && + ['polygon', 'mask'].includes(state.shapeType)) { + initializeWithContour(state); + return; + } + } + + this.onMessage([{ + type: 'text', + content: 'Click a mask or polygon shape you would like to slice', + icon: 'info', + }], 'slice'); + + this.objectSelector.enable(([state]) => { + if (state) { + this.objectSelector.disable(); + initializeWithContour(state); + } + }, { maxCount: 1, shapeType: ['polygon', 'mask'], objectType: ['shape'] }); + } else if (this.enabled && !sliceData.enabled) { + this.release(); + } + } + + public cancel(): void { + if (this.enabled) { + this.release(); + } + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + if (this.slicingLine) { + this.slicingLine.attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale }); + } + + if (this.shapeContour) { + this.shapeContour.attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale }); + } + + this.slicingPoints.forEach((point) => { + point.radius(this.controlPointSize / geometry.scale); + point.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale); + }); + } + + public configure(config: Configuration): void { + this.controlPointSize = config.controlPointsSize || consts.BASE_POINT_SIZE; + this.outlinedBorders = config.outlinedBorders || 'black'; + if (this.slicingLine) this.slicingLine.attr('stroke', this.outlinedBorders); + if (this.shapeContour) this.shapeContour.attr('stroke', this.outlinedBorders); + } +} diff --git a/cvat-canvas/src/typescript/splitHandler.ts b/cvat-canvas/src/typescript/splitHandler.ts new file mode 100644 index 0000000..d4e0f9e --- /dev/null +++ b/cvat-canvas/src/typescript/splitHandler.ts @@ -0,0 +1,110 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import { SplitData } from './canvasModel'; + +export interface SplitHandler { + split(splitData: SplitData): void; + select(state: any): void; + cancel(): void; +} + +export class SplitHandlerImpl implements SplitHandler { + // callback is used to notify about splitting end + private onSplitDone: (object?: any, duration?: number) => void; + private onFindObject: (event: MouseEvent) => void; + private canvas: SVG.Container; + private highlightedShape: SVG.Shape | null; + private initialized: boolean; + private splitDone: boolean; + private startTimestamp: number; + + private resetShape(): void { + if (this.highlightedShape) { + this.highlightedShape.removeClass('cvat_canvas_shape_splitting'); + this.highlightedShape.off('click.split'); + this.highlightedShape = null; + } + } + + private release(): void { + if (this.initialized) { + this.resetShape(); + this.canvas.node.removeEventListener('mousemove', this.findObject); + this.initialized = false; + } + } + + private initSplitting(): void { + this.canvas.node.addEventListener('mousemove', this.findObject); + this.initialized = true; + this.splitDone = false; + this.startTimestamp = Date.now(); + } + + private closeSplitting(): void { + // Split done is true if an object was split + // Split also can be called with { enabled: false } without splitting an object + if (!this.splitDone) { + this.onSplitDone(null); + } + this.release(); + } + + private findObject = (e: MouseEvent): void => { + this.resetShape(); + this.onFindObject(e); + }; + + public constructor( + onSplitDone: SplitHandlerImpl['onSplitDone'], + onFindObject: SplitHandlerImpl['onFindObject'], + canvas: SVG.Container, + ) { + this.onSplitDone = onSplitDone; + this.onFindObject = onFindObject; + this.canvas = canvas; + this.highlightedShape = null; + this.initialized = false; + this.splitDone = false; + this.startTimestamp = Date.now(); + } + + public split(splitData: SplitData): void { + if (splitData.enabled) { + this.initSplitting(); + } else { + this.closeSplitting(); + } + } + + public select(state: any): void { + if (state.objectType === 'track') { + const shape = this.canvas.select(`#cvat_canvas_shape_${state.clientID}`).first(); + if (shape && shape !== this.highlightedShape) { + this.resetShape(); + this.highlightedShape = shape; + this.highlightedShape.addClass('cvat_canvas_shape_splitting'); + this.canvas.node.append(this.highlightedShape.node); + this.highlightedShape.on( + 'click.split', + (): void => { + this.splitDone = true; + this.onSplitDone(state, Date.now() - this.startTimestamp); + }, { once: true }, + ); + } + } + } + + public cancel(): void { + this.release(); + this.onSplitDone(null); + // here is a cycle + // onSplitDone => controller => model => view => closeSplitting + // one call of closeMerging is unuseful, but it's okey + } +} diff --git a/cvat-canvas/src/typescript/svg.patch.ts b/cvat-canvas/src/typescript/svg.patch.ts new file mode 100644 index 0000000..77d298a --- /dev/null +++ b/cvat-canvas/src/typescript/svg.patch.ts @@ -0,0 +1,1132 @@ +// Copyright (C) 2019-2022 Intel Corporation +// +// SPDX-License-Identifier: MIT + +/* eslint-disable */ +import * as SVG from 'svg.js'; +import 'svg.draggable.js'; +import 'svg.resize.js'; +import 'svg.select.js'; +import 'svg.draw.js'; + +import consts from './consts'; +import { Equation, CuboidModel, Orientation, Edge } from './cuboid'; +import { Point, parsePoints, clamp } from './shared'; + +// Update constructor +const originalDraw = SVG.Element.prototype.draw; +SVG.Element.prototype.draw = function constructor(...args: any): any { + let handler = this.remember('_paintHandler'); + if (!handler) { + originalDraw.call(this, ...args); + handler = this.remember('_paintHandler'); + // There is use case (drawing a single point when handler is created and destructed immediately in one stack) + // So, we need to check if handler still exists + if (handler && !handler.set) { + handler.set = new SVG.Set(); + } + } else { + originalDraw.call(this, ...args); + } + + return this; +}; +for (const key of Object.keys(originalDraw)) { + SVG.Element.prototype.draw[key] = originalDraw[key]; +} + +// Create undo for polygons and polylines +function undo(): void { + if (this.set && this.set.length()) { + this.set.members.splice(-1, 1)[0].remove(); + this.el.array().value.splice(-2, 1); + this.el.plot(this.el.array()); + this.el.fire('undopoint'); + } +} + +SVG.Element.prototype.draw.extend( + 'polyline', + Object.assign({}, SVG.Element.prototype.draw.plugins.polyline, { + undo: undo, + }), +); + +SVG.Element.prototype.draw.extend( + 'polygon', + Object.assign({}, SVG.Element.prototype.draw.plugins.polygon, { + undo: undo, + }), +); + +export const CIRCLE_STROKE = '#000'; +// Fix method drawCircles +function drawCircles(): void { + const array = this.el.array().valueOf(); + + this.set.each(function (): void { + this.remove(); + }); + + this.set.clear(); + + for (let i = 0; i < array.length - 1; ++i) { + [this.p.x] = array[i]; + [, this.p.y] = array[i]; + + const p = this.p.matrixTransform( + this.parent.node.getScreenCTM().inverse().multiply(this.el.node.getScreenCTM()), + ); + + this.set.add( + this.parent + .circle(5) + .stroke({ + width: 1, + color: CIRCLE_STROKE, + }) + .fill('#ccc') + .center(p.x, p.y), + ); + } +} + +SVG.Element.prototype.draw.extend( + 'line', + Object.assign({}, SVG.Element.prototype.draw.plugins.line, { + drawCircles: drawCircles, + }), +); + +SVG.Element.prototype.draw.extend( + 'polyline', + Object.assign({}, SVG.Element.prototype.draw.plugins.polyline, { + drawCircles: drawCircles, + }), +); + +SVG.Element.prototype.draw.extend( + 'polygon', + Object.assign({}, SVG.Element.prototype.draw.plugins.polygon, { + drawCircles: drawCircles, + }), +); + +// Fix method drag +const originalDraggable = SVG.Element.prototype.draggable; +SVG.Element.prototype.draggable = function constructor(...args: any): any { + let handler = this.remember('_draggable'); + if (!handler) { + originalDraggable.call(this, ...args); + handler = this.remember('_draggable'); + handler.drag = function (e: any) { + this.m = this.el.node.getScreenCTM().inverse(); + return handler.constructor.prototype.drag.call(this, e); + }; + } else { + originalDraggable.call(this, ...args); + } + + return this; +}; +for (const key of Object.keys(originalDraggable)) { + SVG.Element.prototype.draggable[key] = originalDraggable[key]; +} + +// Fix method resize +const originalResize = SVG.Element.prototype.resize; +SVG.Element.prototype.resize = function constructor(...args: any): any { + let handler = this.remember('_resizeHandler'); + if (!handler) { + originalResize.call(this, ...args); + handler = this.remember('_resizeHandler'); + handler.resize = function (e: any) { + const { event } = e.detail; + this.rotationPointPressed = e.type === 'rot'; + if ( + event.button === 0 && + // ignore shift key for cuboids (change perspective) and rectangles (precise rotation) + (!event.shiftKey || ( + this.el.parent().hasClass('cvat_canvas_shape_cuboid') + || this.el.type === 'rect') + ) && !event.altKey + ) { + return handler.constructor.prototype.resize.call(this, e); + } + }; + handler.update = function (e: any) { + if (!this.rotationPointPressed) { + this.m = this.el.node.getScreenCTM().inverse(); + } + handler.constructor.prototype.update.call(this, e); + }; + } else { + originalResize.call(this, ...args); + } + + return this; +}; +for (const key of Object.keys(originalResize)) { + SVG.Element.prototype.resize[key] = originalResize[key]; +} + +enum EdgeIndex { + FL = 1, + FR = 2, + DR = 3, + DL = 4, +} + +function getEdgeIndex(cuboidPoint: number): EdgeIndex { + switch (cuboidPoint) { + case 0: + case 1: + return EdgeIndex.FL; + case 2: + case 3: + return EdgeIndex.FR; + case 4: + case 5: + return EdgeIndex.DR; + default: + return EdgeIndex.DL; + } +} + +function getTopDown(edgeIndex: EdgeIndex): number[] { + switch (edgeIndex) { + case EdgeIndex.FL: + return [0, 1]; + case EdgeIndex.FR: + return [2, 3]; + case EdgeIndex.DR: + return [4, 5]; + default: + return [6, 7]; + } +} + +(SVG as any).Cube = SVG.invent({ + create: 'g', + inherit: SVG.G, + extend: { + constructorMethod(points: string) { + this.cuboidModel = new CuboidModel(parsePoints(points)); + this.setupFaces(); + this.setupEdges(); + this.setupProjections(); + this.hideProjections(); + + this._attr('points', points); + this.addClass('cvat_canvas_shape_cuboid'); + return this; + }, + + setupFaces() { + this.bot = this.polygon(this.cuboidModel.bot.points); + this.top = this.polygon(this.cuboidModel.top.points); + this.right = this.polygon(this.cuboidModel.right.points); + this.left = this.polygon(this.cuboidModel.left.points); + this.dorsal = this.polygon(this.cuboidModel.dorsal.points); + this.face = this.polygon(this.cuboidModel.front.points); + }, + + setupProjections() { + this.ftProj = this.line( + this.updateProjectionLine( + this.cuboidModel.ft.getEquation(), + this.cuboidModel.ft.points[0], + this.cuboidModel.vpl, + ), + ); + this.fbProj = this.line( + this.updateProjectionLine( + this.cuboidModel.fb.getEquation(), + this.cuboidModel.ft.points[0], + this.cuboidModel.vpl, + ), + ); + this.rtProj = this.line( + this.updateProjectionLine( + this.cuboidModel.rt.getEquation(), + this.cuboidModel.rt.points[1], + this.cuboidModel.vpr, + ), + ); + this.rbProj = this.line( + this.updateProjectionLine( + this.cuboidModel.rb.getEquation(), + this.cuboidModel.rb.points[1], + this.cuboidModel.vpr, + ), + ); + + this.ftProj.stroke({ color: '#C0C0C0' }).addClass('cvat_canvas_cuboid_projections'); + this.fbProj.stroke({ color: '#C0C0C0' }).addClass('cvat_canvas_cuboid_projections'); + this.rtProj.stroke({ color: '#C0C0C0' }).addClass('cvat_canvas_cuboid_projections'); + this.rbProj.stroke({ color: '#C0C0C0' }).addClass('cvat_canvas_cuboid_projections'); + }, + + setupEdges() { + this.frontLeftEdge = this.line(this.cuboidModel.fl.points); + this.frontRightEdge = this.line(this.cuboidModel.fr.points); + this.dorsalRightEdge = this.line(this.cuboidModel.dr.points); + this.dorsalLeftEdge = this.line(this.cuboidModel.dl.points); + + this.frontTopEdge = this.line(this.cuboidModel.ft.points); + this.rightTopEdge = this.line(this.cuboidModel.rt.points); + this.frontBotEdge = this.line(this.cuboidModel.fb.points); + this.rightBotEdge = this.line(this.cuboidModel.rb.points); + }, + + setupGrabPoints(circleType: Function | string) { + const viewModel = this.cuboidModel; + const circle = typeof circleType === 'function' ? circleType : this.circle; + + this.flCenter = circle(0, 0).addClass('svg_select_points').addClass('svg_select_points_l'); + this.frCenter = circle(0, 0).addClass('svg_select_points').addClass('svg_select_points_r'); + this.ftCenter = circle(0, 0).addClass('svg_select_points').addClass('svg_select_points_t'); + this.fbCenter = circle(0, 0).addClass('svg_select_points').addClass('svg_select_points_b'); + + this.drCenter = circle(0, 0).addClass('svg_select_points').addClass('svg_select_points_ew'); + this.dlCenter = circle(0, 0).addClass('svg_select_points').addClass('svg_select_points_ew'); + + const grabPoints = this.getGrabPoints(); + const edges = this.getEdges(); + for (let i = 0; i < grabPoints.length; i += 1) { + const edge = edges[i]; + const cx = (edge.attr('x2') + edge.attr('x1')) / 2; + const cy = (edge.attr('y2') + edge.attr('y1')) / 2; + grabPoints[i].center(cx, cy); + } + + if (viewModel.orientation === Orientation.LEFT) { + this.dlCenter.hide(); + } else { + this.drCenter.hide(); + } + }, + + showProjections() { + if (this.projectionLineEnable) { + this.ftProj.show(); + this.fbProj.show(); + this.rtProj.show(); + this.rbProj.show(); + } + }, + + hideProjections() { + this.ftProj.hide(); + this.fbProj.hide(); + this.rtProj.hide(); + this.rbProj.hide(); + }, + + getEdges() { + const arr = []; + arr.push(this.frontLeftEdge); + arr.push(this.frontRightEdge); + arr.push(this.dorsalRightEdge); + arr.push(this.frontTopEdge); + arr.push(this.frontBotEdge); + arr.push(this.dorsalLeftEdge); + arr.push(this.rightTopEdge); + arr.push(this.rightBotEdge); + return arr; + }, + + getGrabPoints() { + const arr = []; + arr.push(this.flCenter); + arr.push(this.frCenter); + arr.push(this.drCenter); + arr.push(this.ftCenter); + arr.push(this.fbCenter); + arr.push(this.dlCenter); + return arr; + }, + + updateProjectionLine(equation: Equation, source: Point, direction: Point) { + const x1 = source.x; + const y1 = equation.getY(x1); + + const x2 = direction.x; + const y2 = equation.getY(x2); + return [ + [x1, y1], + [x2, y2], + ]; + }, + + selectize(value: boolean, options: object) { + this.face.selectize(value, options); + + if (this.cuboidModel.orientation === Orientation.LEFT) { + this.dorsalLeftEdge.selectize(false, options); + this.dorsalRightEdge.selectize(value, options); + } else { + this.dorsalRightEdge.selectize(false, options); + this.dorsalLeftEdge.selectize(value, options); + } + + if (value === false) { + this.getGrabPoints().forEach((point: SVG.Element) => { + point && point.remove(); + }); + } else { + this.setupGrabPoints( + this.face + .remember('_selectHandler') + .drawPoint.bind({ nested: this, options: this.face.remember('_selectHandler').options }), + ); + + // setup proper classes for selection points for proper cursor + Array.from(this.face.remember('_selectHandler').nested.node.children).forEach( + (point: SVG.LinkedHTMLElement, i: number) => { + point.classList.add(`svg_select_points_${['lt', 'lb', 'rb', 'rt'][i]}`); + }, + ); + + if (this.cuboidModel.orientation === Orientation.LEFT) { + Array.from(this.dorsalRightEdge.remember('_selectHandler').nested.node.children).forEach( + (point: SVG.LinkedHTMLElement, i: number) => { + point.classList.add(`svg_select_points_${['t', 'b'][i]}`); + point.ondblclick = (e: MouseEvent) => { + if (e.shiftKey) { + this.resetPerspective(); + } + }; + }, + ); + } else { + Array.from(this.dorsalLeftEdge.remember('_selectHandler').nested.node.children).forEach( + (point: SVG.LinkedHTMLElement, i: number) => { + point.classList.add(`svg_select_points_${['t', 'b'][i]}`); + point.ondblclick = (e: MouseEvent) => { + if (e.shiftKey) { + this.resetPerspective(); + } + }; + }, + ); + } + } + + return this; + }, + + resize(value?: string | object) { + this.face.resize(value); + + if (value === 'stop') { + this.dorsalRightEdge.resize(value); + this.dorsalLeftEdge.resize(value); + this.face.off('resizing').off('resizedone').off('resizestart'); + this.dorsalRightEdge.off('resizing').off('resizedone').off('resizestart'); + this.dorsalLeftEdge.off('resizing').off('resizedone').off('resizestart'); + + this.getGrabPoints().forEach((point: SVG.Element) => { + if (point) { + point.off('dragstart'); + point.off('dragmove'); + point.off('dragend'); + } + }); + + return; + } + + function getResizedPointIndex(event: CustomEvent): number { + const { target } = event.detail.event.detail.event; + const { parentElement } = target; + return Array.from(parentElement.children).indexOf(target); + } + + let resizedCubePoint: null | number = null; + const accumulatedOffset: Point = { + x: 0, + y: 0, + }; + + this.face + .on('resizestart', (event: CustomEvent) => { + accumulatedOffset.x = 0; + accumulatedOffset.y = 0; + const resizedFacePoint = getResizedPointIndex(event); + resizedCubePoint = [0, 1].includes(resizedFacePoint) ? resizedFacePoint : 5 - resizedFacePoint; // 2,3 -> 3,2 + this.fire(new CustomEvent('resizestart', event)); + }) + .on('resizing', (event: CustomEvent) => { + let { dx, dy } = event.detail; + let dxPortion = dx - accumulatedOffset.x; + let dyPortion = dy - accumulatedOffset.y; + accumulatedOffset.x += dxPortion; + accumulatedOffset.y += dyPortion; + + const edge = getEdgeIndex(resizedCubePoint); + const [edgeTopIndex, edgeBottomIndex] = getTopDown(edge); + + let cuboidPoints = this.cuboidModel.getPoints(); + let x1 = cuboidPoints[edgeTopIndex].x + dxPortion; + let x2 = cuboidPoints[edgeBottomIndex].x + dxPortion; + if ( + edge === EdgeIndex.FL && + cuboidPoints[2].x - (cuboidPoints[0].x + dxPortion) < consts.MIN_EDGE_LENGTH + ) { + x1 = cuboidPoints[edgeTopIndex].x; + x2 = cuboidPoints[edgeBottomIndex].x; + } else if ( + edge === EdgeIndex.FR && + cuboidPoints[2].x + dxPortion - cuboidPoints[0].x < consts.MIN_EDGE_LENGTH + ) { + x1 = cuboidPoints[edgeTopIndex].x; + x2 = cuboidPoints[edgeBottomIndex].x; + } + const y1 = this.cuboidModel.ft.getEquation().getY(x1); + const y2 = this.cuboidModel.fb.getEquation().getY(x2); + const topPoint = { x: x1, y: y1 }; + const botPoint = { x: x2, y: y2 }; + if (edge === 1) { + this.cuboidModel.fl.points = [topPoint, botPoint]; + } else { + this.cuboidModel.fr.points = [topPoint, botPoint]; + } + this.updateViewAndVM(edge === EdgeIndex.FR); + + cuboidPoints = this.cuboidModel.getPoints(); + const midPointUp = { ...cuboidPoints[edgeTopIndex] }; + const midPointDown = { ...cuboidPoints[edgeBottomIndex] }; + (edgeTopIndex === resizedCubePoint ? midPointUp : midPointDown).y += dyPortion; + if (midPointDown.y - midPointUp.y > consts.MIN_EDGE_LENGTH) { + const topPoints = this.computeHeightFace(midPointUp, edge); + const bottomPoints = this.computeHeightFace(midPointDown, edge); + this.cuboidModel.top.points = topPoints; + this.cuboidModel.bot.points = bottomPoints; + this.updateViewAndVM(false); + } + + this.face.plot(this.cuboidModel.front.points); + this.fire(new CustomEvent('resizing', event)); + }) + .on('resizedone', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + + function computeSideEdgeConstraints(edge: Edge, fr: Edge) { + const midLength = fr.points[1].y - fr.points[0].y - 1; + + const minY = edge.points[1].y - midLength; + const maxY = edge.points[0].y + midLength; + + const y1 = edge.points[0].y; + const y2 = edge.points[1].y; + + const miny1 = y2 - midLength; + const maxy1 = y2 - consts.MIN_EDGE_LENGTH; + + const miny2 = y1 + consts.MIN_EDGE_LENGTH; + const maxy2 = y1 + midLength; + + return { + constraint: { + minY, + maxY, + }, + y1Range: { + max: maxy1, + min: miny1, + }, + y2Range: { + max: maxy2, + min: miny2, + }, + }; + } + + function setupDorsalEdge(edge: SVG.Line, orientation: Orientation) { + edge.on('resizestart', (event: CustomEvent) => { + accumulatedOffset.x = 0; + accumulatedOffset.y = 0; + resizedCubePoint = getResizedPointIndex(event) + (orientation === Orientation.LEFT ? 4 : 6); + this.fire(new CustomEvent('resizestart', event)); + }) + .on('resizing', (event: CustomEvent) => { + let { dy } = event.detail; + let dyPortion = dy - accumulatedOffset.y; + accumulatedOffset.y += dyPortion; + + const edge = getEdgeIndex(resizedCubePoint); + const [edgeTopIndex, edgeBottomIndex] = getTopDown(edge); + let cuboidPoints = this.cuboidModel.getPoints(); + + if (!event.detail.event.shiftKey) { + cuboidPoints = this.cuboidModel.getPoints(); + const midPointUp = { ...cuboidPoints[edgeTopIndex] }; + const midPointDown = { ...cuboidPoints[edgeBottomIndex] }; + (edgeTopIndex === resizedCubePoint ? midPointUp : midPointDown).y += dyPortion; + if (midPointDown.y - midPointUp.y > consts.MIN_EDGE_LENGTH) { + const topPoints = this.computeHeightFace(midPointUp, edge); + const bottomPoints = this.computeHeightFace(midPointDown, edge); + this.cuboidModel.top.points = topPoints; + this.cuboidModel.bot.points = bottomPoints; + } + } else { + const midPointUp = { ...cuboidPoints[edgeTopIndex] }; + const midPointDown = { ...cuboidPoints[edgeBottomIndex] }; + (edgeTopIndex === resizedCubePoint ? midPointUp : midPointDown).y += dyPortion; + const dorselEdge = + orientation === Orientation.LEFT ? this.cuboidModel.dr : this.cuboidModel.dl; + const constraints = computeSideEdgeConstraints(dorselEdge, this.cuboidModel.fr); + midPointUp.y = clamp(midPointUp.y, constraints.y1Range.min, constraints.y1Range.max); + midPointDown.y = clamp(midPointDown.y, constraints.y2Range.min, constraints.y2Range.max); + dorselEdge.points = [midPointUp, midPointDown]; + this.updateViewAndVM(edge === EdgeIndex.DL); + } + + this.updateViewAndVM(false); + this.face.plot(this.cuboidModel.front.points); + this.fire(new CustomEvent('resizing', event)); + }) + .on('resizedone', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + } + + if (this.cuboidModel.orientation === Orientation.LEFT) { + this.dorsalRightEdge.resize(value); + setupDorsalEdge.call(this, this.dorsalRightEdge, this.cuboidModel.orientation); + } else { + this.dorsalLeftEdge.resize(value); + setupDorsalEdge.call(this, this.dorsalLeftEdge, this.cuboidModel.orientation); + } + + function horizontalEdgeControl(updatingFace: any, midX: number, midY: number) { + const leftPoints = this.updatedEdge( + this.cuboidModel.fl.points[0], + { x: midX, y: midY }, + this.cuboidModel.vpl, + ); + const rightPoints = this.updatedEdge( + this.cuboidModel.dr.points[0], + { x: midX, y: midY }, + this.cuboidModel.vpr, + ); + + updatingFace.points = [leftPoints, { x: midX, y: midY }, rightPoints, null]; + } + + this.drCenter + .draggable((x: number) => { + let xStatus; + if (this.drCenter.cx() < this.cuboidModel.fr.points[0].x) { + xStatus = + x < this.cuboidModel.fr.points[0].x - consts.MIN_EDGE_LENGTH && + x > this.cuboidModel.vpr.x + consts.MIN_EDGE_LENGTH; + } else { + xStatus = + x > this.cuboidModel.fr.points[0].x + consts.MIN_EDGE_LENGTH && + x < this.cuboidModel.vpr.x - consts.MIN_EDGE_LENGTH; + } + return { x: xStatus, y: this.drCenter.attr('y1') }; + }) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('resizestart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.dorsalRightEdge.center(this.drCenter.cx(), this.drCenter.cy()); + + const x = this.dorsalRightEdge.attr('x1'); + const y1 = this.cuboidModel.rt.getEquation().getY(x); + const y2 = this.cuboidModel.rb.getEquation().getY(x); + const topPoint = { x, y: y1 }; + const botPoint = { x, y: y2 }; + + this.cuboidModel.dr.points = [topPoint, botPoint]; + this.updateViewAndVM(); + this.fire(new CustomEvent('resizing', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + + this.dlCenter + .draggable((x: number) => { + let xStatus; + if (this.dlCenter.cx() < this.cuboidModel.fl.points[0].x) { + xStatus = + x < this.cuboidModel.fl.points[0].x - consts.MIN_EDGE_LENGTH && + x > this.cuboidModel.vpr.x + consts.MIN_EDGE_LENGTH; + } else { + xStatus = + x > this.cuboidModel.fl.points[0].x + consts.MIN_EDGE_LENGTH && + x < this.cuboidModel.vpr.x - consts.MIN_EDGE_LENGTH; + } + return { x: xStatus, y: this.dlCenter.attr('y1') }; + }) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('resizestart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.dorsalLeftEdge.center(this.dlCenter.cx(), this.dlCenter.cy()); + + const x = this.dorsalLeftEdge.attr('x1'); + const y1 = this.cuboidModel.lt.getEquation().getY(x); + const y2 = this.cuboidModel.lb.getEquation().getY(x); + const topPoint = { x, y: y1 }; + const botPoint = { x, y: y2 }; + + this.cuboidModel.dl.points = [topPoint, botPoint]; + this.updateViewAndVM(true); + this.fire(new CustomEvent('resizing', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + + this.flCenter + .draggable((x: number) => { + const vpX = this.flCenter.cx() - this.cuboidModel.vpl.x > 0 ? this.cuboidModel.vpl.x : 0; + return { x: x < this.cuboidModel.fr.points[0].x && x > vpX + consts.MIN_EDGE_LENGTH }; + }) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('resizestart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.frontLeftEdge.center(this.flCenter.cx(), this.flCenter.cy()); + + const x = this.frontLeftEdge.attr('x1'); + const y1 = this.cuboidModel.ft.getEquation().getY(x); + const y2 = this.cuboidModel.fb.getEquation().getY(x); + const topPoint = { x, y: y1 }; + const botPoint = { x, y: y2 }; + + this.cuboidModel.fl.points = [topPoint, botPoint]; + this.updateViewAndVM(); + this.fire(new CustomEvent('resizing', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + + this.frCenter + .draggable((x: number) => { + return { x: x > this.cuboidModel.fl.points[0].x, y: this.frCenter.attr('y1') }; + }) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('resizestart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.frontRightEdge.center(this.frCenter.cx(), this.frCenter.cy()); + + const x = this.frontRightEdge.attr('x1'); + const y1 = this.cuboidModel.ft.getEquation().getY(x); + const y2 = this.cuboidModel.fb.getEquation().getY(x); + const topPoint = { x, y: y1 }; + const botPoint = { x, y: y2 }; + + this.cuboidModel.fr.points = [topPoint, botPoint]; + this.updateViewAndVM(true); + this.fire(new CustomEvent('resizing', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + + this.ftCenter + .draggable((x: number, y: number) => { + return { x: x === this.ftCenter.cx(), y: y < this.fbCenter.cy() - consts.MIN_EDGE_LENGTH }; + }) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('resizestart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.frontTopEdge.center(this.ftCenter.cx(), this.ftCenter.cy()); + horizontalEdgeControl.call( + this, + this.cuboidModel.top, + this.frontTopEdge.attr('x2'), + this.frontTopEdge.attr('y2'), + ); + this.updateViewAndVM(); + this.fire(new CustomEvent('resizing', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + + this.fbCenter + .draggable((x: number, y: number) => { + return { x: x === this.fbCenter.cx(), y: y > this.ftCenter.cy() + consts.MIN_EDGE_LENGTH }; + }) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('resizestart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.frontBotEdge.center(this.fbCenter.cx(), this.fbCenter.cy()); + horizontalEdgeControl.call( + this, + this.cuboidModel.bot, + this.frontBotEdge.attr('x2'), + this.frontBotEdge.attr('y2'), + ); + this.updateViewAndVM(); + this.fire(new CustomEvent('resizing', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('resizedone', event)); + }); + + return this; + }, + + draggable(value: any, constraint: any) { + const { cuboidModel } = this; + const faces = [this.face, this.right, this.dorsal, this.left]; + const accumulatedOffset: Point = { + x: 0, + y: 0, + }; + + if (value === false) { + faces.forEach((face: any) => { + face.draggable(false); + face.off('dragstart'); + face.off('dragmove'); + face.off('dragend'); + }); + return; + } + + this.face + .draggable() + .on('dragstart', (event: CustomEvent) => { + accumulatedOffset.x = 0; + accumulatedOffset.y = 0; + + this.fire(new CustomEvent('dragstart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + const dx = event.detail.p.x - event.detail.handler.startPoints.point.x; + const dy = event.detail.p.y - event.detail.handler.startPoints.point.y; + let dxPortion = dx - accumulatedOffset.x; + let dyPortion = dy - accumulatedOffset.y; + accumulatedOffset.x += dxPortion; + accumulatedOffset.y += dyPortion; + + this.dmove(dxPortion, dyPortion); + + this.fire(new CustomEvent('dragmove', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('dragend', event)); + }); + + this.left + .draggable((x: number, y: number) => ({ + x: x < Math.min(cuboidModel.dr.points[0].x, cuboidModel.fr.points[0].x) - consts.MIN_EDGE_LENGTH, + y, + })) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('dragstart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.cuboidModel.left.points = parsePoints(this.left.attr('points')); + this.updateViewAndVM(); + + this.fire(new CustomEvent('dragmove', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('dragend', event)); + }); + + this.dorsal + .draggable() + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('dragstart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.cuboidModel.dorsal.points = parsePoints(this.dorsal.attr('points')); + this.updateViewAndVM(); + + this.fire(new CustomEvent('dragmove', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('dragend', event)); + }); + + this.right + .draggable((x: number, y: number) => ({ + x: x > Math.min(cuboidModel.dl.points[0].x, cuboidModel.fl.points[0].x) + consts.MIN_EDGE_LENGTH, + y, + })) + .on('dragstart', (event: CustomEvent) => { + this.fire(new CustomEvent('dragstart', event)); + }) + .on('dragmove', (event: CustomEvent) => { + this.cuboidModel.right.points = parsePoints(this.right.attr('points')); + this.updateViewAndVM(true); + + this.fire(new CustomEvent('dragmove', event)); + }) + .on('dragend', (event: CustomEvent) => { + this.fire(new CustomEvent('dragend', event)); + }); + + return this; + }, + + _attr: SVG.Element.prototype.attr, + + attr(a: any, v: any, n: any) { + if ((a === 'fill' || a === 'stroke' || a === 'face-stroke') && v !== undefined) { + this._attr(a, v, n); + this.paintOrientationLines(); + } else if (a === 'points' && typeof v === 'string') { + const points = parsePoints(v); + this.cuboidModel.setPoints(points); + this.updateViewAndVM(); + } else if (a === 'projections') { + this._attr(a, v, n); + if (v === true) { + this.ftProj.show(); + this.fbProj.show(); + this.rtProj.show(); + this.rbProj.show(); + } else { + this.ftProj.hide(); + this.fbProj.hide(); + this.rtProj.hide(); + this.rbProj.hide(); + } + } else if (a === 'stroke-width' && typeof v === 'number') { + this._attr(a, v, n); + this.updateThickness(); + } else if (a === 'data-z-order' && typeof v !== 'undefined') { + this._attr(a, v, n); + [this.face, this.left, this.dorsal, this.right, ...this.getEdges(), ...this.getGrabPoints()].forEach( + (el) => { + if (el) el.attr(a, v, n); + }, + ); + } else { + return this._attr(a, v, n); + } + + return this; + }, + + updateThickness() { + const edges = [this.frontLeftEdge, this.frontRightEdge, this.frontTopEdge, this.frontBotEdge]; + const width = this.attr('stroke-width'); + edges.forEach((edge: SVG.Element) => { + edge.attr('stroke-width', width * (this.strokeOffset || consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH)); + }); + this.on('mouseover', () => { + edges.forEach((edge: SVG.Element) => { + this.strokeOffset = this.node.classList.contains('cvat_canvas_shape_activated') + ? consts.CUBOID_ACTIVE_EDGE_STROKE_WIDTH + : consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH; + edge.attr('stroke-width', width * this.strokeOffset); + }); + }).on('mouseout', () => { + edges.forEach((edge: SVG.Element) => { + this.strokeOffset = consts.CUBOID_UNACTIVE_EDGE_STROKE_WIDTH; + edge.attr('stroke-width', width * this.strokeOffset); + }); + }); + }, + + paintOrientationLines() { + // style has higher priority than attr, so then try to fetch it if exists + // https://stackoverflow.com/questions/47088409/svg-attributes-beaten-by-cssstyle-in-priority] + // we use getComputedStyle to get actual, not-inlined css property (come from the corresponding css class) + const computedStyles = getComputedStyle(this.node); + const fillColor = computedStyles['fill'] || this.attr('fill'); + const strokeColor = computedStyles['stroke'] || this.attr('stroke'); + const selectedColor = this.attr('face-stroke') || '#b0bec5'; + this.frontTopEdge.stroke({ color: selectedColor }); + this.frontLeftEdge.stroke({ color: selectedColor }); + this.frontBotEdge.stroke({ color: selectedColor }); + this.frontRightEdge.stroke({ color: selectedColor }); + + this.rightTopEdge.stroke({ color: strokeColor }); + this.rightBotEdge.stroke({ color: strokeColor }); + this.dorsalRightEdge.stroke({ color: strokeColor }); + this.dorsalLeftEdge.stroke({ color: strokeColor }); + + this.bot.stroke({ color: strokeColor }).fill({ color: fillColor }); + this.top.stroke({ color: strokeColor }).fill({ color: fillColor }); + this.face.stroke({ color: strokeColor, width: 0 }).fill({ color: fillColor }); + this.right.stroke({ color: strokeColor }).fill({ color: fillColor }); + this.dorsal.stroke({ color: strokeColor }).fill({ color: fillColor }); + this.left.stroke({ color: strokeColor }).fill({ color: fillColor }); + }, + + dmove(dx: number, dy: number) { + this.cuboidModel.points.forEach((point: Point) => { + point.x += dx; + point.y += dy; + }); + + this.updateViewAndVM(); + }, + + x(x?: number) { + if (typeof x === 'number') { + const { x: xInitial } = this.bbox(); + this.dmove(x - xInitial, 0); + return this; + } else { + return this.bbox().x; + } + }, + + y(y?: number) { + if (typeof y === 'number') { + const { y: yInitial } = this.bbox(); + this.dmove(0, y - yInitial); + return this; + } else { + return this.bbox().y; + } + }, + + resetPerspective() { + if (this.cuboidModel.orientation === Orientation.LEFT) { + const edgePoints = this.cuboidModel.dl.points; + const constraints = this.cuboidModel.computeSideEdgeConstraints(this.cuboidModel.dl); + edgePoints[0].y = constraints.y1Range.min; + this.cuboidModel.dl.points = [edgePoints[0], edgePoints[1]]; + this.updateViewAndVM(true); + } else { + const edgePoints = this.cuboidModel.dr.points; + const constraints = this.cuboidModel.computeSideEdgeConstraints(this.cuboidModel.dr); + edgePoints[0].y = constraints.y1Range.min; + this.cuboidModel.dr.points = [edgePoints[0], edgePoints[1]]; + this.updateViewAndVM(); + } + }, + + updateViewAndVM(build: boolean) { + this.cuboidModel.updateOrientation(); + this.cuboidModel.buildBackEdge(build); + this.updateView(); + + // to correct getting of points in resizedone, dragdone + this._attr( + 'points', + this.cuboidModel + .getPoints() + .reduce((acc: string, point: Point): string => `${acc} ${point.x},${point.y}`, '') + .trim(), + ); + }, + + computeHeightFace(point: Point, index: number) { + switch (index) { + // fl + case 1: { + const p2 = this.updatedEdge(this.cuboidModel.fr.points[0], point, this.cuboidModel.vpl); + const p3 = this.updatedEdge(this.cuboidModel.dr.points[0], p2, this.cuboidModel.vpr); + const p4 = this.updatedEdge(this.cuboidModel.dl.points[0], point, this.cuboidModel.vpr); + return [point, p2, p3, p4]; + } + // fr + case 2: { + const p1 = this.updatedEdge(this.cuboidModel.fl.points[0], point, this.cuboidModel.vpl); + const p3 = this.updatedEdge(this.cuboidModel.dr.points[0], point, this.cuboidModel.vpr); + const p4 = this.updatedEdge(this.cuboidModel.dl.points[0], p3, this.cuboidModel.vpr); + return [p1, point, p3, p4]; + } + // dr + case 3: { + const p2 = this.updatedEdge(this.cuboidModel.dl.points[0], point, this.cuboidModel.vpl); + const p3 = this.updatedEdge(this.cuboidModel.fr.points[0], point, this.cuboidModel.vpr); + const p4 = this.updatedEdge(this.cuboidModel.fl.points[0], p2, this.cuboidModel.vpr); + return [p4, p3, point, p2]; + } + // dl + case 4: { + const p2 = this.updatedEdge(this.cuboidModel.dr.points[0], point, this.cuboidModel.vpl); + const p3 = this.updatedEdge(this.cuboidModel.fl.points[0], point, this.cuboidModel.vpr); + const p4 = this.updatedEdge(this.cuboidModel.fr.points[0], p2, this.cuboidModel.vpr); + return [p3, p4, p2, point]; + } + default: { + return [null, null, null, null]; + } + } + }, + + updatedEdge(target: Point, base: Point, pivot: Point) { + const targetX = target.x; + const line = new Equation(pivot, base); + const newY = line.getY(targetX); + return { x: targetX, y: newY }; + }, + + updateView() { + this.updateFaces(); + this.updateEdges(); + this.updateProjections(); + this.updateGrabPoints(); + }, + + updateFaces() { + const viewModel = this.cuboidModel; + + this.bot.plot(viewModel.bot.points); + this.top.plot(viewModel.top.points); + this.right.plot(viewModel.right.points); + this.dorsal.plot(viewModel.dorsal.points); + this.left.plot(viewModel.left.points); + this.face.plot(viewModel.front.points); + }, + + updateEdges() { + const viewModel = this.cuboidModel; + + this.frontLeftEdge.plot(viewModel.fl.points); + this.frontRightEdge.plot(viewModel.fr.points); + this.dorsalRightEdge.plot(viewModel.dr.points); + this.dorsalLeftEdge.plot(viewModel.dl.points); + + this.frontTopEdge.plot(viewModel.ft.points); + this.rightTopEdge.plot(viewModel.rt.points); + this.frontBotEdge.plot(viewModel.fb.points); + this.rightBotEdge.plot(viewModel.rb.points); + }, + + updateProjections() { + const viewModel = this.cuboidModel; + + this.ftProj.plot( + this.updateProjectionLine(viewModel.ft.getEquation(), viewModel.ft.points[0], viewModel.vpl), + ); + this.fbProj.plot( + this.updateProjectionLine(viewModel.fb.getEquation(), viewModel.ft.points[0], viewModel.vpl), + ); + this.rtProj.plot( + this.updateProjectionLine(viewModel.rt.getEquation(), viewModel.rt.points[1], viewModel.vpr), + ); + this.rbProj.plot( + this.updateProjectionLine(viewModel.rb.getEquation(), viewModel.rt.points[1], viewModel.vpr), + ); + }, + + updateGrabPoints() { + const centers = this.getGrabPoints(); + const edges = this.getEdges(); + for (let i = 0; i < centers.length; i += 1) { + const edge = edges[i]; + if (centers[i]) centers[i].center(edge.cx(), edge.cy()); + } + }, + }, + construct: { + cube(points: string) { + return this.put(new (SVG as any).Cube()).constructorMethod(points); + }, + }, +}); diff --git a/cvat-canvas/src/typescript/zoomHandler.ts b/cvat-canvas/src/typescript/zoomHandler.ts new file mode 100644 index 0000000..02e9e2b --- /dev/null +++ b/cvat-canvas/src/typescript/zoomHandler.ts @@ -0,0 +1,137 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import consts from './consts'; + +import { translateToSVG } from './shared'; + +import { Geometry } from './canvasModel'; + +export interface ZoomHandler { + zoom(): void; + cancel(): void; + transform(geometry: Geometry): void; +} + +export class ZoomHandlerImpl implements ZoomHandler { + private onZoomRegion: (x: number, y: number, width: number, height: number) => void; + private bindedOnSelectStart: (event: MouseEvent) => void; + private bindedOnSelectUpdate: (event: MouseEvent) => void; + private bindedOnSelectStop: (event: MouseEvent) => void; + private geometry: Geometry; + private canvas: SVG.Container; + private selectionRect: SVG.Rect | null; + private startSelectionPoint: { + x: number; + y: number; + }; + + private onSelectStart(event: MouseEvent): void { + if (!this.selectionRect && event.which === 1) { + const point = translateToSVG((this.canvas.node as any) as SVGSVGElement, [event.clientX, event.clientY]); + this.startSelectionPoint = { + x: point[0], + y: point[1], + }; + + this.selectionRect = this.canvas.rect().addClass('cvat_canvas_zoom_selection'); + this.selectionRect.attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + ...this.startSelectionPoint, + }); + } + } + + private getSelectionBox( + event: MouseEvent, + ): { + x: number; + y: number; + width: number; + height: number; + } { + const point = translateToSVG((this.canvas.node as any) as SVGSVGElement, [event.clientX, event.clientY]); + const stopSelectionPoint = { + x: point[0], + y: point[1], + }; + + const xtl = Math.min(this.startSelectionPoint.x, stopSelectionPoint.x); + const ytl = Math.min(this.startSelectionPoint.y, stopSelectionPoint.y); + const xbr = Math.max(this.startSelectionPoint.x, stopSelectionPoint.x); + const ybr = Math.max(this.startSelectionPoint.y, stopSelectionPoint.y); + + return { + x: xtl, + y: ytl, + width: xbr - xtl, + height: ybr - ytl, + }; + } + + private onSelectUpdate(event: MouseEvent): void { + if (this.selectionRect) { + this.selectionRect.attr({ + ...this.getSelectionBox(event), + }); + } + } + + private onSelectStop(event: MouseEvent): void { + if (this.selectionRect) { + const box = this.getSelectionBox(event); + this.selectionRect.remove(); + this.selectionRect = null; + this.startSelectionPoint = { + x: 0, + y: 0, + }; + const threshold = 5; + if (box.width > threshold && box.height > threshold) { + this.onZoomRegion(box.x, box.y, box.width, box.height); + } + } + } + + public constructor( + onZoomRegion: ZoomHandlerImpl['onZoomRegion'], + canvas: SVG.Container, + geometry: Geometry, + ) { + this.onZoomRegion = onZoomRegion; + this.canvas = canvas; + this.geometry = geometry; + this.selectionRect = null; + this.startSelectionPoint = { + x: 0, + y: 0, + }; + this.bindedOnSelectStart = this.onSelectStart.bind(this); + this.bindedOnSelectUpdate = this.onSelectUpdate.bind(this); + this.bindedOnSelectStop = this.onSelectStop.bind(this); + } + + public zoom(): void { + this.canvas.node.addEventListener('mousedown', this.bindedOnSelectStart); + this.canvas.node.addEventListener('mousemove', this.bindedOnSelectUpdate); + this.canvas.node.addEventListener('mouseup', this.bindedOnSelectStop); + } + + public cancel(): void { + this.canvas.node.removeEventListener('mousedown', this.bindedOnSelectStart); + this.canvas.node.removeEventListener('mousemove', this.bindedOnSelectUpdate); + this.canvas.node.removeEventListener('mouseup ', this.bindedOnSelectStop); + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + if (this.selectionRect) { + this.selectionRect.style({ + 'stroke-width': consts.BASE_STROKE_WIDTH / geometry.scale, + }); + } + } +} diff --git a/cvat-canvas/tsconfig.json b/cvat-canvas/tsconfig.json new file mode 100644 index 0000000..1e2c6dc --- /dev/null +++ b/cvat-canvas/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": ["dom", "dom.iterable", "esnext"], + "skipLibCheck": true, + "strict": false, + "forceConsistentCasingInFileNames": true, + "module": "esnext", + "moduleResolution": "node", + "esModuleInterop": true, + "noEmit": true + }, + "include": ["src/typescript/canvas.ts"] +} diff --git a/cvat-canvas/webpack.config.cjs b/cvat-canvas/webpack.config.cjs new file mode 100644 index 0000000..1bc95fc --- /dev/null +++ b/cvat-canvas/webpack.config.cjs @@ -0,0 +1,77 @@ +// Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const path = require('path'); + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const BundleDeclarationsWebpackPlugin = require('bundle-declarations-webpack-plugin'); + +const styleLoaders = [ + 'style-loader', + { + loader: 'css-loader', + options: { + importLoaders: 2, + }, + }, + { + loader: 'postcss-loader', + options: { + postcssOptions: { + plugins: [ + [ + 'postcss-preset-env', {}, + ], + ], + }, + }, + }, + 'sass-loader', +]; + +module.exports = { + target: 'web', + mode: 'production', + devtool: 'source-map', + entry: { + 'cvat-canvas': './src/typescript/canvas.ts', + }, + output: { + path: path.resolve(__dirname, 'dist'), + filename: '[name].[contenthash].js', + library: 'canvas', + libraryTarget: 'window', + }, + resolve: { + extensions: ['.ts', '.js', '.json'], + }, + module: { + rules: [ + { + test: /\.ts$/, + exclude: /node_modules/, + use: { + loader: 'babel-loader', + options: { + plugins: ['@babel/plugin-proposal-class-properties'], + presets: ['@babel/preset-env', '@babel/typescript'], + sourceType: 'unambiguous', + }, + }, + }, + { + test: /\.scss$/, + exclude: /node_modules/, + use: styleLoaders, + }, + ], + }, + plugins: [ + new BundleDeclarationsWebpackPlugin({ + outFile: "declaration/src/cvat-canvas.d.ts", + }), + ], +}; diff --git a/cvat-canvas3d/.eslintrc.cjs b/cvat-canvas3d/.eslintrc.cjs new file mode 100644 index 0000000..991f43e --- /dev/null +++ b/cvat-canvas3d/.eslintrc.cjs @@ -0,0 +1,17 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +module.exports = { + parserOptions: { + project: './tsconfig.json', + tsconfigRootDir: __dirname, + }, + ignorePatterns: [ + '.eslintrc.cjs', + 'webpack.config.js', + 'node_modules/**', + 'dist/**', + ], +}; diff --git a/cvat-canvas3d/README.md b/cvat-canvas3d/README.md new file mode 100644 index 0000000..fcdbc99 --- /dev/null +++ b/cvat-canvas3d/README.md @@ -0,0 +1,53 @@ +# Module CVAT-CANVAS-3D + +## Description + +The CVAT module written in TypeScript language. +It presents a canvas to viewing, drawing and editing of 3D annotations. + +## Commands + +- Building of the module from sources in the `dist` directory: + +```bash +yarn run build +yarn run build --mode=development # without a minification +``` + +### API Methods + +```ts +interface Canvas3d { + html(): ViewsDOM; + setup(frameData: any, objectStates: any[]): void; + isAbleToChangeFrame(): boolean; + mode(): Mode; + render(): void; + keyControls(keys: KeyboardEvent): void; + draw(drawData: DrawData): void; + cancel(): void; + dragCanvas(enable: boolean): void; + activate(clientID: number | null, attributeID?: number): void; + configure(configuration: Configuration): void; + fitCanvas(): void; + fit(): void; + group(groupData: GroupData): void; +} +``` + +### WEB + +```js +// Create an instance of a canvas +const canvas = new window.canvas.Canvas3d(); + +console.log('Version ', window.canvas.CanvasVersion); +console.log('Current mode is ', window.canvas.mode()); + +// Put canvas to a html container +const views = canvas.html(); +htmlContainer.appendChild(views.perspective); +htmlContainer.appendChild(views.top); +htmlContainer.appendChild(views.side); +htmlContainer.appendChild(views.front); +``` diff --git a/cvat-canvas3d/package.json b/cvat-canvas3d/package.json new file mode 100644 index 0000000..e201a85 --- /dev/null +++ b/cvat-canvas3d/package.json @@ -0,0 +1,24 @@ +{ + "name": "cvat-canvas3d", + "version": "0.0.10", + "type": "module", + "description": "Part of Computer Vision Annotation Tool which presents its canvas3D library", + "main": "src/canvas3d.ts", + "scripts": { + "build": "tsc && webpack --config ./webpack.config.cjs" + }, + "author": "CVAT.ai", + "license": "MIT", + "browserslist": [ + "Chrome >= 99", + "Firefox >= 110", + "not IE 11", + "> 2%" + ], + "dependencies": { + "@types/three": "0.184.0", + "camera-controls": "^1.25.3", + "cvat-core": "link:./../cvat-core", + "three": "0.184.0" + } +} diff --git a/cvat-canvas3d/src/typescript/canvas3d.ts b/cvat-canvas3d/src/typescript/canvas3d.ts new file mode 100644 index 0000000..28a9b0a --- /dev/null +++ b/cvat-canvas3d/src/typescript/canvas3d.ts @@ -0,0 +1,134 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { Canvas3dController, Canvas3dControllerImpl } from './canvas3dController'; +import { + Canvas3dModel, + Canvas3dModelImpl, + Mode, + DrawData, + ViewType, + MouseInteraction, + Configuration, + GroupData, + SplitData, + MergeData, +} from './canvas3dModel'; +import { + Canvas3dView, Canvas3dViewImpl, ViewsDOM, CameraAction, +} from './canvas3dView'; +import { Master } from './master'; + +interface Canvas3d { + html(): ViewsDOM; + setup(frameData: any, objectStates: any[]): void; + isAbleToChangeFrame(): boolean; + mode(): Mode; + render(): void; + keyControls(keys: KeyboardEvent): void; + draw(drawData: DrawData): void; + cancel(): void; + dragCanvas(enable: boolean): void; + activate(clientID: number | null, attributeID?: number): void; + configure(configuration: Configuration): void; + fitCanvas(): void; + fit(): void; + focus(clientID: number): void; + group(groupData: GroupData): void; + merge(mergeData: MergeData): void; + split(splitData: SplitData): void; + destroy(): void; +} + +class Canvas3dImpl implements Canvas3d { + private readonly model: Canvas3dModel & Master; + private readonly controller: Canvas3dController; + private view: Canvas3dView; + + public constructor() { + this.model = new Canvas3dModelImpl(); + this.controller = new Canvas3dControllerImpl(this.model); + this.view = new Canvas3dViewImpl(this.model, this.controller); + } + + public html(): ViewsDOM { + return this.view.html(); + } + + public keyControls(keys: KeyboardEvent): void { + this.view.keyControls(keys); + } + + public render(): void { + this.view.render(); + } + + public draw(drawData: DrawData): void { + this.model.draw(drawData); + } + + public setup(frameData: any, objectStates: any[]): void { + this.model.setup(frameData, objectStates); + } + + public mode(): Mode { + return this.model.mode; + } + + public group(groupData: GroupData): void { + this.model.group(groupData); + } + + public split(splitData: SplitData): void { + this.model.split(splitData); + } + + public merge(mergeData: MergeData): void { + this.model.merge(mergeData); + } + + public isAbleToChangeFrame(): boolean { + return this.model.isAbleToChangeFrame(); + } + + public cancel(): void { + this.model.cancel(); + } + + public dragCanvas(enable: boolean): void { + this.model.dragCanvas(enable); + } + + public configure(configuration: Partial): void { + this.model.configure(configuration); + } + + public activate(clientID: number | null, attributeID: number | null = null): void { + this.model.activate(typeof clientID === 'number' ? String(clientID) : null, attributeID); + } + + public focus(clientID: number): void { + this.view.focusObjectByClientId(clientID, true); + } + + public fit(): void { + this.model.fit(); + } + + public fitCanvas(): void { + // in spite of 2D canvas, 3D version fits automatically when external container resized + // so, nothing to do here, but keep the method to keep the same interface as 2D canvas + } + + public destroy(): void { + this.model.destroy(); + } +} + +export { + Canvas3dImpl as Canvas3d, ViewType, MouseInteraction, CameraAction, Mode as CanvasMode, +}; + +export type { ViewsDOM }; diff --git a/cvat-canvas3d/src/typescript/canvas3dController.ts b/cvat-canvas3d/src/typescript/canvas3dController.ts new file mode 100644 index 0000000..473cf80 --- /dev/null +++ b/cvat-canvas3d/src/typescript/canvas3dController.ts @@ -0,0 +1,70 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ObjectState } from '.'; +import { + Canvas3dModel, Mode, DrawData, ActiveElement, + GroupData, MergeData, SplitData, +} from './canvas3dModel'; + +export interface Canvas3dController { + readonly drawData: DrawData; + readonly activeElement: ActiveElement; + readonly groupData: GroupData; + readonly imageIsDeleted: boolean; + readonly objects: ObjectState[]; + mode: Mode; + group(groupData: GroupData): void; + merge(mergeData: MergeData): void; + split(splitData: SplitData): void; +} + +export class Canvas3dControllerImpl implements Canvas3dController { + private model: Canvas3dModel; + + public constructor(model: Canvas3dModel) { + this.model = model; + } + + public set mode(value: Mode) { + this.model.mode = value; + } + + public get mode(): Mode { + return this.model.mode; + } + + public get drawData(): DrawData { + return this.model.data.drawData; + } + + public get activeElement(): ActiveElement { + return this.model.data.activeElement; + } + + public get imageIsDeleted(): any { + return this.model.imageIsDeleted; + } + + public get groupData(): GroupData { + return this.model.groupData; + } + + public get objects(): ObjectState[] { + return this.model.objects; + } + + public group(groupData: GroupData): void { + this.model.group(groupData); + } + + public merge(mergeData: MergeData): void { + this.model.merge(mergeData); + } + + public split(splitData: SplitData): void { + this.model.split(splitData); + } +} diff --git a/cvat-canvas3d/src/typescript/canvas3dModel.ts b/cvat-canvas3d/src/typescript/canvas3dModel.ts new file mode 100644 index 0000000..bd56e79 --- /dev/null +++ b/cvat-canvas3d/src/typescript/canvas3dModel.ts @@ -0,0 +1,464 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ObjectState } from '.'; +import { MasterImpl } from './master'; +import consts from './consts'; + +export interface Size { + width: number; + height: number; +} + +export interface ActiveElement { + clientID: string | null; + attributeID: number | null; +} + +export interface GroupData { + enabled: boolean; +} + +export interface MergeData { + enabled: boolean; +} + +export interface SplitData { + enabled: boolean; +} + +export interface Image { + renderWidth: number; + renderHeight: number; + imageData: Blob; +} + +export interface DrawData { + enabled: boolean; + initialState?: any; + redraw?: number; + shapeType?: string; +} + +export enum FrameZoom { + MIN = 0.1, + MAX = 10, +} + +export enum Planes { + TOP = 'topPlane', + SIDE = 'sidePlane', + FRONT = 'frontPlane', + PERSPECTIVE = 'perspectivePlane', +} + +export enum ViewType { + PERSPECTIVE = 'perspective', + TOP = 'top', + SIDE = 'side', + FRONT = 'front', +} + +export enum MouseInteraction { + CLICK = 'click', + DOUBLE_CLICK = 'dblclick', + HOVER = 'hover', +} + +export interface OrientationVisibility { + x: boolean; + y: boolean; + z: boolean; +} + +export interface Configuration { + colorBy: string; + shapeOpacity: number; + outlinedBorders: string | false; + selectedShapeOpacity: number; + controlPointsSize: number; + focusedObjectPadding: number; + orientationVisibility: OrientationVisibility; +} + +export enum UpdateReasons { + IMAGE_CHANGED = 'image_changed', + OBJECTS_UPDATED = 'objects_updated', + DRAW = 'draw', + SELECT = 'select', + CANCEL = 'cancel', + DRAG_CANVAS = 'drag_canvas', + SHAPE_ACTIVATED = 'shape_activated', + GROUP = 'group', + MERGE = 'merge', + SPLIT = 'split', + FITTED_CANVAS = 'fitted_canvas', + CONFIG_UPDATED = 'config_updated', + DATA_FAILED = 'data_failed', +} + +export enum Mode { + IDLE = 'idle', + DRAW = 'draw', + EDIT = 'edit', + DRAG_CANVAS = 'drag_canvas', + GROUP = 'group', + MERGE = 'merge', + SPLIT = 'split', +} + +export interface Canvas3dDataModel { + activeElement: ActiveElement; + canvasSize: Size; + image: { imageData: Blob } | null; + imageID: number | null; + imageOffset: number; + imageSize: Size; + imageIsDeleted: boolean; + drawData: DrawData; + mode: Mode; + objects: ObjectState[]; + configuration: Configuration; + groupData: GroupData; + mergeData: MergeData; + splitData: SplitData; + isFrameUpdating: boolean; + nextSetupRequest: { + frameData: any; + objectStates: ObjectState[]; + } | null; + exception: Error | null; +} + +export interface Canvas3dModel { + mode: Mode; + data: Canvas3dDataModel; + readonly imageIsDeleted: boolean; + readonly groupData: GroupData; + readonly mergeData: MergeData; + readonly objects: ObjectState[]; + readonly exception: Error | null; + setup(frameData: any, objectStates: ObjectState[]): void; + isAbleToChangeFrame(): boolean; + draw(drawData: DrawData): void; + cancel(): void; + dragCanvas(enable: boolean): void; + activate(clientID: string | null, attributeID: number | null): void; + configure(configuration: Partial): void; + fit(): void; + group(groupData: GroupData): void; + split(splitData: SplitData): void; + merge(mergeData: MergeData): void; + destroy(): void; + updateCanvasObjects(): void; + unlockFrameUpdating(): void; +} + +export class Canvas3dModelImpl extends MasterImpl implements Canvas3dModel { + public data: Canvas3dDataModel; + + public constructor() { + super(); + this.data = { + activeElement: { + clientID: null, + attributeID: null, + }, + canvasSize: { + height: 0, + width: 0, + }, + objects: [], + image: null, + imageID: null, + imageOffset: 0, + imageSize: { + height: 0, + width: 0, + }, + imageIsDeleted: false, + drawData: { + enabled: false, + initialState: null, + }, + mode: Mode.IDLE, + groupData: { + enabled: false, + }, + mergeData: { + enabled: false, + }, + splitData: { + enabled: false, + }, + configuration: { + colorBy: 'Label', + shapeOpacity: 40, + outlinedBorders: false, + selectedShapeOpacity: 60, + focusedObjectPadding: 50, + controlPointsSize: consts.BASE_POINT_SIZE, + orientationVisibility: { + x: false, + y: false, + z: false, + }, + }, + isFrameUpdating: false, + nextSetupRequest: null, + exception: null, + }; + } + + public updateCanvasObjects(): void { + this.notify(UpdateReasons.OBJECTS_UPDATED); + } + + public unlockFrameUpdating(): void { + this.data.isFrameUpdating = false; + if (this.data.nextSetupRequest) { + try { + const { frameData, objectStates } = this.data.nextSetupRequest; + this.setup(frameData, objectStates); + } finally { + this.data.nextSetupRequest = null; + } + } + } + + public setup(frameData: any, objectStates: ObjectState[]): void { + if (this.data.imageID !== frameData.number) { + if ([Mode.EDIT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + } + + if (this.data.isFrameUpdating) { + this.data.nextSetupRequest = { + frameData, objectStates, + }; + return; + } + + if (frameData.number === this.data.imageID && frameData.deleted === this.data.imageIsDeleted) { + this.data.objects = objectStates; + this.notify(UpdateReasons.OBJECTS_UPDATED); + return; + } + + this.data.isFrameUpdating = true; + this.data.imageID = frameData.number; + frameData + .data((): void => { + this.data.image = null; + this.notify(UpdateReasons.IMAGE_CHANGED); + }) + .then((data: { imageData: Blob }): void => { + this.data.imageSize = { + height: frameData.height as number, + width: frameData.width as number, + }; + this.data.imageIsDeleted = frameData.deleted; + this.data.image = data; + this.data.objects = objectStates; + this.notify(UpdateReasons.IMAGE_CHANGED); + }) + .catch((exception: any): void => { + this.data.isFrameUpdating = false; + // don't notify when the frame is no longer needed + if (typeof exception !== 'number' || exception === this.data.imageID) { + if (exception instanceof Error) { + this.data.exception = exception; + } else { + this.data.exception = new Error('Unknown error occurred when fetching image data'); + } + this.notify(UpdateReasons.DATA_FAILED); + } + }); + } + + public set mode(value: Mode) { + this.data.mode = value; + } + + public get mode(): Mode { + return this.data.mode; + } + + public get objects(): ObjectState[] { + return [...this.data.objects]; + } + + public isAbleToChangeFrame(): boolean { + const isUnable = [Mode.EDIT].includes(this.data.mode) || + this.data.isFrameUpdating || (this.data.mode === Mode.DRAW && typeof this.data.drawData.redraw === 'number'); + return !isUnable; + } + + public draw(drawData: DrawData): void { + if (drawData.enabled && this.data.drawData.enabled && !drawData.initialState) { + throw new Error('Drawing has been already started'); + } + if ([Mode.DRAW, Mode.EDIT].includes(this.data.mode) && !drawData.initialState) { + return; + } + this.data.drawData.enabled = drawData.enabled; + this.data.mode = Mode.DRAW; + + if (typeof drawData.redraw === 'number') { + const clientID = drawData.redraw; + const [state] = this.data.objects.filter((_state: any): boolean => _state.clientID === clientID); + + if (state) { + this.data.drawData = { ...drawData }; + this.data.drawData.initialState = { ...this.data.drawData.initialState, label: state.label }; + this.data.drawData.shapeType = state.shapeType; + } else { + return; + } + } else { + this.data.drawData = { ...drawData }; + if (this.data.drawData.initialState) { + this.data.drawData.shapeType = this.data.drawData.initialState.shapeType; + } + } + this.notify(UpdateReasons.DRAW); + } + + public cancel(): void { + this.notify(UpdateReasons.CANCEL); + } + + public dragCanvas(enable: boolean): void { + if (enable && this.data.mode !== Mode.IDLE) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (!enable && this.data.mode !== Mode.DRAG_CANVAS) { + throw Error(`Canvas is not in the drag mode. Action: ${this.data.mode}`); + } + + this.data.mode = enable ? Mode.DRAG_CANVAS : Mode.IDLE; + this.notify(UpdateReasons.DRAG_CANVAS); + } + + public activate(clientID: string | null, attributeID: number | null): void { + if (this.data.activeElement.clientID === clientID && this.data.activeElement.attributeID === attributeID) { + return; + } + if (this.data.mode !== Mode.IDLE && clientID !== null) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + if (typeof clientID === 'number') { + const [state] = this.data.objects.filter((_state: any): boolean => _state.clientID === clientID); + if (!state || state.objectType === 'tag') { + return; + } + } + this.data.activeElement = { + clientID, + attributeID, + }; + this.notify(UpdateReasons.SHAPE_ACTIVATED); + } + + public group(groupData: GroupData): void { + if (![Mode.IDLE, Mode.GROUP].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (this.data.groupData.enabled && groupData.enabled) { + return; + } + + if (!this.data.groupData.enabled && !groupData.enabled) { + return; + } + this.data.mode = groupData.enabled ? Mode.GROUP : Mode.IDLE; + this.data.groupData = { ...groupData }; + this.notify(UpdateReasons.GROUP); + } + + public split(splitData: SplitData): void { + if (![Mode.IDLE, Mode.SPLIT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + this.data.mode = splitData.enabled ? Mode.SPLIT : Mode.IDLE; + this.data.splitData = { ...splitData }; + this.notify(UpdateReasons.SPLIT); + } + + public merge(mergeData: MergeData): void { + if (![Mode.IDLE, Mode.MERGE].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + this.data.mode = mergeData.enabled ? Mode.MERGE : Mode.IDLE; + this.data.mergeData = { ...mergeData }; + this.notify(UpdateReasons.MERGE); + } + + public configure(configuration: Partial): void { + if (typeof configuration.shapeOpacity === 'number') { + this.data.configuration.shapeOpacity = Math.max(0, Math.min(configuration.shapeOpacity, 100)); + } + + if (typeof configuration.selectedShapeOpacity === 'number') { + this.data.configuration.selectedShapeOpacity = Math.max( + 0, Math.min(configuration.selectedShapeOpacity, 100), + ); + } + + if (['Label', 'Instance', 'Group'].includes(configuration.colorBy)) { + this.data.configuration.colorBy = configuration.colorBy; + } + + if (typeof configuration.outlinedBorders === 'string' || configuration.outlinedBorders === false) { + this.data.configuration.outlinedBorders = configuration.outlinedBorders; + } + + if (typeof configuration.controlPointsSize === 'number') { + this.data.configuration.controlPointsSize = configuration.controlPointsSize; + } + + if (typeof configuration.orientationVisibility === 'object') { + const current = this.data.configuration.orientationVisibility; + current.x = !!(configuration.orientationVisibility?.x ?? current.x); + current.y = !!(configuration.orientationVisibility?.y ?? current.y); + current.z = !!(configuration.orientationVisibility?.z ?? current.z); + } + + if (typeof configuration.focusedObjectPadding === 'number') { + this.data.configuration.focusedObjectPadding = Math.max( + configuration.focusedObjectPadding, 0, + ); + } + + this.notify(UpdateReasons.CONFIG_UPDATED); + } + + public fit(): void { + this.notify(UpdateReasons.FITTED_CANVAS); + } + + public get groupData(): GroupData { + return { ...this.data.groupData }; + } + + public get mergeData(): MergeData { + return { ...this.data.mergeData }; + } + + public get imageIsDeleted(): boolean { + return this.data.imageIsDeleted; + } + + public get exception(): Error | null { + return this.data.exception; + } + + public destroy(): void {} +} diff --git a/cvat-canvas3d/src/typescript/canvas3dView.ts b/cvat-canvas3d/src/typescript/canvas3dView.ts new file mode 100644 index 0000000..695c101 --- /dev/null +++ b/cvat-canvas3d/src/typescript/canvas3dView.ts @@ -0,0 +1,2558 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as THREE from 'three'; +import { PCDLoader } from 'three/examples/jsm/loaders/PCDLoader'; +import CameraControls from 'camera-controls'; +import { Canvas3dController } from './canvas3dController'; +import { Listener, Master } from './master'; +import consts from './consts'; +import { + Canvas3dModel, DrawData, Mode, Planes, UpdateReasons, ViewType, +} from './canvas3dModel'; +import { + createRotationHelper, removeRotationHelper, + createResizeHelper, removeResizeHelper, + createCuboidEdges, removeCuboidEdges, CuboidModel, makeCornerPointsMatrix, +} from './cuboid'; +import { ObjectState, ObjectType } from '.'; +import { disposeScene } from './utils'; + +export interface Canvas3dView { + html(): ViewsDOM; + render(): void; + keyControls(keys: KeyboardEvent): void; + focusObjectByClientId(clientID: number, animate?: boolean): void; +} + +export enum CameraAction { + ZOOM_IN = 'KeyI', + MOVE_UP = 'KeyU', + MOVE_DOWN = 'KeyO', + MOVE_LEFT = 'KeyJ', + ZOOM_OUT = 'KeyK', + MOVE_RIGHT = 'KeyL', + TILT_UP = 'ArrowUp', + TILT_DOWN = 'ArrowDown', + ROTATE_RIGHT = 'ArrowRight', + ROTATE_LEFT = 'ArrowLeft', +} + +export type Views = { + [key in ViewType]: RenderView; +}; + +export type ViewsDOM = { + [key in ViewType]: HTMLCanvasElement; +}; + +export interface RenderView { + renderer: THREE.WebGLRenderer; + scene: THREE.Scene; + camera?: THREE.PerspectiveCamera | THREE.OrthographicCamera; + controls?: CameraControls; + rayCaster?: { + renderer: THREE.Raycaster; + mouseVector: THREE.Vector2; + }; +} + +interface DrawnObjectData { + serverID: number | null; + clientID: number; + labelID: number; + labelColor: string; + points: number[]; + groupID: number | null; + groupColor: string; + color: string; + occluded: boolean; + outside: boolean; + hidden: boolean; + pinned: boolean; + lock: boolean; + updated: number; +} + +interface SideViewsZoomMemory { + serverID: number | null; + [ViewType.TOP]: number | null; + [ViewType.SIDE]: number | null; + [ViewType.FRONT]: number | null; +} + +const BOTTOM_VIEWS = [ + ViewType.TOP, + ViewType.SIDE, + ViewType.FRONT, +]; + +const ALL_VIEWS = [...BOTTOM_VIEWS, ViewType.PERSPECTIVE]; + +function drawnDataFromState(state: ObjectState): DrawnObjectData { + return { + serverID: state.serverID, + clientID: state.clientID, + labelID: state.label.id, + labelColor: state.label.color, + groupID: state.group?.id || null, + groupColor: state.group?.color || '#ffffff', + points: [...state.points], + color: state.color, + hidden: state.hidden, + lock: state.lock, + occluded: state.occluded, + outside: state.outside, + pinned: state.pinned, + updated: state.updated, + }; +} + +export class Canvas3dViewImpl implements Canvas3dView, Listener { + private controller: Canvas3dController; + private views: Views; + private clock: THREE.Clock; + private speed: number; + private cube: CuboidModel; + private isPerspectiveBeingDragged: boolean; + private activatedElementID: number | null; + private isCtrlDown: boolean; + private stateToBeSplitted: ObjectState | null; + private statesToBeGrouped: ObjectState[]; + private statesToBeMerged: ObjectState[]; + private sceneBBox: THREE.Box3; + private sideViewsZoomMemory: Record; + private model: Canvas3dModel & Master; + private drawnObjects: Record; + private hoverNeedsUpdate: boolean; + + public focusObjectByClientId(clientID: number, animate: boolean = true): void { + const cuboidModel = this.drawnObjects[clientID]; + if (cuboidModel) { + this.fitObject(cuboidModel.cuboid, animate); + } + } + + private action: { + translation: any; + resize: { + status: boolean; + previousPosition: null | THREE.Vector3; + helperElement: THREE.Object3D; + }; + scan: any; + rotation: any; + detected: any; + initialMouseVector: any; + }; + + private cameraSettings: { + [key in ViewType]: { + position: [number, number, number], + lookAt: [number, number, number], + up: [number, number, number], + } + }; + + private get selectedCuboid(): CuboidModel | null { + const { clientID } = this.model.data.activeElement; + if (clientID !== null) { + return this.drawnObjects[+clientID].cuboid || null; + } + + return null; + } + + private set mode(value: Mode) { + this.controller.mode = value; + } + + private get mode(): Mode { + return this.controller.mode; + } + + public constructor(model: Canvas3dModel & Master, controller: Canvas3dController) { + this.controller = controller; + this.clock = new THREE.Clock(); + this.speed = consts.MOVEMENT_FACTOR; + this.cube = new CuboidModel('line', '#ffffff'); + this.stateToBeSplitted = null; + this.statesToBeGrouped = []; + this.statesToBeMerged = []; + this.isPerspectiveBeingDragged = false; + this.activatedElementID = null; + this.sideViewsZoomMemory = {}; + this.drawnObjects = {}; + this.model = model; + this.sceneBBox = new THREE.Box3(); + this.cameraSettings = { + perspective: { + position: [-15, 0, 8], + lookAt: [10, 0, 0], + up: [0, 0, 1], + }, + top: { + position: [0, 0, 8], + lookAt: [0, 0, 0], + up: [0, 0, 1], + }, + side: { + position: [0, -8, 0], + lookAt: [0, 0, 0], + up: [0, 0, 1], + }, + front: { + position: [8, 0, 0], + lookAt: [0, 0, 0], + up: [0, 0, 1], + }, + }; + + this.isCtrlDown = false; + this.hoverNeedsUpdate = false; + this.action = { + scan: null, + detected: false, + initialMouseVector: new THREE.Vector2(), + translation: { + status: false, + helper: null, + coordinates: null, + offset: new THREE.Vector3(), + inverseMatrix: new THREE.Matrix4(), + }, + rotation: { + status: false, + helper: null, + recentMouseVector: new THREE.Vector2(0, 0), + screenInit: { + x: 0, + y: 0, + }, + screenMove: { + x: 0, + y: 0, + }, + }, + resize: { + status: false, + helperElement: null, + previousPosition: null, + }, + }; + + this.views = { + perspective: { + renderer: new THREE.WebGLRenderer({ antialias: true }), + scene: new THREE.Scene(), + rayCaster: { + renderer: new THREE.Raycaster(), + mouseVector: new THREE.Vector2(), + }, + }, + top: { + renderer: new THREE.WebGLRenderer({ antialias: true }), + scene: new THREE.Scene(), + rayCaster: { + renderer: new THREE.Raycaster(), + mouseVector: new THREE.Vector2(), + }, + }, + side: { + renderer: new THREE.WebGLRenderer({ antialias: true }), + scene: new THREE.Scene(), + rayCaster: { + renderer: new THREE.Raycaster(), + mouseVector: new THREE.Vector2(), + }, + }, + front: { + renderer: new THREE.WebGLRenderer({ antialias: true }), + scene: new THREE.Scene(), + rayCaster: { + renderer: new THREE.Raycaster(), + mouseVector: new THREE.Vector2(), + }, + }, + }; + CameraControls.install({ THREE }); + + const canvasPerspectiveView = this.views.perspective.renderer.domElement; + const canvasTopView = this.views.top.renderer.domElement; + const canvasSideView = this.views.side.renderer.domElement; + const canvasFrontView = this.views.front.renderer.domElement; + + [ + [canvasPerspectiveView, this.views.perspective.scene], + [canvasTopView, this.views.top.scene], + [canvasSideView, this.views.side.scene], + [canvasFrontView, this.views.front.scene], + ].forEach(([view, scene]) => { + Object.defineProperty(view, 'scene', { + value: scene, + enumerable: false, + configurable: false, + writable: false, + }); + + Object.defineProperty(view, 'getDrawnObjects', { + value: () => Object.values(this.drawnObjects).map((object) => { + const { clientID } = object.data; + return { + ...object, + state: this.model.objects.find((_state: ObjectState) => _state.clientID === clientID), + }; + }), + enumerable: false, + configurable: false, + writable: false, + }); + + Object.defineProperty(view, 'updatePosition', { + value: (state: ObjectState, points: number[]) => { + this.dispatchEvent( + new CustomEvent('canvas.edited', { + bubbles: false, + cancelable: true, + detail: { + state, + points: [...points], + }, + }), + ); + }, + enumerable: false, + configurable: false, + writable: false, + }); + }); + + canvasPerspectiveView.addEventListener('contextmenu', (e: MouseEvent): void => { + if (this.model.data.activeElement.clientID !== null) { + this.dispatchEvent( + new CustomEvent('canvas.contextmenu', { + bubbles: false, + cancelable: true, + detail: { + clientID: Number(this.model.data.activeElement.clientID), + clientX: e.clientX, + clientY: e.clientY, + }, + }), + ); + } + if (this.model.mode === Mode.DRAW && e.ctrlKey && this.model.data.drawData.initialState) { + const { x, y, z } = this.cube.perspective.position; + const { x: width, y: height, z: depth } = this.cube.perspective.scale; + const { x: rotationX, y: rotationY, z: rotationZ } = this.cube.perspective.rotation; + const points = [x, y, z, rotationX, rotationY, rotationZ, width, height, depth, 0, 0, 0, 0, 0, 0, 0]; + const initState = this.model.data.drawData.initialState; + this.dispatchEvent( + new CustomEvent('canvas.drawn', { + bubbles: false, + cancelable: true, + detail: { + state: { + shapeType: 'cuboid', + objectType: initState.objectType, + frame: this.model.data.imageID, + points, + attributes: { ...initState.attributes }, + group: initState.group?.id || null, + label: initState.label, + }, + continue: true, + duration: 0, + }, + }), + ); + } + }); + + canvasPerspectiveView.addEventListener('mousedown', this.onPerspectiveDrag); + window.document.addEventListener('mouseup', () => { + this.disablePerspectiveDragging(); + if (this.isPerspectiveBeingDragged && this.mode !== Mode.DRAG_CANVAS) { + // call this body only of drag was activated inside the canvas, but not globally + this.isPerspectiveBeingDragged = false; + } + }); + + canvasTopView.addEventListener('mousedown', this.startAction.bind(this, 'top')); + canvasSideView.addEventListener('mousedown', this.startAction.bind(this, 'side')); + canvasFrontView.addEventListener('mousedown', this.startAction.bind(this, 'front')); + + canvasTopView.addEventListener('mousemove', this.moveAction.bind(this, 'top')); + canvasSideView.addEventListener('mousemove', this.moveAction.bind(this, 'side')); + canvasFrontView.addEventListener('mousemove', this.moveAction.bind(this, 'front')); + + canvasTopView.addEventListener('mouseup', this.completeActions.bind(this)); + canvasTopView.addEventListener('mouseleave', this.completeActions.bind(this)); + canvasSideView.addEventListener('mouseup', this.completeActions.bind(this)); + canvasSideView.addEventListener('mouseleave', this.completeActions.bind(this)); + canvasFrontView.addEventListener('mouseup', this.completeActions.bind(this)); + canvasFrontView.addEventListener('mouseleave', this.completeActions.bind(this)); + + canvasPerspectiveView.addEventListener('mousemove', (event: MouseEvent): void => { + event.preventDefault(); + this.isCtrlDown = event.ctrlKey; + if (this.mode === Mode.DRAG_CANVAS) return; + const canvas = this.views.perspective.renderer.domElement; + const rect = canvas.getBoundingClientRect(); + const { mouseVector } = this.views.perspective.rayCaster as { mouseVector: THREE.Vector2 }; + mouseVector.x = ((event.clientX - (canvas.offsetLeft + rect.left)) / canvas.clientWidth) * 2 - 1; + mouseVector.y = -((event.clientY - (canvas.offsetTop + rect.top)) / canvas.clientHeight) * 2 + 1; + this.hoverNeedsUpdate = true; + }); + + canvasPerspectiveView.addEventListener('click', (e: MouseEvent): void => { + e.preventDefault(); + const selectionIsBlocked = ![Mode.GROUP, Mode.MERGE, Mode.SPLIT, Mode.IDLE].includes(this.mode) || + !this.views.perspective.rayCaster || + this.isPerspectiveBeingDragged; + + if (e.detail !== 1 || selectionIsBlocked) return; + const intersects = this.views.perspective.rayCaster.renderer + .intersectObjects(this.getAllVisibleCuboids(), false); + const intersectionClientID = +(intersects[0]?.object?.name) || null; + const objectState = Number.isInteger(intersectionClientID) ? this.model.objects + .find((state: ObjectState) => state.clientID === intersectionClientID) : null; + + if (objectState) { + this.dispatchEvent( + new CustomEvent('canvas.clicked', { + bubbles: false, + cancelable: true, + detail: { + clientID: intersectionClientID, + }, + }), + ); + } + + const handleClick = (targetList: ObjectState[]): void => { + const objectStateIdx = targetList + .findIndex((state: ObjectState) => state.clientID === intersectionClientID); + if (objectStateIdx !== -1) { + targetList.splice(objectStateIdx, 1); + } else { + targetList.push(objectState); + } + + this.drawnObjects[intersectionClientID].cuboid.setColor(this.receiveShapeColor(objectState)); + }; + + if (objectState && this.mode === Mode.GROUP) { + handleClick(this.statesToBeGrouped); + } else if (objectState && this.mode === Mode.MERGE) { + const [latest] = this.statesToBeMerged; + const drawnStates = Object.keys(this.drawnObjects).map((key: string): number => +key); + if (!latest || + (latest && + objectState.label.id === latest.label.id && + objectState.shapeType === latest.shapeType && + !this.statesToBeMerged.some((state) => drawnStates.includes(state.clientID))) + ) { + handleClick(this.statesToBeMerged); + } + } else if (objectState?.objectType === ObjectType.TRACK && this.mode === Mode.SPLIT) { + this.onSplitDone(objectState); + } else if (this.mode === Mode.IDLE) { + const intersectedClientID = intersects[0]?.object?.name || null; + if (this.model.data.activeElement.clientID !== intersectedClientID) { + this.dispatchEvent( + new CustomEvent('canvas.selected', { + bubbles: false, + cancelable: true, + detail: { + clientID: typeof intersectedClientID === 'string' ? +intersectedClientID : null, + }, + }), + ); + } + } + }); + + canvasPerspectiveView.addEventListener('dblclick', (e: MouseEvent): void => { + e.preventDefault(); + if (this.mode !== Mode.DRAW) { + const { perspective: viewType } = this.views; + viewType.rayCaster.renderer.setFromCamera(viewType.rayCaster.mouseVector, viewType.camera); + const intersects = viewType.rayCaster.renderer.intersectObjects(this.getAllVisibleCuboids(), false); + if (!intersects.length) { + this.fitCanvas(true); + } else { + const clientID = intersects[0].object.name; + this.fitObject(this.drawnObjects[clientID].cuboid, true); + + this.dispatchEvent( + new CustomEvent('canvas.doubleclicked', { + bubbles: false, + cancelable: true, + detail: { + clientID: Number(clientID), + }, + }), + ); + } + return; + } + + this.controller.drawData.enabled = false; + this.mode = Mode.IDLE; + const { x, y, z } = this.cube.perspective.position; + const { x: width, y: height, z: depth } = this.cube.perspective.scale; + const { x: rotationX, y: rotationY, z: rotationZ } = this.cube.perspective.rotation; + const points = [x, y, z, rotationX, rotationY, rotationZ, width, height, depth, 0, 0, 0, 0, 0, 0, 0]; + const initState = this.model.data.drawData.initialState; + const { redraw } = this.model.data.drawData; + if (typeof redraw === 'number') { + const state = this.model.objects + .find((object: ObjectState): boolean => object.clientID === redraw); + const { cuboid } = this.drawnObjects[redraw]; + cuboid.perspective.visible = true; + + this.dispatchEvent( + new CustomEvent('canvas.edited', { + bubbles: false, + cancelable: true, + detail: { + state, + points, + }, + }), + ); + } else { + this.dispatchEvent( + new CustomEvent('canvas.drawn', { + bubbles: false, + cancelable: true, + detail: { + state: { + shapeType: 'cuboid', + frame: this.model.data.imageID, + points, + ...(initState ? { + attributes: { ...initState.attributes }, + group: initState.group?.id || null, + label: initState.label, + shapeType: initState.shapeType, + objectType: initState.objectType, + } : {}), + }, + duration: 0, + }, + }), + ); + } + + this.views[ViewType.PERSPECTIVE].scene.children[0].remove(this.cube.perspective); + this.dispatchEvent(new CustomEvent('canvas.canceled')); + }); + + this.mode = Mode.IDLE; + + Object.keys(this.views).forEach((view: string): void => { + this.views[view as keyof Views].scene.background = new THREE.Color(0x000000); + }); + + const viewSize = consts.ORTHO_FRUSTUM_HEIGHT; + const height = window.innerHeight; + const width = window.innerWidth; + const aspectRatio = window.innerWidth / window.innerHeight; + + // setting up the camera and adding it in the scene + this.views.perspective.camera = new THREE.PerspectiveCamera(50, aspectRatio, 0.1, 500); + this.views.top.camera = new THREE.OrthographicCamera( + (-aspectRatio * viewSize) / 2 - 2, + (aspectRatio * viewSize) / 2 + 2, + viewSize / 2 + 2, + -viewSize / 2 - 2, + -50, + 50, + ); + this.views.side.camera = new THREE.OrthographicCamera( + (-aspectRatio * viewSize) / 2, + (aspectRatio * viewSize) / 2, + viewSize / 2, + -viewSize / 2, + -50, + 50, + ); + this.views.front.camera = new THREE.OrthographicCamera( + (-aspectRatio * viewSize) / 2, + (aspectRatio * viewSize) / 2, + viewSize / 2, + -viewSize / 2, + -50, + 50, + ); + + for (const cameraType of ALL_VIEWS) { + this.views[cameraType].camera.position.set(...this.cameraSettings[cameraType].position); + this.views[cameraType].camera.up.set(...this.cameraSettings[cameraType].up); + this.views[cameraType].camera.lookAt(...this.cameraSettings[cameraType].lookAt); + this.views[cameraType].camera.name = `camera${cameraType[0].toUpperCase()}${cameraType.slice(1)}`; + } + + Object.keys(this.views).forEach((view: string): void => { + const viewType = this.views[view as keyof Views]; + if (viewType.camera) { + viewType.renderer.setSize(width, height); + if (view !== ViewType.PERSPECTIVE) { + viewType.controls = new CameraControls(viewType.camera, viewType.renderer.domElement); + viewType.controls.mouseButtons.left = CameraControls.ACTION.NONE; + viewType.controls.mouseButtons.right = CameraControls.ACTION.NONE; + } else { + viewType.controls = new CameraControls(viewType.camera, viewType.renderer.domElement); + viewType.controls.mouseButtons.left = CameraControls.ACTION.NONE; + viewType.controls.mouseButtons.right = CameraControls.ACTION.NONE; + viewType.controls.touches.one = CameraControls.ACTION.NONE; + viewType.controls.touches.two = CameraControls.ACTION.NONE; + viewType.controls.touches.three = CameraControls.ACTION.NONE; + } + viewType.controls.minDistance = consts.MIN_DISTANCE; + viewType.controls.maxDistance = consts.MAX_DISTANCE; + } + }); + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + + BOTTOM_VIEWS.forEach((view: ViewType): void => { + this.views[view].renderer.domElement.addEventListener( + 'wheel', + (event: WheelEvent): void => { + event.preventDefault(); + const { camera } = this.views[view]; + + // Adaptive zoom algorithm from 2D canvas + const basicZoomCoef = 6 / 5; + const adjustCoef = 1 / 100; + const scaleFactor = basicZoomCoef ** (-event.deltaY * adjustCoef); + camera.zoom = Math.min( + Math.max( + camera.zoom * scaleFactor, + consts.SIDE_VIEWS_MIN_ZOOM, + ), consts.SIDE_VIEWS_MAX_ZOOM, + ); + + if (this.activatedElementID) { + // try to remember applied zoom level per each object + // also save serverID to be able to track case when objects are reloaded + // (e.g. annotations uploaded), in this rare case zoom memory should be reset + if (!this.sideViewsZoomMemory[this.activatedElementID]) { + this.sideViewsZoomMemory[this.activatedElementID] = { + serverID: null, + top: null, + side: null, + front: null, + }; + } + + this.sideViewsZoomMemory[this.activatedElementID].serverID = + this.drawnObjects[this.activatedElementID].data.serverID; + this.sideViewsZoomMemory[this.activatedElementID][view] = camera.zoom; + } + + // after changing zoom, need to recalculate control points size + this.updateHelperPointsSize(view); + }, + { passive: false }, + ); + }); + + model.subscribe(this); + } + + private fitObject(object: CuboidModel, animate: boolean = false): void { + const camera = this.views.perspective.camera as THREE.PerspectiveCamera; + const mesh = object.perspective as THREE.Mesh; + const geom = mesh.geometry as THREE.BufferGeometry; + if (!geom.boundingBox) { + geom.computeBoundingBox(); + } + + // 1) Collect 8 corners of the local bounding box (base geometry 0.5 cube) + const bb = geom.boundingBox!; + const cornersLocal: THREE.Vector3[] = [ + new THREE.Vector3(bb.min.x, bb.min.y, bb.min.z), + new THREE.Vector3(bb.min.x, bb.min.y, bb.max.z), + new THREE.Vector3(bb.min.x, bb.max.y, bb.min.z), + new THREE.Vector3(bb.min.x, bb.max.y, bb.max.z), + new THREE.Vector3(bb.max.x, bb.min.y, bb.min.z), + new THREE.Vector3(bb.max.x, bb.min.y, bb.max.z), + new THREE.Vector3(bb.max.x, bb.max.y, bb.min.z), + new THREE.Vector3(bb.max.x, bb.max.y, bb.max.z), + ]; + + // 2) Transform corners to world space (includes scale/rotation/translation) + const worldCorners: THREE.Vector3[] = cornersLocal.map((v) => mesh.localToWorld(v.clone())); + + // 3) Build camera basis in world + const forward = new THREE.Vector3(); + camera.getWorldDirection(forward).normalize(); + const up = camera.up.clone().normalize(); + const right = new THREE.Vector3().crossVectors(forward, up).normalize(); + up.copy(new THREE.Vector3().crossVectors(right, forward).normalize()); + + // 4) Project corners onto right/up to get world extents independent of current distance + let minU = +Infinity; + let maxU = -Infinity; + let minV = +Infinity; + let maxV = -Infinity; + for (const c of worldCorners) { + const u = c.dot(right); + const v = c.dot(up); + if (u < minU) minU = u; + if (u > maxU) maxU = u; + if (v < minV) minV = v; + if (v > maxV) maxV = v; + } + const widthWorld = maxU - minU; + const heightWorld = maxV - minV; + + // 5) Compute required distance to fit both width and height within FOV and aspect + const vFov = THREE.MathUtils.degToRad(camera.fov); + const hFov = 2 * Math.atan(Math.tan(vFov / 2) * camera.aspect); + const distByHeight = (heightWorld / 2) / Math.tan(vFov / 2); + const distByWidth = (widthWorld / 2) / Math.tan(hFov / 2); + const distance = Math.max(distByHeight, distByWidth); + + // 6) Center of geometry box in world + const centerLocal = bb.getCenter(new THREE.Vector3()); + const center = mesh.localToWorld(centerLocal); + + // 7) Move along forward and look at center + const margin = 1.5; + const targetPosition = center.clone().sub(forward.multiplyScalar(distance * margin)); + this.views.perspective.controls.setLookAt( + targetPosition.x, + targetPosition.y, + targetPosition.z, + center.x, + center.y, + center.z, + animate, + ); + } + + private fitCanvas(animation: boolean): void { + const [x, y, z] = this.cameraSettings.perspective.lookAt; + this.positionAllViews(x, y, z, animation); + this.updateCameraFrustumPlane(); + } + + private getAllVisibleCuboids(view: ViewType = ViewType.PERSPECTIVE): THREE.Mesh[] { + return Object.values(this.drawnObjects) + .map(({ cuboid }) => cuboid[view]).filter((mesh: THREE.Mesh) => mesh.visible); + } + + private updateCameraFrustumPlane(viewType?: ViewType): void { + const setCameraFrustumPlane = ( + camera: THREE.OrthographicCamera, + center: THREE.Vector3, dimensions: THREE.Vector3, + view: ViewType, + ): void => { + const [width, length] = dimensions.toArray(); + const [cx, cy, cz] = center.toArray(); + const distanceUpToCamera = Math.sqrt( + (camera.position.x - cx) ** 2 + + (camera.position.y - cy) ** 2 + + (camera.position.z - cz) ** 2, + ); + + const MARGIN = 0.1; + if (view === ViewType.FRONT) { + const objectOffset = Math.min(1 - ((width / 2) / distanceUpToCamera)); + camera.near = distanceUpToCamera * objectOffset - MARGIN; + camera.far = camera.near + width + MARGIN * 2; + } else if (view === ViewType.SIDE) { + const objectOffset = Math.min(1, 1 - ((length / 2) / distanceUpToCamera)); + camera.near = distanceUpToCamera * objectOffset - MARGIN; + camera.far = camera.near + length + MARGIN * 2; + } else if (view === ViewType.TOP) { + camera.near = 0; + camera.far = 1000; + } + }; + + const { selectedCuboid } = this; + const sceneCenter = this.sceneBBox.getCenter(new THREE.Vector3()); + const sceneDimensions = this.sceneBBox.max.clone().sub(this.sceneBBox.min); + + let center = sceneCenter; + let dimensions = sceneDimensions; + + if (selectedCuboid) { + center = selectedCuboid.perspective.position.clone(); + dimensions = selectedCuboid.perspective.scale.clone(); + } + + if (viewType !== ViewType.FRONT) { + setCameraFrustumPlane( + this.views.front.camera as THREE.OrthographicCamera, center, dimensions, ViewType.FRONT, + ); + } + + if (viewType !== ViewType.TOP) { + setCameraFrustumPlane( + this.views.top.camera as THREE.OrthographicCamera, center, dimensions, ViewType.TOP, + ); + } + + if (viewType !== ViewType.SIDE) { + setCameraFrustumPlane( + this.views.side.camera as THREE.OrthographicCamera, center, dimensions, ViewType.SIDE, + ); + } + } + + private updateSideViewsZoom(): void { + if (this.activatedElementID !== null) { + const { selectedCuboid } = this; + const { top, front, side } = this.views; + + let memoizedZoom = this.sideViewsZoomMemory[this.activatedElementID]; + const drawnState = this.drawnObjects[this.activatedElementID]; + if (memoizedZoom && + drawnState && + memoizedZoom.serverID !== null && + memoizedZoom.serverID !== drawnState.data.serverID + ) { + // oops, it seems that objects were reloaded, reset zoom memory + this.sideViewsZoomMemory = {}; + memoizedZoom = undefined; + } + + // Padding in pixels around the object comes from cvat-ui + const padding = this.model.data.configuration.focusedObjectPadding; + const baseline = consts.ORTHO_FRUSTUM_HEIGHT; + + // Helper function to compute zoom that fits width/height with corresponding padding + const computeZoomWithMargin = ( + objWidthWorld: number, + objHeightWorld: number, + canvasWidthPx: number, + canvasHeightPx: number, + aspect: number, + ): number => { + // fallback if padding is too high for the current canvas size + const marginPx = Math.max(0, Math.min(padding, Math.min(canvasWidthPx, canvasHeightPx) / 4)); + + const heightFactor = (canvasHeightPx - 2 * marginPx) / canvasHeightPx; + const widthFactor = (canvasWidthPx - 2 * marginPx) / canvasWidthPx; + const zoomFromHeight = (baseline * heightFactor) / Math.max(objHeightWorld, Number.EPSILON); + const zoomFromWidth = (aspect * baseline * widthFactor) / Math.max(objWidthWorld, Number.EPSILON); + + return Math.min(zoomFromHeight, zoomFromWidth); + }; + + const defaultOrMemoized = ( + computed: number, + memoized: number | null, + ): number => (memoized !== null ? memoized : Math.max( + consts.SIDE_VIEWS_MIN_ZOOM, + Math.min(computed, consts.SIDE_VIEWS_MAX_ZOOM), + )); + + const { renderer: { domElement: canvasTop }, camera: cameraTop } = top; + const { renderer: { domElement: canvasFront }, camera: cameraFront } = front; + const { renderer: { domElement: canvasSide }, camera: cameraSide } = side; + + // TOP view: X (width), Y (height) + // considering bbox geometry always equal 1, we only need its scale + const zoomTopDefault = computeZoomWithMargin( + selectedCuboid.top.scale.x, + selectedCuboid.top.scale.y, + canvasTop.offsetWidth, + canvasTop.offsetHeight, + top.renderer.domElement.width / Math.max(top.renderer.domElement.height, 1), + ); + + cameraTop.zoom = defaultOrMemoized(zoomTopDefault, memoizedZoom?.top ?? null); + cameraTop.updateProjectionMatrix(); + cameraTop.updateMatrix(); + this.updateHelperPointsSize(ViewType.TOP); + + // FRONT view: Y (width), Z (height) + const zoomFrontDefault = computeZoomWithMargin( + selectedCuboid.front.scale.y, + selectedCuboid.front.scale.z, + canvasFront.offsetWidth, + canvasFront.offsetHeight, + front.renderer.domElement.width / Math.max(front.renderer.domElement.height, 1), + ); + cameraFront.zoom = defaultOrMemoized(zoomFrontDefault, memoizedZoom?.front ?? null); + cameraFront.updateProjectionMatrix(); + cameraFront.updateMatrix(); + this.updateHelperPointsSize(ViewType.FRONT); + + // SIDE view: X (width), Z (height) + const zoomSideDefault = computeZoomWithMargin( + selectedCuboid.side.scale.x, + selectedCuboid.side.scale.z, + canvasSide.offsetWidth, + canvasSide.offsetHeight, + side.renderer.domElement.width / Math.max(side.renderer.domElement.height, 1), + ); + cameraSide.zoom = defaultOrMemoized(zoomSideDefault, memoizedZoom?.side ?? null); + cameraSide.updateProjectionMatrix(); + cameraSide.updateMatrix(); + this.updateHelperPointsSize(ViewType.SIDE); + } + } + + private enablePerspectiveDragging(): void { + const { controls } = this.views.perspective; + controls.mouseButtons.left = CameraControls.ACTION.ROTATE; + controls.mouseButtons.right = CameraControls.ACTION.TRUCK; + controls.touches.one = CameraControls.ACTION.TOUCH_ROTATE; + controls.touches.two = CameraControls.ACTION.TOUCH_DOLLY_TRUCK; + controls.touches.three = CameraControls.ACTION.TOUCH_TRUCK; + } + + private disablePerspectiveDragging(): void { + const { controls } = this.views.perspective; + controls.mouseButtons.left = CameraControls.ACTION.NONE; + controls.mouseButtons.right = CameraControls.ACTION.NONE; + controls.touches.one = CameraControls.ACTION.NONE; + controls.touches.two = CameraControls.ACTION.NONE; + controls.touches.three = CameraControls.ACTION.NONE; + } + + private onPerspectiveDrag = (): void => { + if (![Mode.DRAG_CANVAS, Mode.IDLE].includes(this.mode)) return; + this.isPerspectiveBeingDragged = true; + this.enablePerspectiveDragging(); + }; + + private onError = (exception: unknown, domain?: string): void => { + this.dispatchEvent( + new CustomEvent('canvas.error', { + bubbles: false, + cancelable: true, + detail: { + domain, + exception: exception instanceof Error ? + exception : new Error(`Unknown exception: "${exception}"`), + }, + }), + ); + }; + + private startAction(view: any, event: MouseEvent): void { + const { clientID } = this.model.data.activeElement; + if (event.detail !== 1 || this.mode !== Mode.IDLE || clientID === null || !(clientID in this.drawnObjects)) { + return; + } + + const canvas = this.views[view as keyof Views].renderer.domElement; + const rect = canvas.getBoundingClientRect(); + const { mouseVector } = this.views[view as keyof Views].rayCaster as { mouseVector: THREE.Vector2 }; + const diffX = event.clientX - rect.left; + const diffY = event.clientY - rect.top; + mouseVector.x = (diffX / canvas.clientWidth) * 2 - 1; + mouseVector.y = -(diffY / canvas.clientHeight) * 2 + 1; + this.action.rotation.screenInit = { x: diffX, y: diffY }; + this.action.rotation.screenMove = { x: diffX, y: diffY }; + const { data } = this.drawnObjects[+clientID]; + + if (!data.lock) { + this.action.scan = view; + this.model.mode = Mode.EDIT; + } + } + + private moveAction(view: any, event: MouseEvent): void { + event.preventDefault(); + const { clientID } = this.model.data.activeElement; + if (this.model.mode === Mode.DRAG_CANVAS || clientID === null) { + return; + } + + const canvas = this.views[view as keyof Views].renderer.domElement; + const rect = canvas.getBoundingClientRect(); + const { mouseVector } = this.views[view as keyof Views].rayCaster as { mouseVector: THREE.Vector2 }; + const diffX = event.clientX - rect.left; + const diffY = event.clientY - rect.top; + mouseVector.x = (diffX / canvas.clientWidth) * 2 - 1; + mouseVector.y = -(diffY / canvas.clientHeight) * 2 + 1; + this.action.rotation.screenMove = { x: diffX, y: diffY }; + } + + private translateReferencePlane(coordinates: any): void { + const topPlane = this.views.top.scene.getObjectByName(Planes.TOP); + if (topPlane) { + topPlane.position.x = coordinates.x; + topPlane.position.y = coordinates.y; + topPlane.position.z = coordinates.z; + } + const sidePlane = this.views.side.scene.getObjectByName(Planes.SIDE); + if (sidePlane) { + sidePlane.position.x = coordinates.x; + sidePlane.position.y = coordinates.y; + sidePlane.position.z = coordinates.z; + } + const frontPlane = this.views.front.scene.getObjectByName(Planes.FRONT); + if (frontPlane) { + frontPlane.position.x = coordinates.x; + frontPlane.position.y = coordinates.y; + frontPlane.position.z = coordinates.z; + } + } + + private resetActions(): void { + this.action = { + ...this.action, + scan: null, + detected: false, + translation: { + status: false, + helper: null, + }, + rotation: { + status: false, + helper: null, + recentMouseVector: new THREE.Vector2(0, 0), + }, + resize: { + ...this.action.resize, + status: false, + helperElement: null, + previousPosition: null, + }, + }; + this.model.mode = Mode.IDLE; + } + + private completeActions(): void { + const { scan, detected } = this.action; + if (this.model.data.activeElement.clientID === null) return; + if (!detected) { + this.resetActions(); + return; + } + + const { x, y, z } = this.selectedCuboid[scan].position; + const { x: width, y: height, z: depth } = this.selectedCuboid[scan].scale; + const { x: rotationX, y: rotationY, z: rotationZ } = this.selectedCuboid[scan].rotation; + const points = [x, y, z, rotationX, rotationY, rotationZ, width, height, depth, 0, 0, 0, 0, 0, 0, 0]; + const [state] = this.model.objects.filter( + (_state: any): boolean => _state.clientID === Number(this.selectedCuboid[scan].name), + ); + this.dispatchEvent( + new CustomEvent('canvas.edited', { + bubbles: false, + cancelable: true, + detail: { + state, + points, + }, + }), + ); + + // this.adjustPerspectiveCameras(); + this.translateReferencePlane(new THREE.Vector3(x, y, z)); + this.resetActions(); + } + + private onGroupDone(objects?: any[], duration?: number): void { + if (objects && objects.length !== 0) { + this.dispatchEvent( + new CustomEvent('canvas.grouped', { + bubbles: false, + cancelable: true, + detail: { + duration, + states: objects, + }, + }), + ); + } else { + this.dispatchEvent( + new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }), + ); + } + + this.mode = Mode.IDLE; + } + + private onMergeDone(objects: any[] | null, duration?: number): void { + if (objects) { + const event: CustomEvent = new CustomEvent('canvas.merged', { + bubbles: false, + cancelable: true, + detail: { + duration, + states: objects, + }, + }); + + this.dispatchEvent(event); + } else { + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.dispatchEvent(event); + } + + this.mode = Mode.IDLE; + } + + private onSplitDone(object: ObjectState, duration?: number): void { + if (object) { + const event: CustomEvent = new CustomEvent('canvas.splitted', { + bubbles: false, + cancelable: true, + detail: { + duration, + state: object, + frame: object.frame, + }, + }); + + this.dispatchEvent(event); + } else { + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.dispatchEvent(event); + } + + this.controller.split({ enabled: false }); + this.mode = Mode.IDLE; + } + + private receiveShapeColor(state: ObjectState | DrawnObjectData): string { + const includedInto = (states: ObjectState[]): boolean => states + .some((_state: ObjectState): boolean => _state.clientID === state.clientID); + const { colorBy } = this.model.data.configuration; + + if (this.mode === Mode.GROUP && includedInto(this.statesToBeGrouped)) { + return consts.GROUPING_COLOR; + } + + if (this.mode === Mode.MERGE && includedInto(this.statesToBeMerged)) { + return consts.MERGING_COLOR; + } + + if (this.mode === Mode.SPLIT && this.stateToBeSplitted?.clientID === state.clientID) { + return consts.SPLITTING_COLOR; + } + + if (state instanceof ObjectState) { + if (colorBy === 'Label') { + return state.label.color; + } + + if (colorBy === 'Group') { + return state.group?.color || consts.DEFAULT_GROUP_COLOR; + } + + return state.color; + } + + if (colorBy === 'Label') { + return state.labelColor; + } + + if (colorBy === 'Group') { + return state.groupColor; + } + + return state.color; + } + + private addCuboid(state: ObjectState): CuboidModel { + const { + shapeOpacity, outlinedBorders, orientationVisibility, + } = this.model.data.configuration; + const clientID = String(state.clientID); + const cuboid = new CuboidModel(state.occluded ? 'dashed' : 'line', outlinedBorders || '#ffffff'); + const color = this.receiveShapeColor(state); + + cuboid.setName(clientID); + cuboid.setColor(color); + cuboid.setOpacity(shapeOpacity); + cuboid.setPosition(state.points[0], state.points[1], state.points[2]); + cuboid.setScale(state.points[6], state.points[7], state.points[8]); + cuboid.setRotation(state.points[3], state.points[4], state.points[5]); + cuboid.attachCameraReference(); + cuboid.setOrientationVisibility(orientationVisibility); + + cuboid[ViewType.PERSPECTIVE].visible = !(state.hidden || state.outside); + for (const view of BOTTOM_VIEWS) { + cuboid[view].visible = false; + } + + return cuboid; + } + + private deactivateObject(): void { + const { shapeOpacity } = this.model.data.configuration; + if (this.activatedElementID !== null) { + const { cuboid } = this.drawnObjects[this.activatedElementID]; + cuboid.setOpacity(shapeOpacity); + for (const view of BOTTOM_VIEWS) { + cuboid[view].visible = false; + removeCuboidEdges(cuboid[view]); + removeResizeHelper(cuboid[view]); + removeRotationHelper(cuboid[view]); + } + this.activatedElementID = null; + } + } + + private activateObject(): void { + const { selectedShapeOpacity } = this.model.data.configuration; + const { clientID } = this.model.data.activeElement; + if (clientID !== null && this.drawnObjects[+clientID]?.cuboid?.perspective?.visible) { + const { cuboid, data } = this.drawnObjects[+clientID]; + cuboid.setOpacity(selectedShapeOpacity); + for (const view of BOTTOM_VIEWS) { + cuboid[view].visible = true; + createCuboidEdges(cuboid[view]); + + if (!data.lock) { + createResizeHelper(cuboid, view); + createRotationHelper(cuboid, view); + BOTTOM_VIEWS + .forEach((type) => this.updateHelperPointsSize(type)); + } + } + + this.activatedElementID = +clientID; + this.rotatePlane(null, null); + this.detachCamera(); + this.updateCameraFrustumPlane(); + } + } + + private createObjects(states: ObjectState[]): void { + states.forEach((state: ObjectState) => { + const cuboid = this.addCuboid(state); + this.addSceneChildren(cuboid); + this.drawnObjects[state.clientID] = { + cuboid, + data: drawnDataFromState(state), + }; + }); + } + + private updateObjects(states: ObjectState[]): void { + const { outlinedBorders } = this.model.data.configuration; + states.forEach((state: ObjectState) => { + const { + clientID, points, color, label, group, occluded, outside, hidden, + } = state; + const { cuboid, data } = this.drawnObjects[clientID]; + + if (points.length !== data.points.length || + points.some((point: number, idx: number) => point !== data.points[idx])) { + cuboid.setPosition(state.points[0], state.points[1], state.points[2]); + cuboid.setScale(state.points[6], state.points[7], state.points[8]); + cuboid.setRotation(state.points[3], state.points[4], state.points[5]); + } + + if ( + color !== data.color || + label.id !== data.labelID || + group.id !== data.groupID || + group.color !== data.groupColor + ) { + const newColor = this.receiveShapeColor(state); + cuboid.setColor(newColor); + if (outlinedBorders) { + cuboid.setOutlineColor(outlinedBorders); + } + } + + if (outside !== data.outside || hidden !== data.hidden) { + cuboid.perspective.visible = !(outside || hidden); + cuboid.top.visible = !(outside || hidden); + cuboid.side.visible = !(outside || hidden); + cuboid.front.visible = !(outside || hidden); + } + + if (occluded !== data.occluded) { + this.deleteObjects([clientID]); + this.createObjects([state]); + return; + } + + this.drawnObjects[clientID].data = drawnDataFromState(state); + }); + } + + private deleteObjects(clientIDs: number[]): void { + clientIDs.forEach((clientID: number): void => { + const { cuboid } = this.drawnObjects[clientID]; + this.removeSceneChildren(cuboid, clientID); + }); + } + + private setupObjectsIncremental(states: ObjectState[]): void { + const created = states.filter((state: ObjectState): boolean => !(state.clientID in this.drawnObjects)); + const updated = states.filter((state: ObjectState): boolean => ( + state.clientID in this.drawnObjects && this.drawnObjects[state.clientID].data.updated !== state.updated + )); + const deleted = Object.keys(this.drawnObjects).map((key: string): number => +key) + .filter((clientID: number): boolean => ( + states.findIndex((state: ObjectState) => state.clientID === clientID) === -1 + )); + + this.deactivateObject(); + this.createObjects(created); + this.updateObjects(updated); + this.deleteObjects(deleted); + this.activateObject(); + } + + private addSceneChildren(shapeObject: CuboidModel): void { + this.views.perspective.scene.children[0].add(shapeObject.perspective); + this.views.top.scene.children[0].add(shapeObject.top); + this.views.side.scene.children[0].add(shapeObject.side); + this.views.front.scene.children[0].add(shapeObject.front); + } + + private removeSceneChildren(shapeObject: CuboidModel, clientID: number): void { + this.views.perspective.scene.children[0].remove(shapeObject.perspective); + this.views.top.scene.children[0].remove(shapeObject.top); + this.views.side.scene.children[0].remove(shapeObject.side); + this.views.front.scene.children[0].remove(shapeObject.front); + + shapeObject.dispose(); + delete this.drawnObjects[clientID]; + } + + private dispatchEvent(event: CustomEvent): void { + this.views.perspective.renderer.domElement.dispatchEvent(event); + } + + public notify(model: Canvas3dModel & Master, reason: UpdateReasons): void { + const resetColor = (list: ObjectState[]): void => { + list.forEach((state: ObjectState) => { + const { clientID } = state; + const { cuboid } = this.drawnObjects[clientID] || {}; + if (cuboid) { + cuboid.setColor(this.receiveShapeColor(state)); + } + }); + }; + + if (reason === UpdateReasons.IMAGE_CHANGED) { + this.statesToBeGrouped = []; + this.clearScene(); + + const onPCDLoadSuccess = (points: any): void => { + try { + this.onSceneImageLoaded(points); + model.updateCanvasObjects(); + } finally { + model.unlockFrameUpdating(); + this.dispatchEvent(new CustomEvent('canvas.setup')); + } + }; + + try { + if (model.data.image) { + const loader = new PCDLoader(); + this.views.perspective.renderer.dispose(); + if (this.controller.imageIsDeleted) { + try { + this.render(); + const [container] = window.document.getElementsByClassName('cvat-canvas-container'); + const overlay = window.document.createElement('canvas'); + overlay.classList.add('cvat_3d_canvas_deleted_overlay'); + overlay.style.width = '100%'; + overlay.style.height = '100%'; + overlay.style.position = 'absolute'; + overlay.style.top = '0px'; + overlay.style.left = '0px'; + container.appendChild(overlay); + const { clientWidth: width, clientHeight: height } = overlay; + overlay.width = width; + overlay.height = height; + const canvasContext = overlay.getContext('2d'); + const fontSize = width / 10; + canvasContext.font = `bold ${fontSize}px serif`; + canvasContext.textAlign = 'center'; + canvasContext.lineWidth = fontSize / 20; + canvasContext.strokeStyle = 'white'; + canvasContext.strokeText('IMAGE REMOVED', width / 2, height / 2); + canvasContext.fillStyle = 'black'; + canvasContext.fillText('IMAGE REMOVED', width / 2, height / 2); + this.dispatchEvent(new CustomEvent('canvas.setup')); + } finally { + model.unlockFrameUpdating(); + } + } else { + model.data.image.imageData.arrayBuffer().then((data) => { + const defaultImpl = console.error; + // loader.parse will throw some errors to console when nan values are in PCD file + // there is not way to prevent it, because three.js opinion is that nan values + // in input data is incorrect + let cloud = null; + try { + console.error = () => { }; + cloud = loader.parse(data) as THREE.Points; + } finally { + console.error = defaultImpl; + } + + let { + position, color, normal, intensity, label, + } = cloud.geometry.attributes; + cloud.material.vertexColors = true; + + ({ + color, position, normal, intensity, label, + } = position.array.reduce((acc, _, i, array) => { + if ( + i % 3 === 0 && + Number.isFinite(array[i]) && + Number.isFinite(array[i + 1]) && + Number.isFinite(array[i + 2]) + ) { + acc.position.push(array[i], array[i + 1], array[i + 2]); + + if ( + color && + Number.isFinite(color.array[i]) && + Number.isFinite(color.array[i + 1]) && + Number.isFinite(color.array[i + 2]) + ) { + acc.color.push(color.array[i], color.array[i + 1], color.array[i + 2]); + } else { + acc.color.push(255, 255, 255); + } + + if ( + normal && + Number.isFinite(normal.array[i]) && + Number.isFinite(normal.array[i + 1]) && + Number.isFinite(normal.array[i + 2]) + ) { + acc.normal.push(normal.array[i], normal.array[i + 1], normal.array[i + 2]); + } + + if (intensity) { + acc.intensity.push(intensity.array[i / 3]); + } + + if (label) { + acc.label.push(label.array[i / 3]); + } + } + + return acc; + }, { + position: [], color: [], normal: [], intensity: [], label: [], + })); + + if (position.length) { + cloud.geometry.setAttribute('position', new THREE.Float32BufferAttribute(position, 3)); + } + + if (color.length) { + cloud.geometry.setAttribute('color', new THREE.Float32BufferAttribute(color, 3)); + } + + if (normal.length) { + cloud.geometry.setAttribute('normal', new THREE.Float32BufferAttribute(normal, 3)); + } + + if (intensity.length) { + cloud.geometry.setAttribute('intensity', new THREE.Float32BufferAttribute(intensity, 1)); + } + + if (label.length) { + cloud.geometry.setAttribute('label', new THREE.Float32BufferAttribute(label, 1)); + } + + cloud.geometry.computeBoundingSphere(); + onPCDLoadSuccess(cloud); + }); + const [overlay] = window.document.getElementsByClassName('cvat_3d_canvas_deleted_overlay'); + if (overlay) { + overlay.remove(); + } + } + } + } catch (error: any) { + model.unlockFrameUpdating(); + throw error; + } + } else if (reason === UpdateReasons.CONFIG_UPDATED) { + const config = { ...model.data.configuration }; + for (const key of Object.keys(this.drawnObjects)) { + const clientID = +key; + const { cuboid, data } = this.drawnObjects[clientID]; + const newColor = this.receiveShapeColor(data); + cuboid.setColor(newColor); + cuboid.setOpacity( + ((clientID === this.activatedElementID) ? config.selectedShapeOpacity : config.shapeOpacity), + ); + + if (config.outlinedBorders) { + cuboid.setOutlineColor(config.outlinedBorders || consts.DEFAULT_OUTLINE_COLOR); + } + + cuboid.setOrientationVisibility(config.orientationVisibility); + } + + BOTTOM_VIEWS.forEach((view) => { + // in case of changing control points size, need to update all helper points size + this.updateHelperPointsSize(view); + }); + } else if (reason === UpdateReasons.SHAPE_ACTIVATED) { + this.deactivateObject(); + this.activateObject(); + this.updateSideViewsZoom(); + } else if (reason === UpdateReasons.DRAW) { + const data: DrawData = this.controller.drawData; + if (Number.isInteger(data.redraw)) { + if (this.drawnObjects[data.redraw]?.cuboid?.perspective?.visible) { + const { cuboid } = this.drawnObjects[data.redraw]; + this.cube.perspective = cuboid.perspective.clone() as THREE.Mesh; + cuboid.perspective.visible = false; + } else { + // an object must be drawn and visible to be redrawn + model.cancel(); + return; + } + } else if (data.initialState) { + if (!data.initialState.outside && !data.initialState.hidden) { + this.cube = this.addCuboid(data.initialState); + } else { + // an object must visible to paste it + model.cancel(); + return; + } + } else { + this.cube = new CuboidModel('line', '#ffffff'); + this.cube.setOrientationVisibility(this.model.data.configuration.orientationVisibility); + } + + this.cube.setName('drawTemplate'); + this.deactivateObject(); + this.views[ViewType.PERSPECTIVE].scene.children[0].add(this.cube.perspective); + } else if (reason === UpdateReasons.OBJECTS_UPDATED) { + this.setupObjectsIncremental(model.objects); + } else if (reason === UpdateReasons.DRAG_CANVAS) { + this.isPerspectiveBeingDragged = true; + this.dispatchEvent( + new CustomEvent('canvas.dragstart', { + bubbles: false, + cancelable: true, + }), + ); + model.data.activeElement.clientID = null; + this.deactivateObject(); + } else if (reason === UpdateReasons.CANCEL) { + if (this.mode === Mode.DRAG_CANVAS) { + this.isPerspectiveBeingDragged = false; + this.dispatchEvent( + new CustomEvent('canvas.dragstop', { + bubbles: false, + cancelable: true, + }), + ); + } + + if (this.mode === Mode.DRAW) { + this.controller.drawData.enabled = false; + const { redraw } = this.controller.drawData; + if (Number.isInteger(redraw)) { + this.drawnObjects[redraw].cuboid.perspective.visible = true; + this.controller.drawData.redraw = undefined; + } + const scene = this.views[ViewType.PERSPECTIVE].scene.children[0]; + const template = scene.getObjectByName('drawTemplate'); + if (template) { + scene.remove(template); + } + } + + if (this.mode === Mode.MERGE) { + const { statesToBeMerged } = this; + this.statesToBeMerged = []; + resetColor(statesToBeMerged); + this.model.merge({ enabled: false }); + } + + if (this.mode === Mode.GROUP) { + const { statesToBeGrouped } = this; + this.statesToBeGrouped = []; + resetColor(statesToBeGrouped); + this.model.group({ enabled: false }); + } + + if (this.mode === Mode.SPLIT) { + if (this.stateToBeSplitted) { + const state = this.stateToBeSplitted; + this.stateToBeSplitted = null; + this.drawnObjects[state.clientID].cuboid.setColor(this.receiveShapeColor(state)); + } + this.model.split({ enabled: false }); + } + + this.mode = Mode.IDLE; + this.dispatchEvent(new CustomEvent('canvas.canceled')); + } else if (reason === UpdateReasons.FITTED_CANVAS) { + this.fitCanvas(false); + this.dispatchEvent(new CustomEvent('canvas.fit')); + } else if (reason === UpdateReasons.GROUP) { + if (!model.groupData.enabled && this.statesToBeGrouped.length) { + this.onGroupDone(this.statesToBeGrouped); + resetColor(this.statesToBeGrouped); + } else if (model.groupData.enabled) { + this.deactivateObject(); + this.statesToBeGrouped = []; + model.data.activeElement.clientID = null; + } + } else if (reason === UpdateReasons.SPLIT) { + this.deactivateObject(); + this.stateToBeSplitted = null; + model.data.activeElement.clientID = null; + } else if (reason === UpdateReasons.MERGE) { + if (!model.mergeData.enabled && this.statesToBeMerged.length) { + this.onMergeDone(this.statesToBeMerged); + resetColor(this.statesToBeMerged); + } else if (model.mergeData.enabled) { + this.deactivateObject(); + this.statesToBeMerged = []; + model.data.activeElement.clientID = null; + } + } else if (reason === UpdateReasons.DATA_FAILED) { + this.onError(model.exception, 'data fetching'); + } + } + + private clearScene(): void { + Object.entries(this.drawnObjects).forEach(([clientID, { cuboid }]) => { + this.removeSceneChildren(cuboid, +clientID); + }); + + this.drawnObjects = {}; + this.activatedElementID = null; + + Object.keys(this.views).forEach((view: string): void => { + const viewData = this.views[view as keyof Views]; + disposeScene(viewData.scene); + }); + } + + private updateRotationHelperPos(): void { + const cuboid = this.selectedCuboid; + if (!cuboid) { + return; + } + + BOTTOM_VIEWS.forEach((view: ViewType): void => { + const rotationHelper = cuboid[view].parent.getObjectByName(consts.ROTATION_HELPER_NAME); + if (rotationHelper) { + const position = cuboid.getRotationHelperPosition(view); + rotationHelper.position.copy(position); + } + }); + } + + private updateResizeHelperPos(): void { + const cuboid = this.selectedCuboid; + if (cuboid === null) { + return; + } + + BOTTOM_VIEWS.forEach((view: ViewType): void => { + const positions = cuboid.getResizeHelperPositions(); + const pointsToBeUpdated = cuboid[view].parent.children + .filter((child: THREE.Object3D) => child.name.startsWith(consts.RESIZE_HELPER_NAME)) + .sort((child1: THREE.Object3D, child2: THREE.Object3D) => { + const order1 = +child1.name.split('_')[1]; + const order2 = +child2.name.split('_')[1]; + return order1 - order2; + }); + + for (let i = 0; i < positions.length; i++) { + const position = positions[i]; + pointsToBeUpdated[i].position.copy(position); + } + }); + } + + private updateHelperPointsSize(viewType: ViewType): void { + const helperWorldPointsSize = this.computeHelperSphereWorldSize(viewType); + if (BOTTOM_VIEWS.includes(viewType)) { + const { scene } = this.views[viewType]; + const isSceneRendered = scene.children.length > 0; + if (!isSceneRendered) { + return; + } + + // as physical size is 1, we just scale to target size + const rotationObject = scene.children[0]?.getObjectByName(consts.ROTATION_HELPER_NAME); + if (rotationObject) { + rotationObject.scale.set(helperWorldPointsSize, helperWorldPointsSize, helperWorldPointsSize); + } + + scene.children[0].children + .filter((child: THREE.Object3D) => child.name.startsWith(consts.RESIZE_HELPER_NAME)) + .forEach((child: THREE.Object3D) => { + child.scale.set(helperWorldPointsSize, helperWorldPointsSize, helperWorldPointsSize); + }); + } + } + + /** + * Compute helper sphere size in world units for orthographic cameras. + * + * Goal: given desired helper size in pixels (controlsPointSize) and the current + * canvas size in pixels, convert that into world-space radius/diameter so that + * helpers (rotation/resize spheres) appear with consistent on-screen size. + * + * This method only targets orthographic views (BOTTOM_VIEWS). For perspective + * views, apparent size depends on distance to the camera and FOV. + */ + private computeHelperSphereWorldSize(viewType: ViewType): number { + const { controlPointsSize: controlPointSizePx } = this.model.data.configuration; + const view = this.views[viewType]; + const camera = view.camera as THREE.OrthographicCamera; + const canvas = view.renderer.domElement; + if (!camera || !canvas) { + return controlPointSizePx; + } + + // 1) Baseline vertical frustum height in world units divided by zoom + const baselineWorldHeight = consts.ORTHO_FRUSTUM_HEIGHT; // ORTHO_FRUSTUM_HEIGHT + const visibleWorldHeight = baselineWorldHeight / Math.max(camera.zoom, Number.EPSILON); + + // 2) World units per one pixel along vertical axis + const canvasHeightPx = canvas.clientHeight || canvas.parentElement?.clientHeight || 1; + const worldPerPixelY = visibleWorldHeight / canvasHeightPx; + + // 3) Convert desired pixel size to world units + // 1.5 is a scaling factor for better visibility + const helperWorldSize = controlPointSizePx * worldPerPixelY * 1.5; + + return helperWorldSize; + } + + private onSceneImageLoaded(points: any): void { + const getCameraSettingsToFitScene = ( + camera: THREE.PerspectiveCamera, + boundingBox: THREE.Box3, + ): { + position: [number, number, number], + lookAt: [number, number, number], + } => { + const width = boundingBox.max.x - boundingBox.min.x; + const height = boundingBox.max.y - boundingBox.min.y; + const depth = boundingBox.max.z - boundingBox.min.z; + + // Center of the scene + const centerX = boundingBox.min.x + width / 2; + const centerY = boundingBox.min.y + height / 2; + const centerZ = boundingBox.min.z + depth / 2; + + // Calculate distance to fit the scene: distance = (size / 2) / tan(fov / 2) + const maxDim = Math.max(width, height, depth); + const fovRadians = camera.fov * (Math.PI / 180); + const cameraDistance = (maxDim / 2) / Math.tan(fovRadians / 2); + + // Position camera above and slightly offset from center + const offset = 5; + + return { + position: [ + centerX + offset, + centerY + offset, + centerZ + cameraDistance + offset, + ], + lookAt: [ + centerX, + centerY, + centerZ, + ], + }; + }; + + // eslint-disable-next-line no-param-reassign + points.material.size = 0.05; + points.material.color.set(new THREE.Color(0xffffff)); + + const { controls } = this.views.perspective; + controls.mouseButtons.wheel = CameraControls.ACTION.DOLLY; + + const material = points.material.clone(); + if (!this.views.perspective.camera) return; + + // updating correct camera settings + points.geometry.computeBoundingBox(); + const { position, lookAt } = getCameraSettingsToFitScene( + this.views.perspective.camera as THREE.PerspectiveCamera, points.geometry.boundingBox, + ); + + this.cameraSettings.perspective.position = position; + this.sceneBBox = new THREE.Box3().setFromObject(points); + const perspectiveCloud = points.clone(); + this.views.perspective.scene.add(perspectiveCloud); + + const origin = new THREE.Vector3(0, 0, 0); + const isOriginInScene = this.sceneBBox.containsPoint(origin); + const axesHelper = new THREE.AxesHelper(5); + if (isOriginInScene) { + this.cameraSettings.perspective.lookAt = [0, 0, 0]; + axesHelper.position.set(0, 0, 0); + } else { + this.cameraSettings.perspective.lookAt = lookAt; + axesHelper.position.set(lookAt[0], lookAt[1], lookAt[2]); + } + + this.views.perspective.scene.add(axesHelper); + // Setup TopView + const canvasTopView = this.views.top.renderer.domElement; + const topScenePlane = new THREE.Mesh( + new THREE.PlaneGeometry( + canvasTopView.offsetHeight, + canvasTopView.offsetWidth, + canvasTopView.offsetHeight, + canvasTopView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0xffffff, + visible: false, + }), + ); + topScenePlane.position.set(0, 0, 0); + topScenePlane.name = Planes.TOP; + (topScenePlane.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + (topScenePlane as any).verticesNeedUpdate = true; + // eslint-disable-next-line no-param-reassign + points.material = material; + material.size = 0.5; + const topCloud = points.clone(); + this.views.top.scene.add(topCloud); + this.views.top.scene.add(topScenePlane); + // Setup Side View + const canvasSideView = this.views.side.renderer.domElement; + const sideScenePlane = new THREE.Mesh( + new THREE.PlaneGeometry( + canvasSideView.offsetHeight, + canvasSideView.offsetWidth, + canvasSideView.offsetHeight, + canvasSideView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0x00ff00, + visible: false, + opacity: 0.5, + }), + ); + sideScenePlane.position.set(0, 0, 0); + sideScenePlane.rotation.set(0, 0, 0); + sideScenePlane.name = Planes.SIDE; + (sideScenePlane.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + (sideScenePlane as any).verticesNeedUpdate = true; + const sideCloud = points.clone(); + this.views.side.scene.add(sideCloud); + this.views.side.scene.add(sideScenePlane); + // Setup front View + const canvasFrontView = this.views.front.renderer.domElement; + const frontScenePlane = new THREE.Mesh( + new THREE.PlaneGeometry( + canvasFrontView.offsetHeight, + canvasFrontView.offsetWidth, + canvasFrontView.offsetHeight, + canvasFrontView.offsetWidth, + ), + new THREE.MeshBasicMaterial({ + color: 0xffffff, + visible: false, + }), + ); + frontScenePlane.position.set(0, 0, 0); + frontScenePlane.rotation.set(0, 0, 0); + frontScenePlane.name = Planes.FRONT; + (frontScenePlane.material as THREE.MeshBasicMaterial).side = THREE.DoubleSide; + (frontScenePlane as any).verticesNeedUpdate = true; + const frontCloud = points.clone(); + this.views.front.scene.add(frontCloud); + this.views.front.scene.add(frontScenePlane); + + if (this.mode === Mode.DRAW) { + this.views[ViewType.PERSPECTIVE].scene.children[0].add(this.cube.perspective); + } + } + + private positionAllViews(x: number, y: number, z: number, animation: boolean): void { + if ( + this.views.perspective.controls && + this.views.top.controls && + this.views.side.controls && + this.views.front.controls + ) { + this.views.perspective.controls.setLookAt( + this.cameraSettings.perspective.position[0], + this.cameraSettings.perspective.position[1], + this.cameraSettings.perspective.position[2], + x, y, z, animation, + ); + + for (const cameraType of BOTTOM_VIEWS) { + const { camera } = this.views[cameraType]; + camera.position.set( + x + this.cameraSettings[cameraType].position[0], + y + this.cameraSettings[cameraType].position[1], + z + this.cameraSettings[cameraType].position[2], + ); + camera.lookAt(x, y, z); + camera.zoom = consts.FOV_DEFAULT; + } + } + } + + private resizeRendererToDisplaySize(viewType: ViewType, view: RenderView): void { + const { camera, renderer } = view; + const canvas = renderer.domElement; + if (!canvas.parentElement) { + return; + } + const width = canvas.parentElement.clientWidth; + const height = canvas.parentElement.clientHeight; + const shouldResize = canvas.clientWidth !== width || canvas.clientHeight !== height; + if (shouldResize && camera && view.camera) { + if (camera instanceof THREE.PerspectiveCamera) { + camera.aspect = width / height; + } else { + const viewSize = consts.ORTHO_FRUSTUM_HEIGHT; + const aspectRatio = width / height; + camera.left = (-aspectRatio * viewSize) / 2; + camera.right = (aspectRatio * viewSize) / 2; + camera.top = viewSize / 2; + camera.bottom = -viewSize / 2; + this.updateHelperPointsSize(viewType); + } + + view.renderer.setSize(width, height); + view.camera.updateProjectionMatrix(); + } + } + + private renderRayCaster = (viewType: RenderView): void => { + viewType.rayCaster.renderer.setFromCamera(viewType.rayCaster.mouseVector, viewType.camera); + if (this.mode === Mode.DRAW) { + const [intersection] = viewType.rayCaster.renderer + .intersectObjects(this.views.perspective.scene.children, false); + if (intersection) { + const object = this.views.perspective.scene.getObjectByName('drawTemplate'); + const { x, y, z } = intersection.point; + object.position.set(x, y, z); + } + } else { + const { renderer } = this.views.perspective.rayCaster; + const intersects = renderer.intersectObjects(this.getAllVisibleCuboids(), false); + if (intersects.length !== 0 && !this.isPerspectiveBeingDragged) { + const clientID = intersects[0].object.name; + const castedClientID = +clientID; + + if (this.mode === Mode.SPLIT) { + const objectState = Number.isInteger(castedClientID) ? this.model.objects + .find((state: ObjectState) => state.clientID === castedClientID) : null; + this.stateToBeSplitted = objectState; + this.drawnObjects[castedClientID].cuboid.setColor(this.receiveShapeColor(objectState)); + } else if (this.mode === Mode.IDLE && !this.isCtrlDown) { + const intersectedClientID = intersects[0]?.object?.name || null; + const activeClientID = this.model.data.activeElement.clientID; + if (activeClientID !== null && !this.hoverNeedsUpdate) { + return; + } + this.hoverNeedsUpdate = false; + + if (activeClientID !== intersectedClientID) { + const object = intersectedClientID ? + this.views.perspective.scene.getObjectByName(intersectedClientID) : + null; + if (intersectedClientID && object === undefined) return; + + const numericClientID = + typeof intersectedClientID === 'string' ? +intersectedClientID : null; + + this.dispatchEvent( + new CustomEvent('canvas.selected', { + bubbles: false, + cancelable: true, + detail: { + clientID: numericClientID, + }, + }), + ); + } + } + } else if (this.mode === Mode.SPLIT && this.stateToBeSplitted) { + const state = this.stateToBeSplitted; + this.stateToBeSplitted = null; + this.drawnObjects[state.clientID].cuboid.setColor(this.receiveShapeColor(state)); + } + } + }; + + public render(): void { + Object.keys(this.views).forEach((view: string): void => { + const viewType = this.views[view as keyof Views]; + if (!(viewType.controls && viewType.camera && viewType.rayCaster)) return; + + const { clientID } = this.model.data.activeElement; + this.resizeRendererToDisplaySize(view as ViewType, viewType); + if (viewType.controls.enabled) { + viewType.controls.update(this.clock.getDelta()); + } else { + viewType.camera.updateProjectionMatrix(); + } + viewType.renderer.render(viewType.scene, viewType.camera); + if (view === ViewType.PERSPECTIVE && viewType.scene.children.length !== 0) { + this.renderRayCaster(viewType); + } + if (clientID !== null && view !== ViewType.PERSPECTIVE) { + viewType.rayCaster.renderer.setFromCamera(viewType.rayCaster.mouseVector, viewType.camera); + // First Scan + if (this.action.scan === view) { + if (!(this.action.translation.status || this.action.resize.status || this.action.rotation.status)) { + this.initiateAction(view as ViewType, viewType); + } + // Action Operations + if (this.action.detected) { + if (this.action.translation.status) { + this.renderTranslateAction(view as ViewType, viewType); + } else if (this.action.resize.status) { + this.renderResizeAction(view as ViewType, viewType); + } else { + this.renderRotateAction(view as ViewType, viewType); + } + } else { + this.resetActions(); + } + } + } + }); + } + + private adjustPerspectiveCameras(): void { + const { camera: cameraTop } = this.views.top; + const { camera: cameraSide } = this.views.side; + const { camera: cameraFront } = this.views.front; + const { selectedCuboid } = this; + + const coordinatesTop = this.selectedCuboid.getReferenceCoordinates(ViewType.TOP); + const sphericalTop = new THREE.Spherical(); + sphericalTop.setFromVector3(coordinatesTop); + cameraTop.position.setFromSpherical(sphericalTop); + cameraTop.updateProjectionMatrix(); + + const coordinatesSide = selectedCuboid.getReferenceCoordinates(ViewType.SIDE); + const sphericalSide = new THREE.Spherical(); + sphericalSide.setFromVector3(coordinatesSide); + cameraSide.position.setFromSpherical(sphericalSide); + cameraSide.updateProjectionMatrix(); + + const coordinatesFront = selectedCuboid.getReferenceCoordinates(ViewType.FRONT); + const sphericalFront = new THREE.Spherical(); + sphericalFront.setFromVector3(coordinatesFront); + cameraFront.position.setFromSpherical(sphericalFront); + cameraFront.updateProjectionMatrix(); + } + + private renderTranslateAction(view: ViewType, viewType: any): void { + if ( + this.action.translation.helper.x === this.views[view].rayCaster.mouseVector.x && + this.action.translation.helper.y === this.views[view].rayCaster.mouseVector.y + ) { + return; + } + const intersects = viewType.rayCaster.renderer.intersectObjects( + [viewType.scene.getObjectByName(`${view}Plane`)], + true, + ); + + if (intersects.length !== 0 && intersects[0].point) { + const coordinates = intersects[0].point; + this.action.translation.coordinates = coordinates; + this.moveObject(coordinates); + this.detachCamera(view); + this.updateCameraFrustumPlane(view); + } + } + + private moveObject(coordinates: THREE.Vector3): void { + const { + perspective, top, side, front, + } = this.selectedCuboid; + let localCoordinates = coordinates; + if (this.action.translation.status) { + localCoordinates = coordinates + .clone() + .sub(this.action.translation.offset) + .applyMatrix4(this.action.translation.inverseMatrix); + } + perspective.position.copy(localCoordinates.clone()); + top.position.copy(localCoordinates.clone()); + side.position.copy(localCoordinates.clone()); + front.position.copy(localCoordinates.clone()); + + this.updateResizeHelperPos(); + this.updateRotationHelperPos(); + } + + private renderResizeAction(view: ViewType, viewType: any): void { + const cuboid = this.selectedCuboid; + const intersects = viewType.rayCaster.renderer + .intersectObjects([viewType.scene.getObjectByName(`${view}Plane`)], true); + + if ( + cuboid === null || intersects.length === 0) { + return; + } + + if (!this.action.resize.previousPosition) { + this.action.resize.previousPosition = intersects[0].object.worldToLocal(intersects[0].point.clone()); + return; + } + + if (Math.abs(this.action.resize.previousPosition.x - intersects[0].point.x) < Number.EPSILON || + Math.abs(this.action.resize.previousPosition.y - intersects[0].point.y) < Number.EPSILON) { + return; + } + + // first let's find the point that is used to resize + // and the opposite point in another corner + const currentPointNumber = +this.action.resize.helperElement.name.split('_')[1]; + const cuboidNodes = makeCornerPointsMatrix(0.5, 0.5, 0.5); + const crosslyingPointInternalCoordinates = (new THREE.Vector3()) + .fromArray(cuboidNodes[+currentPointNumber]).multiply(new THREE.Vector3(-1, -1, -1)); + const crosslyingHelperIndex = cuboidNodes + .findIndex(([x, y, z]): boolean => ( + Math.sign(crosslyingPointInternalCoordinates.x) === Math.sign(x) && + Math.sign(crosslyingPointInternalCoordinates.y) === Math.sign(y) && + Math.sign(crosslyingPointInternalCoordinates.z) === Math.sign(z) + )); + const crosslyingHelper = cuboid.perspective.getObjectByName(`cuboidNodeHelper_${crosslyingHelperIndex}`); + const crosslyingPointCoordinates = crosslyingHelper.getWorldPosition(new THREE.Vector3()); + + // after we've found two points + // we can get all the information from them (scale and center) + // but first we need to move the current point + // we will move point in "internal" cuboid coordinates + // and then using localToWorld we will receive world coordinates + const currentPointCoordOnPlane = intersects[0].object.worldToLocal(intersects[0].point.clone()); + const scale = cuboid.perspective.scale.clone(); + const currentPointInternalCoordinates = new THREE.Vector3(); + if (view === ViewType.FRONT) { + const diffX = currentPointCoordOnPlane.x - this.action.resize.previousPosition.x; + const diffY = currentPointCoordOnPlane.y - this.action.resize.previousPosition.y; + currentPointInternalCoordinates + .fromArray(cuboidNodes[currentPointNumber]).add(new THREE.Vector3(0, diffY, -diffX).divide(scale)); + } else if (view === ViewType.SIDE) { + const diffX = currentPointCoordOnPlane.x - this.action.resize.previousPosition.x; + const diffY = currentPointCoordOnPlane.y - this.action.resize.previousPosition.y; + currentPointInternalCoordinates + .fromArray(cuboidNodes[currentPointNumber]).add(new THREE.Vector3(-diffX, 0, diffY).divide(scale)); + } else if (view === ViewType.TOP) { + const diffX = currentPointCoordOnPlane.x - this.action.resize.previousPosition.x; + const diffY = currentPointCoordOnPlane.y - this.action.resize.previousPosition.y; + currentPointInternalCoordinates + .fromArray(cuboidNodes[currentPointNumber]).add(new THREE.Vector3(diffX, diffY, 0).divide(scale)); + } + const perspectivePosition = cuboid.perspective.localToWorld(currentPointInternalCoordinates.clone()); + + // small check to avoid case when points change their relative orientation + if ( + Math.sign(crosslyingPointInternalCoordinates.x - cuboidNodes[currentPointNumber][0]) !== + Math.sign(crosslyingPointInternalCoordinates.x - currentPointInternalCoordinates.x) || + Math.sign(crosslyingPointInternalCoordinates.y - cuboidNodes[currentPointNumber][1]) !== + Math.sign(crosslyingPointInternalCoordinates.y - currentPointInternalCoordinates.y) || + Math.sign(crosslyingPointInternalCoordinates.z - cuboidNodes[currentPointNumber][2]) !== + Math.sign(crosslyingPointInternalCoordinates.z - currentPointInternalCoordinates.z) + ) { + return; + } + + // finally let's compute new center and scale + scale.x *= Math.abs(crosslyingPointInternalCoordinates.x - currentPointInternalCoordinates.x); + scale.y *= Math.abs(crosslyingPointInternalCoordinates.y - currentPointInternalCoordinates.y); + scale.z *= Math.abs(crosslyingPointInternalCoordinates.z - currentPointInternalCoordinates.z); + const newPosition = crosslyingPointCoordinates.clone().add(perspectivePosition).divideScalar(2); + + // and apply them + this.moveObject(newPosition); + cuboid.setScale(scale.x, scale.y, scale.z); + this.adjustPerspectiveCameras(); + this.updateCameraFrustumPlane(view); + + this.action.resize.previousPosition = currentPointCoordOnPlane; + } + + private static isLeft(a: any, b: any, c: any): boolean { + // For reference + // A + // |\ // A = Rotation Center + // | \ // B = Previous Frame Position + // | C // C = Current Frame Position + // B + return (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x) > 0; + } + + private rotateCube(instance: CuboidModel, direction: number, view: ViewType): void { + switch (view) { + case ViewType.TOP: + instance.perspective.rotateZ(direction); + instance.top.rotateZ(direction); + instance.side.rotateZ(direction); + instance.front.rotateZ(direction); + this.rotateCamera(direction, view); + break; + case ViewType.FRONT: + instance.perspective.rotateX(direction); + instance.top.rotateX(direction); + instance.side.rotateX(direction); + instance.front.rotateX(direction); + this.rotateCamera(direction, view); + break; + case ViewType.SIDE: + instance.perspective.rotateY(direction); + instance.top.rotateY(direction); + instance.side.rotateY(direction); + instance.front.rotateY(direction); + this.rotateCamera(direction, view); + break; + default: + } + } + + private rotateCamera(direction: any, view: ViewType): void { + switch (view) { + case ViewType.TOP: + this.views.top.camera.rotateZ(direction); + break; + case ViewType.FRONT: + this.views.front.camera.rotateZ(direction); + break; + case ViewType.SIDE: + this.views.side.camera.rotateZ(direction); + break; + default: + } + } + + private detachCamera(view?: ViewType): void { + const coordTop = this.selectedCuboid.getReferenceCoordinates(ViewType.TOP); + const sphericaltop = new THREE.Spherical(); + sphericaltop.setFromVector3(coordTop); + + const coordSide = this.selectedCuboid.getReferenceCoordinates(ViewType.SIDE); + const sphericalside = new THREE.Spherical(); + sphericalside.setFromVector3(coordSide); + + const coordFront = this.selectedCuboid.getReferenceCoordinates(ViewType.FRONT); + const sphericalfront = new THREE.Spherical(); + sphericalfront.setFromVector3(coordFront); + + const { side: objectSideView, front: objectFrontView, top: objectTopView } = this.selectedCuboid; + const sideWorld = objectSideView.getWorldPosition(new THREE.Vector3()); + const frontWorld = objectFrontView.getWorldPosition(new THREE.Vector3()); + const topWorld = objectTopView.getWorldPosition(new THREE.Vector3()); + + const { camera: sideCamera } = this.views.side; + const { camera: frontCamera } = this.views.front; + const { camera: topCamera } = this.views.top; + + const camFrontRotate = objectFrontView + .getObjectByName('camRefRot') + .getWorldQuaternion(new THREE.Quaternion()); + switch (view) { + case ViewType.TOP: { + sideCamera.position.setFromSpherical(sphericalside); + sideCamera.lookAt(objectSideView.position.x, objectSideView.position.y, objectSideView.position.z); + sideCamera.rotation.z = this.views.side.scene.getObjectByName(Planes.SIDE).rotation.z; + sideCamera.scale.set(1, 1, 1); + + frontCamera.position.setFromSpherical(sphericalfront); + frontCamera.lookAt(objectFrontView.position.x, objectFrontView.position.y, objectFrontView.position.z); + frontCamera.setRotationFromQuaternion(camFrontRotate); + frontCamera.scale.set(1, 1, 1); + break; + } + case ViewType.SIDE: { + frontCamera.position.setFromSpherical(sphericalfront); + frontCamera.lookAt(objectFrontView.position.x, objectFrontView.position.y, objectFrontView.position.z); + frontCamera.setRotationFromQuaternion(camFrontRotate); + frontCamera.scale.set(1, 1, 1); + + topCamera.position.setFromSpherical(sphericaltop); + topCamera.lookAt(objectTopView.position.x, objectTopView.position.y, objectTopView.position.z); + topCamera.setRotationFromEuler(objectTopView.rotation); + topCamera.scale.set(1, 1, 1); + break; + } + case ViewType.FRONT: { + sideCamera.position.setFromSpherical(sphericalside); + sideCamera.lookAt(objectSideView.position.x, objectSideView.position.y, objectSideView.position.z); + sideCamera.rotation.z = this.views.side.scene.getObjectByName(Planes.SIDE).rotation.z; + sideCamera.scale.set(1, 1, 1); + + topCamera.position.setFromSpherical(sphericaltop); + topCamera.lookAt(objectTopView.position.x, objectTopView.position.y, objectTopView.position.z); + topCamera.setRotationFromEuler(objectTopView.rotation); + topCamera.scale.set(1, 1, 1); + break; + } + default: { + sideCamera.position.setFromSpherical(sphericalside); + sideCamera.lookAt(sideWorld.x, sideWorld.y, sideWorld.z); + sideCamera.rotation.z = this.views.side.scene.getObjectByName(Planes.SIDE).rotation.z; + sideCamera.scale.set(1, 1, 1); + + topCamera.position.setFromSpherical(sphericaltop); + topCamera.lookAt(topWorld.x, topWorld.y, topWorld.z); + topCamera.setRotationFromEuler(objectTopView.rotation); + topCamera.scale.set(1, 1, 1); + + frontCamera.position.setFromSpherical(sphericalfront); + frontCamera.lookAt(frontWorld.x, frontWorld.y, frontWorld.z); + frontCamera.setRotationFromQuaternion(camFrontRotate); + frontCamera.scale.set(1, 1, 1); + } + } + } + + private rotatePlane(direction: number, view: ViewType): void { + const sceneTopPlane = this.views.top.scene.getObjectByName(Planes.TOP); + const sceneSidePlane = this.views.side.scene.getObjectByName(Planes.SIDE); + const sceneFrontPlane = this.views.front.scene.getObjectByName(Planes.FRONT); + switch (view) { + case ViewType.TOP: + sceneTopPlane.rotateZ(direction); + sceneSidePlane.rotateY(direction); + sceneFrontPlane.rotateX(-direction); + break; + case ViewType.SIDE: + sceneTopPlane.rotateY(direction); + sceneSidePlane.rotateZ(direction); + sceneFrontPlane.rotateY(direction); + break; + case ViewType.FRONT: + sceneTopPlane.rotateX(direction); + sceneSidePlane.rotateX(-direction); + sceneFrontPlane.rotateZ(direction); + break; + default: { + const { top: objectTopView, side: objectSideView, front: objectFrontView } = this.selectedCuboid; + + const quaternionSide = objectSideView.getObjectByName(consts.PLANE_ROTATION_HELPER) + .getWorldQuaternion(new THREE.Quaternion()); + const rotationSide = new THREE.Euler().setFromQuaternion(quaternionSide); + + const quaternionFront = objectFrontView.getObjectByName(consts.PLANE_ROTATION_HELPER) + .getWorldQuaternion(new THREE.Quaternion()); + const rotationFront = new THREE.Euler().setFromQuaternion(quaternionFront); + + const quaternionTop = objectTopView.getObjectByName(consts.PLANE_ROTATION_HELPER) + .getWorldQuaternion(new THREE.Quaternion()); + const rotationTop = new THREE.Euler().setFromQuaternion(quaternionTop); + + const coordinates = { + x: objectTopView.position.x, + y: objectTopView.position.y, + z: objectTopView.position.z, + }; + + sceneTopPlane.rotation.set(rotationTop.x, rotationTop.y, rotationTop.z); + sceneSidePlane.rotation.set(rotationSide.x, rotationSide.y, rotationSide.z); + sceneFrontPlane.rotation.set(rotationFront.x, rotationFront.y, rotationFront.z); + + this.translateReferencePlane(coordinates); + } + } + } + + private renderRotateAction(view: ViewType, viewType: any): void { + const { renderer } = viewType; + const canvas = renderer.domElement; + if (!canvas) return; + + const canvasCentre = { + x: canvas.offsetLeft + canvas.offsetWidth / 2, + y: canvas.offsetTop + canvas.offsetHeight / 2, + }; + + if ( + this.action.rotation.screenInit.x === this.action.rotation.screenMove.x && + this.action.rotation.screenInit.y === this.action.rotation.screenMove.y + ) { + return; + } + + const startVector = { + x: this.action.rotation.screenInit.x - canvasCentre.x, + y: this.action.rotation.screenInit.y - canvasCentre.y, + }; + const endVector = { + x: this.action.rotation.screenMove.x - canvasCentre.x, + y: this.action.rotation.screenMove.y - canvasCentre.y, + }; + + const startMagnitude = Math.sqrt(startVector.x ** 2 + startVector.y ** 2); + const endMagnitude = Math.sqrt(endVector.x ** 2 + endVector.y ** 2); + + if (startMagnitude === 0 || endMagnitude === 0) { + return; + } + + const dotProduct = startVector.x * endVector.x + startVector.y * endVector.y; + let angle = Math.acos(dotProduct / (startMagnitude * endMagnitude)); + + if (Canvas3dViewImpl.isLeft(canvasCentre, this.action.rotation.screenInit, this.action.rotation.screenMove)) { + angle = -angle; + } + + this.action.rotation.recentMouseVector = this.views[view].rayCaster.mouseVector.clone(); + this.rotateCube(this.selectedCuboid, angle, view); + this.rotatePlane(angle, view); + + this.updateResizeHelperPos(); + this.updateRotationHelperPos(); + this.detachCamera(); + this.updateCameraFrustumPlane(); + + this.action.rotation.screenInit.x = this.action.rotation.screenMove.x; + this.action.rotation.screenInit.y = this.action.rotation.screenMove.y; + } + + private initiateAction(view: ViewType, viewType: any): void { + const { clientID } = this.model.data.activeElement; + const { cuboid, data } = this.drawnObjects[+clientID] || {}; + if (!data || !cuboid || data.lock) return; + + const intersectsHelperResize = viewType.rayCaster.renderer.intersectObjects( + cuboid[view].parent.children + .filter((child: THREE.Object3D) => child.name.startsWith(consts.RESIZE_HELPER_NAME)), + false, + ); + + const intersectsPlane = viewType.rayCaster.renderer.intersectObjects( + [viewType.scene.getObjectByName(`${view}Plane`)], + false, + ); + + if (intersectsHelperResize.length !== 0 && intersectsPlane.length !== 0) { + this.action.detected = true; + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + this.action.resize.status = true; + this.action.resize.helperElement = intersectsHelperResize[0].object; + this.action.resize.previousPosition = null; + return; + } + + const intersectsHelperRotation = viewType.rayCaster.renderer.intersectObjects( + cuboid[view].parent.children + .filter((child: THREE.Object3D) => child.name.startsWith(consts.ROTATION_HELPER_NAME)), + false, + ); + if (intersectsHelperRotation.length !== 0) { + this.action.rotation.helper = viewType.rayCaster.mouseVector.clone(); + this.action.rotation.status = true; + this.action.detected = true; + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + return; + } + + const intersectsBox = viewType.rayCaster.renderer.intersectObjects([cuboid[view]], false); + const intersectsPointCloud = viewType.rayCaster.renderer.intersectObjects( + [viewType.scene.getObjectByName(`${view}Plane`)], + true, + ); + if (intersectsBox.length !== 0 && !data.pinned) { + this.action.translation.helper = viewType.rayCaster.mouseVector.clone(); + this.action.translation.inverseMatrix = intersectsBox[0].object.parent.matrixWorld.invert(); + this.action.translation.offset = intersectsPointCloud[0].point.sub( + new THREE.Vector3().setFromMatrixPosition(intersectsBox[0].object.matrixWorld), + ); + this.action.translation.status = true; + this.action.detected = true; + this.views.top.controls.enabled = false; + this.views.side.controls.enabled = false; + this.views.front.controls.enabled = false; + } + } + + public keyControls(key: any): void { + const { controls } = this.views.perspective; + if (!controls) return; + if (key.shiftKey) { + switch (key.code) { + case CameraAction.ROTATE_RIGHT: + controls.rotate(0.1 * THREE.MathUtils.DEG2RAD * this.speed, 0, true); + break; + case CameraAction.ROTATE_LEFT: + controls.rotate(-0.1 * THREE.MathUtils.DEG2RAD * this.speed, 0, true); + break; + case CameraAction.TILT_UP: + controls.rotate(0, -0.05 * THREE.MathUtils.DEG2RAD * this.speed, true); + break; + case CameraAction.TILT_DOWN: + controls.rotate(0, 0.05 * THREE.MathUtils.DEG2RAD * this.speed, true); + break; + default: + break; + } + } else if (key.altKey === true) { + switch (key.code) { + case CameraAction.ZOOM_IN: + controls.dolly(consts.DOLLY_FACTOR, true); + break; + case CameraAction.ZOOM_OUT: + controls.dolly(-consts.DOLLY_FACTOR, true); + break; + case CameraAction.MOVE_LEFT: + controls.truck(-0.01 * this.speed, 0, true); + break; + case CameraAction.MOVE_RIGHT: + controls.truck(0.01 * this.speed, 0, true); + break; + case CameraAction.MOVE_DOWN: + controls.truck(0, -0.01 * this.speed, true); + break; + case CameraAction.MOVE_UP: + controls.truck(0, 0.01 * this.speed, true); + break; + default: + break; + } + } + } + + public html(): ViewsDOM { + return { + perspective: this.views.perspective.renderer.domElement, + top: this.views.top.renderer.domElement, + side: this.views.side.renderer.domElement, + front: this.views.front.renderer.domElement, + }; + } +} diff --git a/cvat-canvas3d/src/typescript/consts.ts b/cvat-canvas3d/src/typescript/consts.ts new file mode 100644 index 0000000..1835b50 --- /dev/null +++ b/cvat-canvas3d/src/typescript/consts.ts @@ -0,0 +1,50 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +const BASE_GRID_WIDTH = 2; +const MOVEMENT_FACTOR = 200; +const DOLLY_FACTOR = 5; +const MAX_DISTANCE = 100; +const MIN_DISTANCE = 0.3; +const ORTHO_FRUSTUM_HEIGHT = 1; +const ROTATION_HELPER_OFFSET = 0.75; +const CAMERA_REFERENCE = 'camRef'; +const CUBOID_EDGE_NAME = 'edges'; +const ROTATION_HELPER_NAME = '2DRotationHelper'; +const PLANE_ROTATION_HELPER = 'planeRotationHelper'; +const RESIZE_HELPER_NAME = '2DResizeHelper'; +const FOV_DEFAULT = 1; +const SIDE_VIEWS_MAX_ZOOM = 20; +const SIDE_VIEWS_MIN_ZOOM = 0.01; +const DEFAULT_GROUP_COLOR = '#e0e0e0'; +const DEFAULT_OUTLINE_COLOR = '#000000'; +const GROUPING_COLOR = '#8b008b'; +const MERGING_COLOR = '#0000ff'; +const SPLITTING_COLOR = '#1e90ff'; +const BASE_POINT_SIZE = 4; + +export default { + BASE_GRID_WIDTH, + MOVEMENT_FACTOR, + DOLLY_FACTOR, + MAX_DISTANCE, + MIN_DISTANCE, + ORTHO_FRUSTUM_HEIGHT, + ROTATION_HELPER_OFFSET, + CAMERA_REFERENCE, + CUBOID_EDGE_NAME, + ROTATION_HELPER_NAME, + PLANE_ROTATION_HELPER, + RESIZE_HELPER_NAME, + FOV_DEFAULT, + SIDE_VIEWS_MAX_ZOOM, + SIDE_VIEWS_MIN_ZOOM, + DEFAULT_GROUP_COLOR, + DEFAULT_OUTLINE_COLOR, + GROUPING_COLOR, + MERGING_COLOR, + SPLITTING_COLOR, + BASE_POINT_SIZE, +}; diff --git a/cvat-canvas3d/src/typescript/controlPointTexture.ts b/cvat-canvas3d/src/typescript/controlPointTexture.ts new file mode 100644 index 0000000..27c1bc2 --- /dev/null +++ b/cvat-canvas3d/src/typescript/controlPointTexture.ts @@ -0,0 +1,23 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as THREE from 'three'; + +function getCircleTexture(size: number): THREE.CanvasTexture { + const canvas = new OffscreenCanvas(size, size); + const ctx = canvas.getContext('2d'); + + if (ctx) { + ctx.fillStyle = 'white'; + ctx.beginPath(); + const r = size / 2; + ctx.arc(r, r, r, 0, Math.PI * 2); + ctx.closePath(); + ctx.fill(); + } + + return new THREE.CanvasTexture(canvas); +} + +export default getCircleTexture(256); diff --git a/cvat-canvas3d/src/typescript/cuboid.ts b/cvat-canvas3d/src/typescript/cuboid.ts new file mode 100644 index 0000000..5df51fc --- /dev/null +++ b/cvat-canvas3d/src/typescript/cuboid.ts @@ -0,0 +1,337 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as THREE from 'three'; +import { OrientationVisibility, ViewType } from './canvas3dModel'; +import constants from './consts'; +import getCircleTexture from './controlPointTexture'; +import { disposeObject3D, disposeObjectResources } from './utils'; + +export interface Indexable { + [key: string]: any; +} + +export interface ObjectArrowHelper { + x: THREE.ArrowHelper; + y: THREE.ArrowHelper; + z: THREE.ArrowHelper; +} + +export function makeCornerPointsMatrix(x: number, y: number, z: number): number[][] { + return ([ + [1 * x, 1 * y, 1 * z], + [1 * x, 1 * y, -1 * z], + [1 * x, -1 * y, 1 * z], + [1 * x, -1 * y, -1 * z], + [-1 * x, 1 * y, 1 * z], + [-1 * x, 1 * y, -1 * z], + [-1 * x, -1 * y, 1 * z], + [-1 * x, -1 * y, -1 * z], + ]); +} + +export class CuboidModel { + public perspective: THREE.Mesh; + public top: THREE.Mesh; + public side: THREE.Mesh; + public front: THREE.Mesh; + public wireframe: THREE.LineSegments; + + public orientationArrows: Record = { + [ViewType.PERSPECTIVE]: null, + [ViewType.TOP]: null, + [ViewType.SIDE]: null, + [ViewType.FRONT]: null, + }; + + public constructor(outline: string, outlineColor: string) { + const geometry = new THREE.BoxGeometry(1, 1, 1); + const material = new THREE.MeshBasicMaterial({ + color: 0x00ff00, + wireframe: false, + transparent: true, + opacity: 0.4, + }); + + this.perspective = new THREE.Mesh(geometry, material); + const geo = new THREE.EdgesGeometry(this.perspective.geometry); + this.wireframe = new THREE.LineSegments( + geo, + outline === 'line' ? new THREE.LineBasicMaterial({ color: outlineColor, linewidth: 4 }) : + new THREE.LineDashedMaterial({ + color: outlineColor, + dashSize: 0.05, + gapSize: 0.05, + }), + ); + this.wireframe.computeLineDistances(); + this.wireframe.renderOrder = 1; + this.perspective.add(this.wireframe); + + this.top = new THREE.Mesh(geometry, material); + this.side = new THREE.Mesh(geometry, material); + this.front = new THREE.Mesh(geometry, material); + + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + this.orientationArrows[view] = this.createArrows(); + Object.values(this.orientationArrows[view]).forEach((arrow) => { + this[view].add(arrow); + }); + }); + + const planeTop = new THREE.Mesh( + new THREE.PlaneGeometry(1, 1, 1, 1), + new THREE.MeshBasicMaterial({ + color: 0xff0000, + visible: false, + }), + ); + + const planeSide = new THREE.Mesh( + new THREE.PlaneGeometry(1, 1, 1, 1), + new THREE.MeshBasicMaterial({ + color: 0xff0000, + visible: false, + }), + ); + + const planeFront = new THREE.Mesh( + new THREE.PlaneGeometry(1, 1, 1, 1), + new THREE.MeshBasicMaterial({ + color: 0xff0000, + visible: false, + }), + ); + + this.top.add(planeTop); + planeTop.rotation.set(0, 0, 0); + planeTop.position.set(0, 0, 0.5); + planeTop.name = constants.PLANE_ROTATION_HELPER; + + this.side.add(planeSide); + planeSide.rotation.set(-Math.PI / 2, 0, Math.PI); + planeTop.position.set(0, 0.5, 0); + planeSide.name = constants.PLANE_ROTATION_HELPER; + + this.front.add(planeFront); + planeFront.rotation.set(0, Math.PI / 2, 0); + planeTop.position.set(0.5, 0, 0); + planeFront.name = constants.PLANE_ROTATION_HELPER; + + const cornerPoints = makeCornerPointsMatrix(0.5, 0.5, 0.5); + for (let i = 0; i < cornerPoints.length; i++) { + const point = new THREE.Vector3().fromArray(cornerPoints[i]); + const helper = new THREE.Mesh(new THREE.SphereGeometry(0.1)); + helper.visible = false; + helper.name = `cuboidNodeHelper_${i}`; + this.perspective.add(helper); + helper.position.copy(point); + } + + const camRotateHelper = new THREE.Object3D(); + camRotateHelper.translateX(-2); + camRotateHelper.name = 'camRefRot'; + camRotateHelper.up = new THREE.Vector3(0, 0, 1); + camRotateHelper.lookAt(new THREE.Vector3(0, 0, 0)); + this.front.add(camRotateHelper.clone()); + } + + private createArrows(): ObjectArrowHelper { + return { + x: new THREE.ArrowHelper(new THREE.Vector3(1, 0, 0), new THREE.Vector3(0.5, 0, 0), 1, 0xff0000), + y: new THREE.ArrowHelper(new THREE.Vector3(0, 1, 0), new THREE.Vector3(0, 0.5, 0), 1, 0x00ff00), + z: new THREE.ArrowHelper(new THREE.Vector3(0, 0, 1), new THREE.Vector3(0, 0, 0.5), 1, 0x0000ff), + }; + } + + public getRotationHelperPosition(viewType: ViewType): THREE.Vector3 { + const position = viewType === ViewType.TOP ? + new THREE.Vector3(0, constants.ROTATION_HELPER_OFFSET, 0) : + new THREE.Vector3(0, 0, constants.ROTATION_HELPER_OFFSET); + return this[viewType].localToWorld(position); + } + + public getResizeHelperPositions(): THREE.Vector3[] { + const cornerPoints = makeCornerPointsMatrix(0.5, 0.5, 0.5); + return cornerPoints.map((point) => { + const localPoint = new THREE.Vector3().fromArray(point); + return this.perspective.localToWorld(localPoint.clone()); + }); + } + + public setOrientationVisibility(orientationVisibility: OrientationVisibility): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + Object.entries(this.orientationArrows[view]).forEach(([axis, arrow]) => { + // eslint-disable-next-line no-param-reassign + arrow.visible = orientationVisibility[axis]; + }); + }); + } + + public setPosition(x: number, y: number, z: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].position.set(x, y, z); + }); + } + + public setScale(x: number, y: number, z: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].scale.set(x, y, z); + + // Arrow direction specifies its local Y axis, where it points to. + // When we change its direction to align with the X or Z axis, + // the arrow’s local coordinate system rotates accordingly. + // To maintain correct proportions, we apply the X or Z scaling of the cuboid + // to the arrow's Y axis (its original forward direction). + const xscale = 1.0 / x; + const yscale = 1.0 / y; + const zscale = 1.0 / z; + this.orientationArrows[view].x.scale.set(yscale, xscale, zscale); + this.orientationArrows[view].y.scale.set(xscale, yscale, zscale); + this.orientationArrows[view].z.scale.set(xscale, zscale, yscale); + }); + } + + public setRotation(x: number, y: number, z: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].rotation.set(x, y, z); + }); + } + + public attachCameraReference(): void { + const topCameraReference = new THREE.Object3D(); + topCameraReference.translateZ(2); + topCameraReference.name = constants.CAMERA_REFERENCE; + this.top.add(topCameraReference); + + const sideCameraReference = new THREE.Object3D(); + sideCameraReference.translateY(2); + sideCameraReference.name = constants.CAMERA_REFERENCE; + this.side.add(sideCameraReference); + + const frontCameraReference = new THREE.Object3D(); + frontCameraReference.translateX(2); + frontCameraReference.name = constants.CAMERA_REFERENCE; + this.front.add(frontCameraReference); + } + + public getReferenceCoordinates(viewType: string): THREE.Vector3 { + const camRef = (this as Indexable)[viewType].getObjectByName(constants.CAMERA_REFERENCE); + return camRef.getWorldPosition(new THREE.Vector3()); + } + + public setName(clientId: any): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + (this as Indexable)[view].name = clientId; + }); + } + + public setColor(color: string): void { + this.setOutlineColor(color); + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + ((this as Indexable)[view].material as THREE.MeshBasicMaterial).color.set(color); + }); + } + + public setOutlineColor(color: string): void { + (this.wireframe.material as THREE.MeshBasicMaterial).color.set(color); + } + + public setOpacity(opacity: number): void { + [ViewType.PERSPECTIVE, ViewType.TOP, ViewType.SIDE, ViewType.FRONT].forEach((view): void => { + ((this as Indexable)[view].material as THREE.MeshBasicMaterial).opacity = opacity / 100; + }); + } + + public dispose(): void { + disposeObject3D(this.perspective); + disposeObject3D(this.top); + disposeObject3D(this.side); + disposeObject3D(this.front); + + this.perspective = null; + this.top = null; + this.side = null; + this.front = null; + this.wireframe = null; + this.orientationArrows = { + [ViewType.PERSPECTIVE]: null, + [ViewType.TOP]: null, + [ViewType.SIDE]: null, + [ViewType.FRONT]: null, + }; + } +} + +export function createCuboidEdges(instance: THREE.Mesh): THREE.LineSegments { + const geometry = new THREE.EdgesGeometry(instance.geometry); + const edges = new THREE.LineSegments(geometry, new THREE.LineBasicMaterial({ color: '#ffffff', linewidth: 3 })); + edges.name = constants.CUBOID_EDGE_NAME; + instance.add(edges); + return edges; +} + +export function removeCuboidEdges(instance: THREE.Mesh): void { + const edges = instance.getObjectByName(constants.CUBOID_EDGE_NAME) as THREE.LineSegments; + if (edges) { + instance.remove(edges); + disposeObjectResources(edges); + } +} + +export function createResizeHelper(cuboid: CuboidModel, viewType: ViewType): void { + const material = new THREE.SpriteMaterial({ + color: '#ff0000', + opacity: 1, + map: getCircleTexture, + }); + + const positions = cuboid.getResizeHelperPositions(); + for (let i = 0; i < positions.length; i++) { + const position = positions[i]; + const helper = new THREE.Sprite(material); + helper.renderOrder = Number.MAX_SAFE_INTEGER; + helper.name = `${constants.RESIZE_HELPER_NAME}_${i}`; + helper.position.copy(position); + cuboid[viewType].parent.add(helper); + } +} + +export function removeResizeHelper(instance: THREE.Mesh): void { + instance.parent.children.filter((child: THREE.Object3D) => child.name.startsWith(constants.RESIZE_HELPER_NAME)) + .forEach((helper) => { + instance.parent.remove(helper); + if ((helper as THREE.Sprite).material) { + const material = (helper as THREE.Sprite).material as THREE.SpriteMaterial; + material.dispose(); + } + }); +} + +export function createRotationHelper(cuboid: CuboidModel, viewType: ViewType): void { + if ([ViewType.TOP, ViewType.SIDE, ViewType.FRONT].includes(viewType)) { + const helperPosition = cuboid.getRotationHelperPosition(viewType); + const rotationHelper = new THREE.Sprite(new THREE.SpriteMaterial({ + color: '#33b864', + opacity: 1, + map: getCircleTexture, + })); + rotationHelper.renderOrder = Number.MAX_SAFE_INTEGER; + rotationHelper.name = constants.ROTATION_HELPER_NAME; + rotationHelper.position.copy(helperPosition); + cuboid[viewType].parent.add(rotationHelper); + } +} + +export function removeRotationHelper(instance: THREE.Mesh): void { + const helper = instance.parent.getObjectByName(constants.ROTATION_HELPER_NAME); + if (helper) { + instance.parent.remove(helper); + if ((helper as THREE.Sprite).material) { + const material = (helper as THREE.Sprite).material as THREE.SpriteMaterial; + material.dispose(); + } + } +} diff --git a/cvat-canvas3d/src/typescript/index.ts b/cvat-canvas3d/src/typescript/index.ts new file mode 100644 index 0000000..b99d561 --- /dev/null +++ b/cvat-canvas3d/src/typescript/index.ts @@ -0,0 +1,6 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +export { default as ObjectState } from 'cvat-core/src/object-state'; +export { ObjectType } from 'cvat-core/src/enums'; diff --git a/cvat-canvas3d/src/typescript/master.ts b/cvat-canvas3d/src/typescript/master.ts new file mode 100644 index 0000000..e3f9d58 --- /dev/null +++ b/cvat-canvas3d/src/typescript/master.ts @@ -0,0 +1,44 @@ +// Copyright (C) 2021-2022 Intel Corporation +// +// SPDX-License-Identifier: MIT + +export interface Master { + subscribe(listener: Listener): void; + unsubscribe(listener: Listener): void; + unsubscribeAll(): void; + notify(reason: string): void; +} + +export interface Listener { + notify(master: Master, reason: string): void; +} + +export class MasterImpl implements Master { + private listeners: Listener[]; + + public constructor() { + this.listeners = []; + } + + public subscribe(listener: Listener): void { + this.listeners.push(listener); + } + + public unsubscribe(listener: Listener): void { + for (let i = 0; i < this.listeners.length; i++) { + if (this.listeners[i] === listener) { + this.listeners.splice(i, 1); + } + } + } + + public unsubscribeAll(): void { + this.listeners = []; + } + + public notify(reason: string): void { + for (const listener of this.listeners) { + listener.notify(this, reason); + } + } +} diff --git a/cvat-canvas3d/src/typescript/utils.ts b/cvat-canvas3d/src/typescript/utils.ts new file mode 100644 index 0000000..647085c --- /dev/null +++ b/cvat-canvas3d/src/typescript/utils.ts @@ -0,0 +1,55 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import * as THREE from 'three'; + +function disposeMaterial(material: THREE.Material): void { + Object.keys(material).forEach((prop) => { + const value = material[prop]; + if (value instanceof THREE.Texture) { + value?.dispose(); + } + }); + material.dispose(); +} + +export function disposeObjectResources(object: THREE.Object3D): void { + if ('geometry' in object && object.geometry instanceof THREE.BufferGeometry) { + object.geometry.dispose(); + } + + if ('material' in object) { + if (Array.isArray(object.material)) { + object.material.forEach((material) => { + if (material instanceof THREE.Material) { + disposeMaterial(material); + } + }); + } else if (object.material instanceof THREE.Material) { + disposeMaterial(object.material); + } + } + + if ('renderTarget' in object && object.renderTarget instanceof THREE.WebGLRenderTarget) { + object.renderTarget.dispose(); + } +} + +export function disposeObject3D(object: THREE.Object3D): void { + while (object.children.length > 0) { + const child = object.children[0]; + object.remove(child); + disposeObject3D(child); + } + + disposeObjectResources(object); +} + +export function disposeScene(scene: THREE.Scene): void { + while (scene.children.length > 0) { + const child = scene.children[0]; + scene.remove(child); + disposeObject3D(child); + } +} diff --git a/cvat-canvas3d/tsconfig.json b/cvat-canvas3d/tsconfig.json new file mode 100644 index 0000000..74336d0 --- /dev/null +++ b/cvat-canvas3d/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": ["dom", "dom.iterable", "esnext"], + "skipLibCheck": true, + "strict": false, + "forceConsistentCasingInFileNames": true, + "module": "esnext", + "moduleResolution": "node", + "esModuleInterop": true, + "isolatedModules": true, + "noEmit": true + }, + "include": ["src/typescript/canvas3d.ts"] +} diff --git a/cvat-canvas3d/webpack.config.cjs b/cvat-canvas3d/webpack.config.cjs new file mode 100644 index 0000000..21561b3 --- /dev/null +++ b/cvat-canvas3d/webpack.config.cjs @@ -0,0 +1,77 @@ +// Copyright (C) 2021-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const path = require('path'); + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const BundleDeclarationsWebpackPlugin = require('bundle-declarations-webpack-plugin'); + +const styleLoaders = [ + 'style-loader', + { + loader: 'css-loader', + options: { + importLoaders: 2, + }, + }, + { + loader: 'postcss-loader', + options: { + postcssOptions: { + plugins: [ + [ + 'postcss-preset-env', {}, + ], + ], + }, + }, + }, + 'sass-loader', +]; + +module.exports = { + target: 'web', + mode: 'production', + devtool: 'source-map', + entry: { + 'cvat-canvas3d': './src/typescript/canvas3d.ts', + }, + output: { + path: path.resolve(__dirname, 'dist'), + filename: '[name].[contenthash].js', + library: 'canvas3d', + libraryTarget: 'window', + }, + resolve: { + extensions: ['.ts', '.js', '.json'], + }, + module: { + rules: [ + { + test: /\.ts$/, + exclude: /node_modules/, + use: { + loader: 'babel-loader', + options: { + plugins: ['@babel/plugin-proposal-class-properties'], + presets: ['@babel/preset-env', '@babel/typescript'], + sourceType: 'unambiguous', + }, + }, + }, + { + test: /\.scss$/, + exclude: /node_modules/, + use: styleLoaders, + }, + ], + }, + plugins: [ + new BundleDeclarationsWebpackPlugin({ + outFile: "declaration/src/cvat-canvas.d.ts", + }), + ], +}; diff --git a/cvat-cli/.gitignore b/cvat-cli/.gitignore new file mode 100644 index 0000000..43995bd --- /dev/null +++ b/cvat-cli/.gitignore @@ -0,0 +1,66 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +venv/ +.venv/ +.python-version +.pytest_cache + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#Ipython Notebook +.ipynb_checkpoints diff --git a/cvat-cli/README.md b/cvat-cli/README.md new file mode 100644 index 0000000..113e9b0 --- /dev/null +++ b/cvat-cli/README.md @@ -0,0 +1,79 @@ +# Command-line client for CVAT + +A simple command line interface for working with CVAT. At the moment it +implements a basic feature set but may serve as the starting point for a more +comprehensive CVAT administration tool in the future. + +The following subcommands are supported: + +- Projects: + - `backup` - back up a project + - `create` - create a new project + - `create-from-backup` - create a project from a backup file + - `delete` - delete projects + - `export-dataset` - export a project as a dataset + - `import-dataset` - import annotations into a project from a dataset + - `ls` - list all projects + +- Tasks: + - `create` - create a new task + - `create-from-backup` - create a task from a backup file + - `delete` - delete tasks + - `ls` - list all tasks + - `frames` - download frames from a task + - `export-dataset` - export a task as a dataset + - `import-dataset` - import annotations into a task from a dataset + - `backup` - back up a task + - `auto-annotate` - automatically annotate a task using a local function + +- Functions (Enterprise/Cloud only): + - `create-native` - create a function that can be powered by an agent + - `delete` - delete a function + - `run-agent` - process requests for a native function + +## Installation + +`pip install cvat-cli` + +## Usage + +The general form of a CLI command is: + +```console +$ cvat-cli +``` + +where: + +- `` are options shared between all subcommands; +- `` is a CVAT resource, such as `task`; +- `` is the action to do with the resource, such as `create`; +- `` is any options specific to a particular resource and action. + +You can list available subcommands and options using the `--help` option: + +``` +$ cvat-cli --help # get help on available common options and resources +$ cvat-cli --help # get help on actions for the given resource +$ cvat-cli --help # get help on action-specific options +``` + +## Examples + +Create a task with local images: + +```bash +cvat-cli --auth user task create + --labels '[{"name": "car"}, {"name": "person"}]' + "test_task" + "local" + "image1.jpg" "image2.jpg" +``` + +List tasks on a custom server with auth: + +```bash +cvat-cli --auth admin:password \ + --server-host cvat.my.server.com --server-port 30123 \ + task ls +``` diff --git a/cvat-cli/pyproject.toml b/cvat-cli/pyproject.toml new file mode 100644 index 0000000..4ddb7ee --- /dev/null +++ b/cvat-cli/pyproject.toml @@ -0,0 +1,42 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +[project] +dynamic = ["version"] + +name = "cvat-cli" +description = "Command-line client for CVAT" +readme = "README.md" +authors = [{name = "CVAT.ai Corporation", email = "support@cvat.ai"}] +license = "MIT" +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +requires-python = ">=3.10" +dependencies = [ + "cvat-sdk==2.69.1", + + "attrs>=24.2.0", + "Pillow>=10.3.0", +] + +[project.scripts] +cvat-cli = "cvat_cli.__main__:main" + +[project.urls] +homepage = "https://github.com/cvat-ai/cvat/" + +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.dynamic] +version = {attr = "cvat_cli.version.VERSION"} + +[tool.isort] +profile = "black" +forced_separate = ["tests"] +line_length = 100 +skip_gitignore = true # align tool behavior with Black diff --git a/cvat-cli/src/cvat_cli/__init__.py b/cvat-cli/src/cvat_cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cvat-cli/src/cvat_cli/__main__.py b/cvat-cli/src/cvat_cli/__main__.py new file mode 100755 index 0000000..7ad2277 --- /dev/null +++ b/cvat-cli/src/cvat_cli/__main__.py @@ -0,0 +1,45 @@ +# Copyright (C) 2020-2022 Intel Corporation +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import logging +import sys + +import urllib3.exceptions +from cvat_sdk import exceptions + +from ._internal.commands_all import COMMANDS +from ._internal.common import ( + CriticalError, + build_client, + configure_common_arguments, + configure_logger, +) +from ._internal.utils import popattr + +logger = logging.getLogger(__name__) + + +def main(args: list[str] = None): + parser = argparse.ArgumentParser(description=COMMANDS.description) + configure_common_arguments(parser) + COMMANDS.configure_parser(parser) + + parsed_args = parser.parse_args(args) + + configure_logger(logger, parsed_args) + + try: + with build_client(parsed_args, logger=logger) as client: + popattr(parsed_args, "_executor")(client, **vars(parsed_args)) + except (exceptions.ApiException, urllib3.exceptions.HTTPError, CriticalError) as e: + logger.critical(e) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cvat-cli/src/cvat_cli/_internal/__init__.py b/cvat-cli/src/cvat_cli/_internal/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cvat-cli/src/cvat_cli/_internal/agent.py b/cvat-cli/src/cvat_cli/_internal/agent.py new file mode 100644 index 0000000..ab5d5f0 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/agent.py @@ -0,0 +1,655 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import concurrent.futures +import contextlib +import json +import multiprocessing +import random +import secrets +import shutil +import tempfile +import threading +import time +from collections.abc import Generator, Iterator +from datetime import datetime, timedelta, timezone +from http import HTTPStatus +from pathlib import Path +from typing import TYPE_CHECKING + +import attrs +import cvat_sdk.auto_annotation as cvataa +import cvat_sdk.datasets as cvatds +import urllib3.exceptions +from cvat_sdk import Client +from cvat_sdk.datasets.caching import make_cache_manager +from cvat_sdk.exceptions import ApiException + +from .agent_driver import ( + AgentFunctionDriver, + BadArError, + IncompatibleFunctionError, + set_worker_current_function, + worker_current_function, +) +from .agent_driver_detection import AgentDetectionFunctionDriver +from .agent_driver_tracking import AgentTrackingFunctionDriver, TrackingStateIdGenerator +from .common import CriticalError, FunctionLoader + +if TYPE_CHECKING: + from _typeshed import SupportsReadline + +FUNCTION_PROVIDER_NATIVE = "native" +REQUEST_CATEGORY_BATCH = "batch" +REQUEST_CATEGORY_INTERACTIVE = "interactive" + +REQUEST_CATEGORIES_WITH_DECREASING_PRIORITY = (REQUEST_CATEGORY_INTERACTIVE, REQUEST_CATEGORY_BATCH) + +_POLLING_INTERVAL_MEAN_FREQUENT = timedelta(seconds=60) +_POLLING_INTERVAL_MEAN_RARE = timedelta(minutes=10) +_JITTER_AMOUNT = 0.15 +_DEFAULT_RETRY_DELAY = timedelta(seconds=5) + +_UPDATE_INTERVAL = timedelta(seconds=30) + + +class _ExponentialBackoff: + def __init__(self, max_delay: timedelta, current_delay: timedelta) -> None: + self._max_delay = max_delay + self._current_delay = current_delay + + def reset(self, current_delay: timedelta) -> None: + self._current_delay = current_delay + + def next(self) -> timedelta: + delay = self._current_delay + self._current_delay = min(self._current_delay * 2, self._max_delay) + return delay + + +class RecoverableExecutor: + # A wrapper around ProcessPoolExecutor that recreates the underlying + # executor when a worker crashes. + def __init__(self, initializer, initargs): + self._mp_context = multiprocessing.get_context("spawn") + self._initializer = initializer + self._initargs = initargs + self._executor = self._new_executor() + + def _new_executor(self): + return concurrent.futures.ProcessPoolExecutor( + max_workers=1, + mp_context=self._mp_context, + initializer=self._initializer, + initargs=self._initargs, + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._executor.shutdown() + + def submit(self, func, /, *args, **kwargs): + return self._executor.submit(func, *args, **kwargs) + + def result(self, future: concurrent.futures.Future): + try: + return future.result() + except concurrent.futures.BrokenExecutor: + self._executor.shutdown() + self._executor = self._new_executor() + raise + + +def _default_tracking_state_id_generator() -> str: + # This is defined as a separate function so that tests can monkeypatch it + # in order to get deterministic state IDs. + return secrets.token_urlsafe(32) + + +def _worker_init( + function_loader: FunctionLoader, state_id_generator: TrackingStateIdGenerator +) -> None: + current_function = function_loader.load() + set_worker_current_function(current_function) + + get_function_driver_class(current_function.spec).init_worker(state_id_generator) + + +def _worker_job_get_function_spec(): + return worker_current_function().spec + + +@attrs.frozen +class _Event: + type: str + data: str + + +@attrs.frozen +class _NewReconnectionDelay: + delay: timedelta + + +class _TaskCacheLimiter: + """ + This class deletes least-recently used tasks from the dataset cache, + so that at any time the cache contains at most _MAX_CACHED_TASKS tasks. + + This helps manage disk usage, since agents may run indefinitely, and + we don't want the dataset cache to keep growing. + """ + + _MAX_CACHED_TASKS = 10 + + def __init__(self, client: Client) -> None: + self._client = client + self._cache_manager = make_cache_manager(client, cvatds.UpdatePolicy.IF_MISSING_OR_STALE) + + self._cached_task_ids = [] + + self._task_ids_in_use = set() + + @contextlib.contextmanager + def using_cache_for_task(self, task_id: int) -> Generator[None, None, None]: + if task_id in self._task_ids_in_use: + yield + return + + if task_id in self._cached_task_ids: + self._cached_task_ids.remove(task_id) + + self._task_ids_in_use.add(task_id) + + if len(self._cached_task_ids) + len(self._task_ids_in_use) > self._MAX_CACHED_TASKS: + self._delete_task_cache(self._cached_task_ids.pop(0)) + + try: + yield + finally: + self._task_ids_in_use.remove(task_id) + self._cached_task_ids.append(task_id) + + def _delete_task_cache(self, task_id: int) -> None: + self._client.logger.info("Deleting task %d from the cache to make room...", task_id) + shutil.rmtree(self._cache_manager.task_dir(task_id), ignore_errors=True) + + +def _parse_event_stream( + stream: SupportsReadline[bytes], +) -> Iterator[_Event | _NewReconnectionDelay]: + # https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation + + event_type = event_data = "" + + while True: + line_bytes = stream.readline() + if not line_bytes: + return + + line = line_bytes.decode("UTF-8").removesuffix("\n").removesuffix("\r") + + # Technically, a standalone \r is supposed to be treated as a line terminator, + # but it's annoying to implement, and there's no reason for CVAT to use that. + if "\r" in line: + raise ValueError("CR found in event stream") + + if not line: + yield _Event(event_type, event_data.removesuffix("\n")) + event_type = event_data = "" + continue + + if line.startswith(":"): + # it's a comment/keepalive + continue + + if ":" in line: + field_name, field_value = line.split(":", maxsplit=1) + field_value = field_value.removeprefix(" ") + else: + field_name = line + field_value = "" + + if field_name == "event": + event_type = field_value + elif field_name == "data": + event_data += field_value + "\n" + elif field_name == "retry": + if field_value.isascii() and field_value.isdecimal(): + yield _NewReconnectionDelay(timedelta(milliseconds=int(field_value))) + + +def get_function_driver_class(function_spec: object) -> type[AgentFunctionDriver]: + if isinstance(function_spec, cvataa.DetectionFunctionSpec): + return AgentDetectionFunctionDriver + + if isinstance(function_spec, cvataa.TrackingFunctionSpec): + return AgentTrackingFunctionDriver + + raise CriticalError(f"Unsupported function spec type: {type(function_spec).__name__}") + + +class _Agent: + def __init__(self, client: Client, executor: RecoverableExecutor, function_id: int): + self._rng = random.Random() # nosec + + self._client = client + self._function_id = function_id + function_spec = executor.result(executor.submit(_worker_job_get_function_spec)) + self._function_driver = get_function_driver_class(function_spec)( + client, executor, function_spec + ) + + _, response = self._client.api_client.call_api( + "/api/functions/{function_id}", + "GET", + path_params={"function_id": self._function_id}, + ) + + remote_function = json.loads(response.data) + + self._validate_function_compatibility(remote_function) + + self._agent_id = secrets.token_hex(16) + self._client.logger.info("Agent starting with ID %r", self._agent_id) + + self._task_cache_limiter = _TaskCacheLimiter(client) + + self._queue_watch_response = None + self._queue_watch_response_lock = threading.Lock() + self._queue_watcher_should_stop = threading.Event() + + self._potential_work_condition = threading.Condition(threading.Lock()) + self._potential_work_per_category = { + category: True for category in REQUEST_CATEGORIES_WITH_DECREASING_PRIORITY + } + + self._polling_interval = _POLLING_INTERVAL_MEAN_FREQUENT + + # If we fail to connect to the queue event stream, it might be because + # the server is too old and doesn't support the watch endpoint. + # In this case, it doesn't make sense to continue trying to connect frequently, + # although we should still be trying occasionally in case the error is transient. + # Once we're successful, we'll rely on the server to set a new reconnection delay. + self._queue_reconnection_delay = _ExponentialBackoff( + _POLLING_INTERVAL_MEAN_RARE, _POLLING_INTERVAL_MEAN_RARE + ) + + def _validate_function_compatibility(self, remote_function: dict) -> None: + function_id = remote_function["id"] + + if remote_function["provider"] != FUNCTION_PROVIDER_NATIVE: + raise CriticalError( + f"Function #{function_id} has provider {remote_function['provider']!r}. " + f"Agents can only be run for functions with provider {FUNCTION_PROVIDER_NATIVE!r}." + ) + + try: + if remote_function["kind"] != self._function_driver.FUNCTION_KIND: + raise IncompatibleFunctionError( + f"kind is {remote_function['kind']!r} " + f"(expected {self._function_driver.FUNCTION_KIND!r})." + ) + + self._function_driver.validate_function_compatibility(remote_function) + except IncompatibleFunctionError as ex: + raise CriticalError( + f"Function #{function_id} is incompatible with function object: {ex}" + ) from ex + + def _wait_between_polls(self): + # offset the interval randomly to avoid synchronization between workers + timeout_multiplier = self._rng.uniform(1 - _JITTER_AMOUNT, 1 + _JITTER_AMOUNT) + + with self._potential_work_condition: + wait_succeeded = self._potential_work_condition.wait_for( + lambda: any(self._potential_work_per_category.values()), + timeout=self._polling_interval.total_seconds() * timeout_multiplier, + ) + + if not wait_succeeded: + # If we timed out, there is a possibility that the queue watcher is broken or + # that it somehow missed an event. Either way, we'll force a poll to make sure + # we don't miss anything. + for category in self._potential_work_per_category: + self._potential_work_per_category[category] = True + + def _dispatch_queue_event(self, event: _Event) -> None: + if event.type == "newrequest": + event_data_object = json.loads(event.data) + request_category = event_data_object["request_category"] + + with self._potential_work_condition: + if request_category in self._potential_work_per_category: + self._client.logger.info( + "Received notification about a new request of category %r", + request_category, + ) + self._potential_work_per_category[request_category] = True + self._potential_work_condition.notify() + else: + self._client.logger.warning( + "Received notification about a new request of unknown category: %r", + request_category, + ) + else: + self._client.logger.warning("Received event of unknown type: %r", event.type) + + def _wait_before_reconnecting_to_queue(self): + delay_multiplier = self._rng.uniform(1, 1 + _JITTER_AMOUNT) + self._queue_watcher_should_stop.wait( + timeout=self._queue_reconnection_delay.next().total_seconds() * delay_multiplier + ) + + def _watch_queue(self) -> None: + while not self._queue_watcher_should_stop.is_set(): + # Until we can (re)connect to the event stream, poll more frequently. + self._polling_interval = _POLLING_INTERVAL_MEAN_FREQUENT + + with self._queue_watch_response_lock: + self._client.logger.info("Attempting to watch the function's queue...") + + try: + _, self._queue_watch_response = self._client.api_client.call_api( + "/api/functions/queues/{queue_id}/watch", + "GET", + path_params={"queue_id": f"function:{self._function_id}"}, + _parse_response=False, + ) + except Exception: + self._client.logger.error( + "Failed to connect to the queue event stream; will retry", + exc_info=True, + ) + self._wait_before_reconnecting_to_queue() + continue + else: + self._client.logger.info("Connected to the queue event stream") + + # Now we can rely on notifications, so slow down polling. + self._polling_interval = _POLLING_INTERVAL_MEAN_RARE + + try: + for message in _parse_event_stream(self._queue_watch_response): + if isinstance(message, _Event): + self._dispatch_queue_event(message) + elif isinstance(message, _NewReconnectionDelay): + self._queue_reconnection_delay.reset(message.delay) + self._client.logger.info( + "New queue event stream reconnection delay is %fs", + message.delay.total_seconds(), + ) + else: + assert False, f"unexpected message type {type(message)}" + + self._queue_watch_response.release_conn() + + # We should normally not get here unless the function is deleted on the server. + # However, we don't know that for sure, so instead of quitting immediately, + # we'll ask the main thread to poll for an AR. + # If the function did get deleted, the main thread will get a 404 and quit. + # Otherwise, we'll just reconnect again. + with self._potential_work_condition: + for category in self._potential_work_per_category: + self._potential_work_per_category[category] = True + self._potential_work_condition.notify() + + self._client.logger.warning("Event stream ended; will reconnect") + except Exception: + # This is an extra check to prevent useless messages. + # If we crashed, but the main thread wants us to stop anyway, + # we should just stop and not spam the log. + if self._queue_watcher_should_stop.is_set(): + break + + self._client.logger.error( + "Event stream interrupted or other error; will reconnect", exc_info=True + ) + finally: + self._queue_watch_response.close() + + self._wait_before_reconnecting_to_queue() + + def run(self, *, burst: bool) -> None: + if burst: + self._process_all_available_ars() + self._client.logger.info("No annotation requests left in queue; exiting.") + else: + watcher = threading.Thread(name="Queue Watcher", target=self._watch_queue) + watcher.start() + + try: + while True: + self._process_all_available_ars() + self._wait_between_polls() + finally: + self._queue_watcher_should_stop.set() + + with self._queue_watch_response_lock: + if self._queue_watch_response: + with contextlib.suppress(Exception): + # shutdown() requires urllib3 2.3.0, whereas we only require 1.25 + # (via the SDK). The reason we can't bump the requirement is that + # the testsuite depends on botocore, which is incompatible with urllib3 + # 2.x on Python 3.9 and earlier. + # Since pip will, by default, install the latest dependency versions, + # most users should not be affected. For the ones that are, shutdown + # will be broken, but everything else should still work fine. + # This should be revisited once we drop Python 3.9 support. + # TODO: check in newer versions + self._queue_watch_response.shutdown() + + watcher.join() + + def _process_all_available_ars(self): + for category in REQUEST_CATEGORIES_WITH_DECREASING_PRIORITY: + self._process_available_ars(category) + + def _process_available_ars(self, category) -> None: + with self._potential_work_condition: + if not self._potential_work_per_category[category]: + return + + self._potential_work_per_category[category] = False + + while ar_assignment := self._poll_for_ar(category): + self._process_ar(ar_assignment) + + def _process_ar(self, ar_assignment: dict) -> None: + ar_id = ar_assignment["ar_id"] + ar_params = ar_assignment["ar_params"] + + self._client.logger.info( + "Got assigned annotation request %r of type %r (%s)", + ar_id, + ar_params["type"], + # Log only a few key parameters to avoid cluttering the info-level log. + " ".join([f"{k}={ar_params[k]!r}" for k in ("task", "frame") if k in ar_params]), + ) + self._client.logger.debug("AR %r parameters: %r", ar_id, ar_params) + + last_update_timestamp = datetime.now(tz=timezone.utc) + + def check_in(*, current_progress: float) -> None: + nonlocal last_update_timestamp + current_timestamp = datetime.now(tz=timezone.utc) + + if current_timestamp >= last_update_timestamp + _UPDATE_INTERVAL: + self._update_ar(ar_id, current_progress) + last_update_timestamp = current_timestamp + + # Interactive requests are time sensitive, so if there are any, + # we have to put the current AR on hold and process them ASAP. + self._process_available_ars(REQUEST_CATEGORY_INTERACTIVE) + + try: + with self._task_cache_limiter.using_cache_for_task(ar_params["task"]): + result = self._function_driver.calculate_result_for_ar(ar_params, check_in) + self._complete_ar(ar_id, result) + except Exception as ex: + self._client.logger.error("Failed to process AR %r", ar_id, exc_info=True) + + # Arbitrary exceptions may contain details of the client's system or code, which + # shouldn't be exposed to the server (and to users of the function). + # Therefore, we only produce a limited amount of detail, and only in known failure cases. + error_message = "Unknown error" + + if isinstance(ex, ApiException): + if ex.status: + error_message = f"Received HTTP status {ex.status}" + else: + error_message = "Failed an API call" + elif isinstance(ex, urllib3.exceptions.RequestError): + if isinstance(ex, urllib3.exceptions.MaxRetryError): + ex_type = type(ex.reason) + else: + ex_type = type(ex) + + error_message = f"Failed to make an HTTP request to {ex.url} ({ex_type.__name__})" + elif isinstance(ex, urllib3.exceptions.HTTPError): + error_message = "Failed to make an HTTP request" + elif isinstance(ex, cvataa.BadFunctionError): + error_message = "Underlying function returned incorrect result: " + str(ex) + elif isinstance(ex, BadArError): + error_message = "Invalid annotation request: " + str(ex) + elif isinstance(ex, concurrent.futures.BrokenExecutor): + error_message = "Worker process crashed" + + try: + self._client.api_client.call_api( + "/api/functions/queues/{queue_id}/requests/{request_id}/fail", + "POST", + path_params={ + "queue_id": f"function:{self._function_id}", + "request_id": ar_id, + }, + body={"agent_id": self._agent_id, "exc_info": error_message}, + ) + except Exception: + self._client.logger.error("Couldn't fail AR %r", ar_id, exc_info=True) + else: + self._client.logger.info("AR %r failed", ar_id) + + def _handle_retryable_post_error(self, ex: Exception, delay: _ExponentialBackoff) -> bool: + # Normally, urllib3 handles retries for HTTP requests, + # but it only does it for idempotent ones. + # So for POST requests that are safe to retry, we have to do it ourselves. + # This function must be called from an exception handler. + # It will return True if the operation should be retried, + # or False if the exception should be re-raised. + + is_rate_limit = False + delay_sec = None + + if isinstance(ex, ApiException): + try: + delay_sec = int(ex.headers["Retry-After"]) + except (KeyError, ValueError): + pass + + if ex.status == HTTPStatus.TOO_MANY_REQUESTS: + is_rate_limit = True + elif ex.status and 400 <= ex.status < 500: + # We did something wrong; no point in retrying. + return False + + if delay_sec is None: + delay_multiplier = self._rng.uniform(1, 1 + _JITTER_AMOUNT) + delay_sec = delay.next().total_seconds() * delay_multiplier + + if is_rate_limit: + self._client.logger.warning("Rate limited; will retry in %.2fs", delay_sec) + else: + self._client.logger.error( + "Request failed; will retry in %.2fs", delay_sec, exc_info=True + ) + time.sleep(delay_sec) + return True + + def _poll_for_ar(self, category: str) -> dict | None: + retry_delay = _ExponentialBackoff(_POLLING_INTERVAL_MEAN_RARE, _DEFAULT_RETRY_DELAY) + + while True: + self._client.logger.info( + "Trying to acquire an annotation request of category %r...", category + ) + try: + _, response = self._client.api_client.call_api( + "/api/functions/queues/{queue_id}/requests/acquire", + "POST", + path_params={"queue_id": f"function:{self._function_id}"}, + body={"agent_id": self._agent_id, "request_category": category}, + ) + break + except (urllib3.exceptions.HTTPError, ApiException) as ex: + if not self._handle_retryable_post_error(ex, retry_delay): + raise + + response_data = json.loads(response.data) + return response_data["ar_assignment"] + + def _update_ar(self, ar_id: str, progress: float) -> None: + self._client.logger.info("Updating AR %r progress to %.2f%%...", ar_id, progress * 100) + + try: + self._client.api_client.call_api( + "/api/functions/queues/{queue_id}/requests/{request_id}/update", + "POST", + path_params={"queue_id": f"function:{self._function_id}", "request_id": ar_id}, + body={"agent_id": self._agent_id, "progress": progress}, + ) + except (urllib3.exceptions.HTTPError, ApiException): + # Updating the progress is not critical, so log and continue onwards. + self._client.logger.error("Failed to update AR %r progress", ar_id, exc_info=True) + + def _complete_ar(self, ar_id: str, result: dict) -> None: + # It would be frustrating for the user if we calculate the result of an AR and then fail + # due to a transient error when submitting it, so we should retry at least a couple times. + + delay = _ExponentialBackoff(_POLLING_INTERVAL_MEAN_RARE, _DEFAULT_RETRY_DELAY) + attempt_num = 0 + + while True: + self._client.logger.info("Submitting result for AR %r...", ar_id) + try: + self._client.api_client.call_api( + "/api/functions/queues/{queue_id}/requests/{request_id}/complete", + "POST", + path_params={"queue_id": f"function:{self._function_id}", "request_id": ar_id}, + body={"agent_id": self._agent_id, **result}, + ) + break + except (urllib3.exceptions.HTTPError, ApiException) as ex: + if attempt_num >= 3: + self._client.logger.error( + "Exceeded maximum retries for submitting AR %r", ar_id + ) + raise + + if not self._handle_retryable_post_error(ex, delay): + raise + + attempt_num += 1 + + self._client.logger.info("AR %r completed", ar_id) + + +def run_agent( + client: Client, function_loader: FunctionLoader, function_id: int, *, burst: bool +) -> None: + with ( + RecoverableExecutor( + initializer=_worker_init, + initargs=[function_loader, _default_tracking_state_id_generator], + ) as executor, + tempfile.TemporaryDirectory() as cache_dir, + ): + client.config.cache_dir = Path(cache_dir, "cache") + client.logger.info("Will store cache at %s", client.config.cache_dir) + + agent = _Agent(client, executor, function_id) + agent.run(burst=burst) diff --git a/cvat-cli/src/cvat_cli/_internal/agent_driver.py b/cvat-cli/src/cvat_cli/_internal/agent_driver.py new file mode 100644 index 0000000..86bfc65 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/agent_driver.py @@ -0,0 +1,116 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, ClassVar, Generic, Protocol, TypeVar + +import cvat_sdk.auto_annotation as cvataa +import cvat_sdk.datasets as cvatds +from cvat_sdk import Client +from typing_extensions import Self + +if TYPE_CHECKING: + from .agent import RecoverableExecutor + from .agent_driver_tracking import TrackingStateIdGenerator + + +class BadArError(Exception): + pass + + +class IncompatibleFunctionError(Exception): + # This should only be thrown from inside validate_function_compatibility methods. + pass + + +_current_function: cvataa.AutoAnnotationFunction + + +def worker_current_function() -> cvataa.AutoAnnotationFunction: + return _current_function + + +def set_worker_current_function(func: cvataa.AutoAnnotationFunction) -> None: + global _current_function + _current_function = func + + +class CheckInCallback(Protocol): + def __call__(self, *, current_progress: float) -> None: + """ + Temporarily suspends the processing of the current AR to report progress back to the server + and optionally process any pending interactive ARs. Should only be called during batch AR + processing (interactive ARs should be processed quickly enough to not require this). + """ + + +SpecT = TypeVar("SpecT") + + +class AgentFunctionDriver(Generic[SpecT]): + FUNCTION_KIND: ClassVar[str] + + def __init__(self, client: Client, executor: RecoverableExecutor, function_spec: SpecT) -> None: + self._client = client + self._executor = executor + self._function_spec = function_spec + + @classmethod + def init_worker(cls, state_id_generator: TrackingStateIdGenerator) -> None: + pass + + @classmethod + def get_remote_function_fields(cls, spec: SpecT) -> dict[str, Any]: + raise NotImplementedError + + def validate_function_compatibility(self, remote_function: dict) -> None: + raise NotImplementedError + + _CALCULATE_RESULT_PER_AR_TYPE: ClassVar[ + dict[str, Callable[[Self, dict[str, Any], CheckInCallback], dict[str, Any]]] + ] + + def calculate_result_for_ar( + self, ar_params: dict[str, Any], check_in: CheckInCallback + ) -> dict[str, Any]: + if calc := self._CALCULATE_RESULT_PER_AR_TYPE.get(ar_params["type"]): + return calc(self, ar_params, check_in) + + raise BadArError(f"unsupported type: {ar_params['type']!r}") + + def _get_sample_from_ar_params(self, ar_params): + ds = cvatds.TaskDataset( + self._client, + ar_params["task"], + load_annotations=False, + media_download_policy=cvatds.MediaDownloadPolicy.FETCH_FRAMES_ON_DEMAND, + ) + + frame_index = ar_params["frame"] + + # Since ds.samples excludes deleted frames, we can't just do sample = ds.samples[frame_index]. + # Once we drop Python 3.9, we can change this to use bisect instead of the linear search. + for sample in ds.samples: + if sample.frame_index == frame_index: + break + else: + raise BadArError(f"Frame with index {frame_index} does not exist in the task") + + return sample, ds.labels + + def _load_image_for_ar(self, sample, ar_params): + image = sample.media.load_image() + + if roi := ar_params.get("roi"): + xtl, ytl, xbr, ybr = roi + width, height = image.size + + if xbr > width or ybr > height: + raise BadArError("Invalid ROI") + + return image.crop((xtl, ytl, xbr, ybr)) + + return image diff --git a/cvat-cli/src/cvat_cli/_internal/agent_driver_detection.py b/cvat-cli/src/cvat_cli/_internal/agent_driver_detection.py new file mode 100644 index 0000000..8db5a11 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/agent_driver_detection.py @@ -0,0 +1,207 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from collections.abc import Sequence +from typing import Any, cast + +import cvat_sdk.auto_annotation as cvataa +import cvat_sdk.datasets as cvatds +import PIL.Image +from cvat_sdk import models +from cvat_sdk.auto_annotation.driver import ( + _AnnotationMapper, + _DetectionFunctionContextImpl, + _SpecNameMapping, +) + +from .agent_driver import ( + AgentFunctionDriver, + CheckInCallback, + IncompatibleFunctionError, + worker_current_function, +) + + +def _worker_job_detect( + context: _DetectionFunctionContextImpl, image: PIL.Image.Image +) -> Sequence[cvataa.DetectionAnnotation]: + current_function = cast(cvataa.DetectionFunction, worker_current_function()) + return current_function.detect(context, image) + + +class AgentDetectionFunctionDriver(AgentFunctionDriver[cvataa.DetectionFunctionSpec]): + FUNCTION_KIND = "detector" + + @staticmethod + def _dump_sublabel_spec( + sl_spec: models.SublabelRequest | models.PatchedLabelRequest, + ) -> dict: + result = { + "name": sl_spec.name, + "attributes": [ + { + "name": attribute_spec.name, + "input_type": attribute_spec.input_type, + "values": attribute_spec.values, + } + for attribute_spec in getattr(sl_spec, "attributes", []) + ], + } + + if getattr(sl_spec, "type", "any") != "any": + # Add the type conditionally, to stay compatible with older + # CVAT versions when the function doesn't define label types. + result["type"] = sl_spec.type + + return result + + @classmethod + def get_remote_function_fields(cls, spec: cvataa.DetectionFunctionSpec) -> dict[str, Any]: + labels_v2 = [] + + for label_spec in spec.labels: + labels_v2.append(cls._dump_sublabel_spec(label_spec)) + + if sublabels := getattr(label_spec, "sublabels", None): + labels_v2[-1]["sublabels"] = [ + cls._dump_sublabel_spec(sublabel) for sublabel in sublabels + ] + + return {"labels_v2": labels_v2} + + def _validate_sublabel_compatibility( + self, remote_sl: dict, sl: models.Sublabel | None, sl_desc: str + ): + if not sl: + raise IncompatibleFunctionError(f"{sl_desc} is not supported.") + + if remote_sl["type"] not in {"any", "unknown"} and remote_sl["type"] != sl.type: + raise IncompatibleFunctionError( + f"{sl_desc} has type {remote_sl['type']!r}, " + f"but the function object declares type {sl.type!r}." + ) + + attrs_by_name = {attr.name: attr for attr in getattr(sl, "attributes", [])} + + for remote_attr in remote_sl["attributes"]: + attr_desc = f"attribute {remote_attr['name']!r} of {sl_desc}" + attr = attrs_by_name.get(remote_attr["name"]) + + if not attr: + raise IncompatibleFunctionError(f"{attr_desc} is not supported.") + + if remote_attr["input_type"] != attr.input_type.value: + raise IncompatibleFunctionError( + f"{attr_desc} has input type {remote_attr['input_type']!r}," + f" but the function object declares input type {attr.input_type.value!r}." + ) + + if remote_attr["values"] != attr.values: + raise IncompatibleFunctionError( + f"{attr_desc} has values {remote_attr['values']!r}," + f" but the function object declares values {attr.values!r}." + ) + + def validate_function_compatibility(self, remote_function: dict) -> None: + labels_by_name = {label.name: label for label in self._function_spec.labels} + + for remote_label in remote_function["labels_v2"]: + label_desc = f"label {remote_label['name']!r}" + label = labels_by_name.get(remote_label["name"]) + + self._validate_sublabel_compatibility(remote_label, label, label_desc) + + sublabels_by_name = {sl.name: sl for sl in getattr(label, "sublabels", [])} + + for remote_sl in remote_label.get("sublabels", []): + sl_desc = f"sublabel {remote_sl['name']!r} of {label_desc}" + sl = sublabels_by_name.get(remote_sl["name"]) + + self._validate_sublabel_compatibility(remote_sl, sl, sl_desc) + + def _create_annotation_mapper_for_detection_ar( + self, ar_params: dict, ds_labels: Sequence[models.ILabel] + ) -> _AnnotationMapper: + spec_nm = _SpecNameMapping.from_api( + { + k: models.LabelMappingEntryRequest._from_openapi_data(**v) + for k, v in ar_params["mapping"].items() + } + ) + + return _AnnotationMapper( + self._client.logger, + self._function_spec.labels, + ds_labels, + allow_unmatched_labels=False, + spec_nm=spec_nm, + conv_mask_to_poly=ar_params["conv_mask_to_poly"], + ) + + def _create_detection_function_context( + self, ar_params: dict, frame_name: str + ) -> cvataa.DetectionFunctionContext: + return _DetectionFunctionContextImpl( + frame_name=frame_name, + conf_threshold=ar_params["threshold"], + conv_mask_to_poly=ar_params["conv_mask_to_poly"], + ) + + def _calculate_result_for_annotate_task_ar( + self, ar_params: dict[str, Any], check_in: CheckInCallback + ) -> dict[str, Any]: + ds = cvatds.TaskDataset( + self._client, + ar_params["task"], + load_annotations=False, + media_download_policy=cvatds.MediaDownloadPolicy.FETCH_CHUNKS_ON_DEMAND, + ) + + # Fetching the dataset might take a while, so check in to let the server + # know we're still alive. + check_in(current_progress=0) + + mapper = self._create_annotation_mapper_for_detection_ar(ar_params, ds.labels) + + all_annotations = models.PatchedLabeledDataRequest(tags=[], shapes=[]) + + with ds.iter_samples(temporary_chunks=True) as samples: + for sample_index, sample in enumerate(samples): + context = self._create_detection_function_context(ar_params, sample.frame_name) + annotations = self._executor.result( + self._executor.submit( + _worker_job_detect, context, self._load_image_for_ar(sample, ar_params) + ) + ) + + tags, shapes = mapper.validate_and_remap(annotations, sample.frame_index) + all_annotations.tags.extend(tags) + all_annotations.shapes.extend(shapes) + + check_in(current_progress=(sample_index + 1) / len(ds.samples)) + + return {"annotations": all_annotations} + + def _calculate_result_for_annotate_frame_ar( + self, ar_params: dict[str, Any], check_in: object + ) -> dict[str, Any]: + sample, ds_labels = self._get_sample_from_ar_params(ar_params) + + mapper = self._create_annotation_mapper_for_detection_ar(ar_params, ds_labels) + + context = self._create_detection_function_context(ar_params, sample.frame_name) + + annotations = self._executor.result( + self._executor.submit( + _worker_job_detect, context, self._load_image_for_ar(sample, ar_params) + ) + ) + + tags, shapes = mapper.validate_and_remap(annotations, sample.frame_index) + return {"annotations": models.PatchedLabeledDataRequest(tags=tags, shapes=shapes)} + + _CALCULATE_RESULT_PER_AR_TYPE = { + "annotate_task": _calculate_result_for_annotate_task_ar, + "annotate_frame": _calculate_result_for_annotate_frame_ar, + } diff --git a/cvat-cli/src/cvat_cli/_internal/agent_driver_tracking.py b/cvat-cli/src/cvat_cli/_internal/agent_driver_tracking.py new file mode 100644 index 0000000..d9ca9d5 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/agent_driver_tracking.py @@ -0,0 +1,217 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from collections import OrderedDict +from datetime import datetime, timedelta, timezone +from typing import Any, Callable, TypeAlias, cast + +import attrs +import cvat_sdk.auto_annotation as cvataa +import PIL.Image + +from .agent_driver import ( + AgentFunctionDriver, + BadArError, + IncompatibleFunctionError, + worker_current_function, +) + +_MAX_AGE_OF_TRACKING_STATE = timedelta(hours=8) + + +@attrs.define +class _ExtendedTrackingState: + inner_state: Any # the state produced by the AA function + original_shape_type: str + original_task_id: int + original_image_dims: tuple[int, int] + last_accessed_at: datetime = attrs.field(factory=lambda: datetime.now(tz=timezone.utc)) + + +class _TrackingStateContainer: + def __init__(self): + self._id_to_ext_state: OrderedDict[str, _ExtendedTrackingState] = OrderedDict() + + def store(self, state: Any, shape_type: str, task_id: int, image_dims: tuple[int, int]) -> str: + state_id = _tracking_state_id_generator() + self._id_to_ext_state[state_id] = _ExtendedTrackingState( + inner_state=state, + original_shape_type=shape_type, + original_task_id=task_id, + original_image_dims=image_dims, + ) + return state_id + + def retrieve(self, state_id: str, task_id: int, image_dims: tuple[int, int]) -> Any: + ext_state = self._id_to_ext_state.get(state_id) + + if not ext_state: + raise BadArError(f"Tracking state {state_id!r} not found - possibly expired") + + if ext_state.original_task_id != task_id: + # This is a defense-in-depth measure. State IDs are supposed to be unguessable, + # but even if an attacker manages to obtain one, they will not be able to use it + # to get any information about a task they don't have access to. + raise BadArError(f"Tracking state {state_id!r} is not for task #{task_id}") + + if image_dims != ext_state.original_image_dims: + raise BadArError("Image sizes of the start frame and the current frame are different") + + ext_state.last_accessed_at = datetime.now(tz=timezone.utc) + self._id_to_ext_state.move_to_end(state_id) + + return ext_state.inner_state, ext_state.original_shape_type + + def prune(self) -> None: + cutoff = datetime.now(tz=timezone.utc) - _MAX_AGE_OF_TRACKING_STATE + + while ( + self._id_to_ext_state + and next(iter(self._id_to_ext_state.values())).last_accessed_at < cutoff + ): + self._id_to_ext_state.popitem(last=False) + + +TrackingStateIdGenerator: TypeAlias = Callable[[], str] + + +_tracking_states: _TrackingStateContainer +_tracking_state_id_generator: TrackingStateIdGenerator + + +def _worker_job_init_tracking( + task_id: int, + image: PIL.Image.Image, + shapes: list[cvataa.TrackableShape], +) -> list[str]: + _tracking_states.prune() + + current_function = cast(cvataa.TrackingFunction, worker_current_function()) + + if hasattr(current_function, "preprocess_image"): + pp_image = current_function.preprocess_image(_TrackingFunctionContextImpl(), image) + else: + pp_image = image + + return [ + _tracking_states.store( + state=current_function.init_tracking_state( + _TrackingFunctionShapeContextImpl(original_shape_type=shape.type), pp_image, shape + ), + shape_type=shape.type, + task_id=task_id, + image_dims=image.size, + ) + for shape in shapes + ] + + +def _worker_job_track( + task_id: int, image: PIL.Image.Image, states: list[str] +) -> list[cvataa.TrackableShape | None]: + _tracking_states.prune() + + current_function = cast(cvataa.TrackingFunction, worker_current_function()) + + pp_image = current_function.preprocess_image(_TrackingFunctionContextImpl(), image) + + def track(state_id): + inner_state, original_shape_type = _tracking_states.retrieve( + state_id=state_id, task_id=task_id, image_dims=image.size + ) + + output_shape = current_function.track( + _TrackingFunctionShapeContextImpl(original_shape_type=original_shape_type), + pp_image, + inner_state, + ) + + if output_shape and output_shape.type != original_shape_type: + raise cvataa.BadFunctionError( + f"function output shape of type {output_shape.type!r}, " + f"but original shape was of type {original_shape_type!r}" + ) + return output_shape + + return list(map(track, states)) + + +class _TrackingFunctionContextImpl(cvataa.TrackingFunctionContext): + pass + + +@attrs.frozen(kw_only=True) +class _TrackingFunctionShapeContextImpl(cvataa.TrackingFunctionShapeContext): + original_shape_type: str + + +class AgentTrackingFunctionDriver(AgentFunctionDriver[cvataa.TrackingFunctionSpec]): + FUNCTION_KIND = "tracker" + + @classmethod + def init_worker(cls, state_id_generator: TrackingStateIdGenerator) -> None: + global _tracking_states + _tracking_states = _TrackingStateContainer() + + global _tracking_state_id_generator + _tracking_state_id_generator = state_id_generator + + @classmethod + def get_remote_function_fields(cls, spec: cvataa.TrackingFunctionSpec) -> dict[str, Any]: + return {"supported_shape_types": sorted(spec.supported_shape_types)} + + def validate_function_compatibility(self, remote_function: dict) -> None: + remote_supported_shape_types = frozenset(remote_function["supported_shape_types"]) + unsupported = remote_supported_shape_types - self._function_spec.supported_shape_types + + if unsupported: + raise IncompatibleFunctionError( + "the function object does not support the following shape types: " + + ", ".join(map(repr, unsupported)) + ) + + def _calculate_result_for_init_tracking_ar( + self, ar_params: dict[str, Any], check_in: object + ) -> dict[str, Any]: + sample, _ = self._get_sample_from_ar_params(ar_params) + + def convert_shape(shape: dict) -> cvataa.TrackableShape: + if shape["type"] not in self._function_spec.supported_shape_types: + raise BadArError(f"Unsupported shape type {shape['type']!r}") + return cvataa.TrackableShape(type=shape["type"], points=shape["points"]) + + shapes = list(map(convert_shape, ar_params["shapes"])) + + states = self._executor.result( + self._executor.submit( + _worker_job_init_tracking, + ar_params["task"], + sample.media.load_image(), + shapes, + ) + ) + + return {"states": states} + + def _calculate_result_for_track_ar( + self, ar_params: dict[str, Any], check_in: object + ) -> dict[str, Any]: + sample, _ = self._get_sample_from_ar_params(ar_params) + + states = ar_params["states"] + shapes = self._executor.result( + self._executor.submit( + _worker_job_track, ar_params["task"], sample.media.load_image(), states + ) + ) + + return { + "states": states, + "shapes": [attrs.asdict(shape) if shape else None for shape in shapes], + } + + _CALCULATE_RESULT_PER_AR_TYPE = { + "init_tracking": _calculate_result_for_init_tracking_ar, + "track": _calculate_result_for_track_ar, + } diff --git a/cvat-cli/src/cvat_cli/_internal/command_base.py b/cvat-cli/src/cvat_cli/_internal/command_base.py new file mode 100644 index 0000000..c1c209f --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/command_base.py @@ -0,0 +1,302 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import json +import os +import textwrap +import types +from abc import ABCMeta, abstractmethod +from collections.abc import Callable, Mapping, Sequence +from typing import Protocol + +from attr.converters import to_bool +from cvat_sdk import Client +from cvat_sdk.core.helpers import DeferredTqdmProgressReporter +from cvat_sdk.core.proxies.types import Location + + +class Command(Protocol): + @property + def description(self) -> str: ... + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: ... + + # The exact parameters accepted by `execute` vary between commands, + # so we're forced to declare it like this instead of as a method. + @property + def execute(self) -> Callable[..., None]: ... + + +class CommandGroup: + def __init__(self, *, description: str) -> None: + self._commands: dict[str, Command] = {} + self.description = description + + def command_class(self, name: str): + def decorator(cls: type): + self._commands[name] = cls() + return cls + + return decorator + + def add_command(self, name: str, command: Command) -> None: + self._commands[name] = command + + @property + def commands(self) -> Mapping[str, Command]: + return types.MappingProxyType(self._commands) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + subparsers = parser.add_subparsers(required=True) + + for name, command in self._commands.items(): + subparser = subparsers.add_parser(name, description=command.description) + subparser.set_defaults(_executor=command.execute) + command.configure_parser(subparser) + + def execute(self) -> None: + # It should be impossible for a command group to be executed, + # because configure_parser requires that a subcommand is specified. + assert False, "unreachable code" + + +class DeprecatedAlias: + def __init__(self, command: Command, replacement: str) -> None: + self._command = command + self._replacement = replacement + + @property + def description(self) -> str: + return textwrap.dedent(f"""\ + {self._command.description} + (Deprecated; use "{self._replacement}" instead.) + """) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + self._command.configure_parser(parser) + + def execute(self, client: Client, **kwargs) -> None: + client.logger.warning('This command is deprecated. Use "%s" instead.', self._replacement) + self._command.execute(client, **kwargs) + + +class GenericCommand(metaclass=ABCMeta): + @abstractmethod + def repo(self, client: Client): ... + + @property + @abstractmethod + def resource_type_str(self) -> str: ... + + +class GenericListCommand(GenericCommand): + @property + def description(self) -> str: + return f"List all CVAT {self.resource_type_str}s in either basic or JSON format." + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--json", + dest="use_json_output", + default=False, + action="store_true", + help="output JSON data", + ) + + def execute(self, client: Client, *, use_json_output: bool = False): + results = self.repo(client).list(return_json=use_json_output) + if use_json_output: + print(json.dumps(json.loads(results), indent=2)) + else: + for r in results: + print(r.id) + + +class GenericDeleteCommand(GenericCommand): + @property + def description(self): + return f"Delete a list of {self.resource_type_str}s, ignoring those which don't exist." + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "ids", type=int, help=f"list of {self.resource_type_str} IDs", nargs="+" + ) + + def execute(self, client: Client, *, ids: Sequence[int]) -> None: + self.repo(client).remove_by_ids(ids) + + +class GenericDownloadBackupCommand(GenericCommand): + @property + def description(self) -> str: + return f"Download a {self.resource_type_str} backup." + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "resource_id", + metavar=f"{self.resource_type_str}_id", + type=int, + help=f"{self.resource_type_str} ID", + ) + parser.add_argument( + "filename", + type=str, + nargs="?", + default="", + help="output file or directory (default: current directory)", + ) + parser.add_argument( + "--completion_verification_period", + dest="status_check_period", + default=2, + type=float, + help="time interval between checks if archive building has been finished, in seconds", + ) + + def execute( + self, client: Client, *, resource_id: int, filename: str, status_check_period: float + ) -> None: + if not filename: + filename = os.getcwd() + + if filename.endswith((os.sep, os.altsep or os.sep)): + os.makedirs(filename, exist_ok=True) + + self.repo(client).retrieve(obj_id=resource_id).download_backup( + filename=filename, + status_check_period=status_check_period, + pbar=DeferredTqdmProgressReporter(), + location=Location.LOCAL, + ) + + +class GenericCreateFromBackupCommand(GenericCommand): + @property + def description(self) -> str: + return f"Create a {self.resource_type_str} from a backup file." + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("filename", type=str, help="upload file") + parser.add_argument( + "--completion_verification_period", + dest="status_check_period", + default=2, + type=float, + help="time interval between checks if archive processing was finished, in seconds", + ) + + def execute(self, client: Client, *, filename: str, status_check_period: float) -> None: + resource = self.repo(client).create_from_backup( + filename=filename, + status_check_period=status_check_period, + pbar=DeferredTqdmProgressReporter(), + ) + print(resource.id) + + +class GenericExportDatasetCommand(GenericCommand): + @property + def description(self) -> str: + return textwrap.dedent(f"""\ + Export a {self.resource_type_str} as a dataset in the specified format + (e.g. 'YOLO 1.1'). + """) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "resource_id", + metavar=f"{self.resource_type_str}_id", + type=int, + help=f"{self.resource_type_str} ID", + ) + parser.add_argument( + "filename", + type=str, + nargs="?", + default="", + help="output file or directory (default: current directory)", + ) + parser.add_argument( + "--format", + dest="fileformat", + type=str, + default="CVAT for images 1.1", + help="annotation format (default: %(default)s)", + ) + parser.add_argument( + "--completion_verification_period", + dest="status_check_period", + default=2, + type=float, + help="number of seconds to wait until checking if dataset building finished", + ) + parser.add_argument( + "--with-images", + type=to_bool, + default=False, + dest="include_images", + help="Whether to include images or not (default: %(default)s)", + ) + + def execute( + self, + client: Client, + *, + fileformat: str, + filename: str, + status_check_period: int, + include_images: bool, + resource_id: int, + ) -> None: + if not filename: + filename = os.getcwd() + + if filename.endswith((os.sep, os.altsep or os.sep)): + os.makedirs(filename, exist_ok=True) + + self.repo(client).retrieve(obj_id=resource_id).export_dataset( + format_name=fileformat, + filename=filename, + pbar=DeferredTqdmProgressReporter(), + status_check_period=status_check_period, + include_images=include_images, + location=Location.LOCAL, + ) + + +class GenericImportDatasetCommand(GenericCommand): + import_method_name = "import_dataset" + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "resource_id", + metavar=f"{self.resource_type_str}_id", + type=int, + help=f"{self.resource_type_str} ID", + ) + parser.add_argument("filename", type=str, help="upload file") + parser.add_argument( + "--format", + dest="fileformat", + type=str, + default="CVAT 1.1", + help="annotation format (default: %(default)s)", + ) + + def execute( + self, + client: Client, + *, + fileformat: str, + filename: str, + resource_id: int, + ) -> None: + resource = self.repo(client).retrieve(obj_id=resource_id) + import_dataset = getattr(resource, self.import_method_name) + import_dataset( + format_name=fileformat, + filename=filename, + pbar=DeferredTqdmProgressReporter(), + ) diff --git a/cvat-cli/src/cvat_cli/_internal/commands_all.py b/cvat-cli/src/cvat_cli/_internal/commands_all.py new file mode 100644 index 0000000..aa15a62 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/commands_all.py @@ -0,0 +1,29 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from .command_base import CommandGroup, DeprecatedAlias +from .commands_functions import COMMANDS as COMMANDS_FUNCTIONS +from .commands_projects import COMMANDS as COMMANDS_PROJECTS +from .commands_tasks import COMMANDS as COMMANDS_TASKS + +COMMANDS = CommandGroup(description="Perform operations on CVAT resources.") + +COMMANDS.add_command("function", COMMANDS_FUNCTIONS) +COMMANDS.add_command("project", COMMANDS_PROJECTS) +COMMANDS.add_command("task", COMMANDS_TASKS) + +_legacy_mapping = { + "create": "create", + "ls": "ls", + "delete": "delete", + "frames": "frames", + "dump": "export-dataset", + "upload": "import-dataset", + "export": "backup", + "import": "create-from-backup", + "auto-annotate": "auto-annotate", +} + +for _legacy, _new in _legacy_mapping.items(): + COMMANDS.add_command(_legacy, DeprecatedAlias(COMMANDS_TASKS.commands[_new], f"task {_new}")) diff --git a/cvat-cli/src/cvat_cli/_internal/commands_functions.py b/cvat-cli/src/cvat_cli/_internal/commands_functions.py new file mode 100644 index 0000000..ea10ae5 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/commands_functions.py @@ -0,0 +1,127 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import json +import textwrap +from collections.abc import Sequence +from typing import Any + +from cvat_sdk import Client + +from .agent import FUNCTION_PROVIDER_NATIVE, get_function_driver_class, run_agent +from .command_base import CommandGroup +from .common import FunctionLoader, configure_function_implementation_arguments + +COMMANDS = CommandGroup(description="Perform operations on CVAT lambda functions.") + + +@COMMANDS.command_class("create-native") +class FunctionCreateNative: + description = textwrap.dedent("""\ + Create a CVAT function that can be powered by an agent running the given local function. + """) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "name", + help="a human-readable name for the function", + ) + parser.add_argument( + "--visibility", + choices=("private", "public"), + default="private", + help="visibility setting for the function", + ) + + configure_function_implementation_arguments(parser) + + def execute( + self, + client: Client, + *, + name: str, + visibility: str, + function_loader: FunctionLoader, + ) -> None: + function = function_loader.load() + driver_class = get_function_driver_class(function.spec) + + remote_function: dict[str, Any] = { + "provider": FUNCTION_PROVIDER_NATIVE, + "name": name, + "visibility": visibility, + "kind": driver_class.FUNCTION_KIND, + **driver_class.get_remote_function_fields(function.spec), + } + + _, response = client.api_client.call_api( + "/api/functions", + "POST", + body=remote_function, + ) + + remote_function = json.loads(response.data) + + client.logger.info( + "Created function #%d: %s", remote_function["id"], remote_function["name"] + ) + print(remote_function["id"]) + + +@COMMANDS.command_class("delete") +class FunctionDelete: + description = "Delete a list of functions, ignoring those which don't exist." + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("function_ids", type=int, help="IDs of functions to delete", nargs="+") + + def execute(self, client: Client, *, function_ids: Sequence[int]) -> None: + for function_id in function_ids: + _, response = client.api_client.call_api( + "/api/functions/{function_id}", + "DELETE", + path_params={"function_id": function_id}, + _check_status=False, + ) + + if 200 <= response.status <= 299: + client.logger.info(f"Function #{function_id} deleted") + elif response.status == 404: + client.logger.warning(f"Function #{function_id} not found") + else: + client.logger.error( + f"Failed to delete function #{function_id}: " + f"{response.msg} (status {response.status})" + ) + + +@COMMANDS.command_class("run-agent") +class FunctionRunAgent: + description = "Process requests for a given native function, indefinitely." + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "function_id", + type=int, + help="ID of the function to process requests for", + ) + + configure_function_implementation_arguments(parser) + + parser.add_argument( + "--burst", + action="store_true", + help="process all pending requests and then exit", + ) + + def execute( + self, + client: Client, + *, + function_id: int, + function_loader: FunctionLoader, + burst: bool, + ) -> None: + run_agent(client, function_loader, function_id, burst=burst) diff --git a/cvat-cli/src/cvat_cli/_internal/commands_projects.py b/cvat-cli/src/cvat_cli/_internal/commands_projects.py new file mode 100644 index 0000000..88b7a51 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/commands_projects.py @@ -0,0 +1,121 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import textwrap + +from cvat_sdk import Client, models + +from .command_base import ( + CommandGroup, + GenericCommand, + GenericCreateFromBackupCommand, + GenericDeleteCommand, + GenericDownloadBackupCommand, + GenericExportDatasetCommand, + GenericImportDatasetCommand, + GenericListCommand, +) +from .parsers import parse_label_arg + +COMMANDS = CommandGroup(description="Perform operations on CVAT projects.") + + +class GenericProjectCommand(GenericCommand): + resource_type_str = "project" + + def repo(self, client: Client): + return client.projects + + +@COMMANDS.command_class("ls") +class ProjectList(GenericListCommand, GenericProjectCommand): + pass + + +@COMMANDS.command_class("create") +class ProjectCreate: + description = textwrap.dedent("""\ + Create a new CVAT project, optionally importing a dataset. + """) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("name", type=str, help="name of the project") + parser.add_argument( + "--bug_tracker", "--bug", default=argparse.SUPPRESS, type=str, help="bug tracker URL" + ) + parser.add_argument( + "--labels", + default=[], + type=parse_label_arg, + help="string or file containing JSON labels specification (default: %(default)s)", + ) + parser.add_argument( + "--dataset_path", + default="", + type=str, + help="path to the dataset file to import", + ) + parser.add_argument( + "--dataset_format", + default="CVAT 1.1", + type=str, + help="format of the dataset file being uploaded" + " (only applies when --dataset_path is specified; default: %(default)s)", + ) + parser.add_argument( + "--completion_verification_period", + dest="status_check_period", + default=2, + type=float, + help="period between status checks" + " (only applies when --dataset_path is specified; default: %(default)s)", + ) + + def execute( + self, + client: Client, + *, + name: str, + labels: dict, + dataset_path: str, + dataset_format: str, + status_check_period: int, + **kwargs, + ) -> None: + project = client.projects.create_from_dataset( + spec=models.ProjectWriteRequest(name=name, labels=labels, **kwargs), + dataset_path=dataset_path, + dataset_format=dataset_format, + status_check_period=status_check_period, + ) + print(project.id) + + +@COMMANDS.command_class("delete") +class ProjectDelete(GenericDeleteCommand, GenericProjectCommand): + pass + + +@COMMANDS.command_class("backup") +class ProjectBackup(GenericDownloadBackupCommand, GenericProjectCommand): + pass + + +@COMMANDS.command_class("create-from-backup") +class ProjectCreateFromBackup(GenericCreateFromBackupCommand, GenericProjectCommand): + pass + + +@COMMANDS.command_class("export-dataset") +class ProjectExportDataset(GenericExportDatasetCommand, GenericProjectCommand): + pass + + +@COMMANDS.command_class("import-dataset") +class ProjectImportDataset(GenericImportDatasetCommand, GenericProjectCommand): + description = textwrap.dedent("""\ + Create tasks in a project from a dataset in the specified format + (e.g. 'YOLO 1.1'), including images and annotations. + """) diff --git a/cvat-cli/src/cvat_cli/_internal/commands_tasks.py b/cvat-cli/src/cvat_cli/_internal/commands_tasks.py new file mode 100644 index 0000000..1874173 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/commands_tasks.py @@ -0,0 +1,342 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import argparse +import textwrap +from collections.abc import Sequence + +import cvat_sdk.auto_annotation as cvataa +from cvat_sdk import Client, models +from cvat_sdk.core.helpers import DeferredTqdmProgressReporter +from cvat_sdk.core.proxies.tasks import ResourceType + +from .command_base import ( + CommandGroup, + GenericCommand, + GenericCreateFromBackupCommand, + GenericDeleteCommand, + GenericDownloadBackupCommand, + GenericExportDatasetCommand, + GenericImportDatasetCommand, + GenericListCommand, +) +from .common import FunctionLoader, configure_function_implementation_arguments +from .parsers import parse_label_arg, parse_resource_type, parse_threshold + +COMMANDS = CommandGroup(description="Perform operations on CVAT tasks.") + + +class GenericTaskCommand(GenericCommand): + resource_type_str = "task" + + def repo(self, client: Client): + return client.tasks + + +@COMMANDS.command_class("ls") +class TaskList(GenericListCommand, GenericTaskCommand): + pass + + +@COMMANDS.command_class("create") +class TaskCreate: + description = textwrap.dedent("""\ + Create a new CVAT task. To create a task, you need + to specify labels using the --labels argument or + attach the task to an existing project using the + --project_id argument. + """) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("name", type=str, help="name of the task") + parser.add_argument( + "resource_type", + default="local", + choices=list(ResourceType), + type=parse_resource_type, + help="type of files specified", + ) + parser.add_argument("resources", type=str, help="list of paths or URLs", nargs="+") + parser.add_argument( + "--annotation_path", default="", type=str, help="path to annotation file" + ) + parser.add_argument( + "--annotation_format", + default="CVAT 1.1", + type=str, + help="format of the annotation file being uploaded, e.g. CVAT 1.1", + ) + parser.add_argument( + "--bug_tracker", "--bug", default=argparse.SUPPRESS, type=str, help="bug tracker URL" + ) + parser.add_argument( + "--chunk_size", + default=argparse.SUPPRESS, + type=int, + help="the number of frames per chunk", + ) + parser.add_argument( + "--completion_verification_period", + dest="status_check_period", + default=2, + type=float, + help=textwrap.dedent("""\ + number of seconds to wait until checking + if data compression finished (necessary before uploading annotations) + """), + ) + parser.add_argument( + "--copy_data", + default=False, + action="store_true", + help=textwrap.dedent("""\ + set the option to copy the data, only used when resource type is + share (default: %(default)s) + """), + ) + parser.add_argument( + "--frame_step", + default=argparse.SUPPRESS, + type=int, + help=textwrap.dedent("""\ + set the frame step option in the advanced configuration + when uploading image series or videos + """), + ) + parser.add_argument( + "--image_quality", + default=70, + type=int, + help=textwrap.dedent("""\ + set the image quality option in the advanced configuration + when creating tasks.(default: %(default)s) + """), + ) + parser.add_argument( + "--labels", + default="[]", + type=parse_label_arg, + help="string or file containing JSON labels specification", + ) + parser.add_argument( + "--project_id", default=argparse.SUPPRESS, type=int, help="project ID if project exists" + ) + parser.add_argument( + "--overlap", + default=argparse.SUPPRESS, + type=int, + help="the number of intersected frames between different segments", + ) + parser.add_argument( + "--segment_size", + default=argparse.SUPPRESS, + type=int, + help="the number of frames in a segment", + ) + parser.add_argument( + "--sorting-method", + default="lexicographical", + choices=["lexicographical", "natural", "predefined", "random"], + help="""data sorting method (default: %(default)s)""", + ) + parser.add_argument( + "--start_frame", + default=argparse.SUPPRESS, + type=int, + help="the start frame of the video", + ) + parser.add_argument( + "--stop_frame", default=argparse.SUPPRESS, type=int, help="the stop frame of the video" + ) + parser.add_argument( + "--use_cache", + action="store_true", + help="""use cache""", # automatically sets default=False + ) + parser.add_argument( + "--use_zip_chunks", + action="store_true", # automatically sets default=False + help="""zip chunks before sending them to the server""", + ) + parser.add_argument( + "--cloud_storage_id", + default=argparse.SUPPRESS, + type=int, + help="cloud storage ID if you would like to use data from cloud storage", + ) + parser.add_argument( + "--filename_pattern", + default=argparse.SUPPRESS, + type=str, + help=textwrap.dedent("""\ + pattern for filtering data from the manifest file for the upload. + Only shell-style wildcards are supported: + * - matches everything; + ? - matches any single character; + [seq] - matches any character in 'seq'; + [!seq] - matches any character not in seq + """), + ) + + def execute( + self, + client, + *, + name: str, + labels: list[dict[str, str]], + resources: Sequence[str], + resource_type: ResourceType, + annotation_path: str, + annotation_format: str, + status_check_period: int, + **kwargs, + ) -> None: + task_params = {} + data_params = {} + + for k, v in kwargs.items(): + if k in models.DataRequest.attribute_map or k == "frame_step": + data_params[k] = v + else: + task_params[k] = v + + task = client.tasks.create_from_data( + spec=models.TaskWriteRequest(name=name, labels=labels, **task_params), + resource_type=resource_type, + resources=resources, + data_params=data_params, + annotation_path=annotation_path, + annotation_format=annotation_format, + status_check_period=status_check_period, + pbar=DeferredTqdmProgressReporter(), + ) + print(task.id) + + +@COMMANDS.command_class("delete") +class TaskDelete(GenericDeleteCommand, GenericTaskCommand): + pass + + +@COMMANDS.command_class("frames") +class TaskFrames: + description = textwrap.dedent("""\ + Download the requested frame numbers for a task and save images as + task__frame_.jpg. + """) + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("task_id", type=int, help="task ID") + parser.add_argument("frame_ids", type=int, help="list of frame IDs to download", nargs="+") + parser.add_argument( + "--outdir", type=str, default="", help="directory to save images (default: CWD)" + ) + parser.add_argument( + "--quality", + type=str, + choices=("original", "compressed"), + default="original", + help="choose quality of images (default: %(default)s)", + ) + + def execute( + self, + client: Client, + *, + task_id: int, + frame_ids: Sequence[int], + outdir: str, + quality: str, + ) -> None: + client.tasks.retrieve(obj_id=task_id).download_frames( + frame_ids=frame_ids, + outdir=outdir, + quality=quality, + filename_pattern=f"task_{task_id}" + "_frame_{frame_id:06d}{frame_ext}", + ) + + +@COMMANDS.command_class("export-dataset") +class TaskExportDataset(GenericExportDatasetCommand, GenericTaskCommand): + pass + + +@COMMANDS.command_class("import-dataset") +class TaskImportDataset(GenericImportDatasetCommand, GenericTaskCommand): + description = textwrap.dedent("""\ + Import annotations into a task from a dataset in the specified format + (e.g. 'YOLO 1.1'). + """) + import_method_name = "import_annotations" + + +@COMMANDS.command_class("backup") +class TaskBackup(GenericDownloadBackupCommand, GenericTaskCommand): + pass + + +@COMMANDS.command_class("create-from-backup") +class TaskCreateFromBackup(GenericCreateFromBackupCommand, GenericTaskCommand): + pass + + +@COMMANDS.command_class("auto-annotate") +class TaskAutoAnnotate: + description = "Automatically annotate a CVAT task by running a function on the local machine." + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("task_id", type=int, help="task ID") + + configure_function_implementation_arguments(parser) + + parser.add_argument( + "--clear-existing", + action="store_true", + help="Remove existing annotations from the task", + ) + + parser.add_argument( + "--allow-unmatched-labels", + action="store_true", + help="Allow the function to declare labels/sublabels/attributes not configured in the task", + ) + + parser.add_argument( + "--conf-threshold", + type=parse_threshold, + help="Confidence threshold for filtering detections", + default=None, + ) + + parser.add_argument( + "--conv-mask-to-poly", + action="store_true", + help="Convert mask shapes to polygon shapes", + ) + + def execute( + self, + client: Client, + *, + task_id: int, + function_loader: FunctionLoader, + clear_existing: bool = False, + allow_unmatched_labels: bool = False, + conf_threshold: float | None, + conv_mask_to_poly: bool, + ) -> None: + function = function_loader.load() + + cvataa.annotate_task( + client, + task_id, + function, + pbar=DeferredTqdmProgressReporter(), + clear_existing=clear_existing, + allow_unmatched_labels=allow_unmatched_labels, + conf_threshold=conf_threshold, + conv_mask_to_poly=conv_mask_to_poly, + ) diff --git a/cvat-cli/src/cvat_cli/_internal/common.py b/cvat-cli/src/cvat_cli/_internal/common.py new file mode 100644 index 0000000..781bcd0 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/common.py @@ -0,0 +1,148 @@ +# Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import importlib +import importlib.util +import logging +import sys +from http.client import HTTPConnection +from pathlib import Path +from typing import Any + +import attrs +import cvat_sdk.auto_annotation as cvataa +from cvat_sdk.core.auth import ( + ClientAuthParameters, + configure_client_auth_arguments, + make_client_from_cli, +) +from cvat_sdk.core.client import Client +from cvat_sdk.core.exceptions import AuthStoreError + +from ..version import VERSION +from .parsers import BuildDictAction, parse_function_parameter +from .utils import popattr + + +class CriticalError(Exception): + pass + + +def configure_common_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--version", action="version", version=VERSION) + configure_client_auth_arguments(parser) + parser.add_argument( + "--debug", + action="store_const", + dest="loglevel", + const=logging.DEBUG, + default=logging.INFO, + help="show debug output", + ) + + +def configure_logger(logger: logging.Logger, parsed_args: argparse.Namespace) -> None: + level = popattr(parsed_args, "loglevel") + formatter = logging.Formatter( + "[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S", style="%" + ) + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(level) + if level <= logging.DEBUG: + HTTPConnection.debuglevel = 1 + + +def build_client(parsed_args: argparse.Namespace, logger: logging.Logger) -> Client: + auth_args = ClientAuthParameters.from_namespace(parsed_args) + for field in attrs.fields(ClientAuthParameters): + popattr(parsed_args, field.name) + + try: + client = make_client_from_cli(auth_args, logger=logger) + except AuthStoreError as e: + raise CriticalError(str(e)) from e + + client.check_server_version(fail_if_unsupported=False) + return client + + +def configure_function_implementation_arguments(parser: argparse.ArgumentParser) -> None: + function_group = parser.add_mutually_exclusive_group(required=True) + + function_group.add_argument( + "--function-module", + metavar="MODULE", + help="qualified name of a module to use as the function", + ) + + function_group.add_argument( + "--function-file", + metavar="PATH", + type=Path, + help="path to a Python source file to use as the function", + ) + + parser.add_argument( + "--function-parameter", + "-p", + metavar="NAME=TYPE:VALUE", + type=parse_function_parameter, + action=BuildDictAction, + dest="function_parameters", + help="parameter for the function", + ) + + original_executor = parser.get_default("_executor") + + def execute_with_function_loader( + client, + *, + function_module: str | None, + function_file: Path | None, + function_parameters: dict[str, Any], + **kwargs, + ): + original_executor( + client, + function_loader=FunctionLoader(function_module, function_file, function_parameters), + **kwargs, + ) + + parser.set_defaults(_executor=execute_with_function_loader) + + +@attrs.frozen +class FunctionLoader: + function_module: str | None + function_file: Path | None + function_parameters: dict[str, Any] + + def __attrs_post_init__(self): + assert self.function_module is not None or self.function_file is not None + + def load(self) -> cvataa.AutoAnnotationFunction: + if self.function_module is not None: + function = importlib.import_module(self.function_module) + else: + module_spec = importlib.util.spec_from_file_location( + "__cvat_function__", self.function_file + ) + function = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(function) + + if hasattr(function, "create"): + # this is actually a function factory + function = function.create(**self.function_parameters) + else: + if self.function_parameters: + raise TypeError("function takes no parameters") + + if not hasattr(function, "spec"): + raise cvataa.BadFunctionError("function has no 'spec' attribute") + + return function diff --git a/cvat-cli/src/cvat_cli/_internal/parsers.py b/cvat-cli/src/cvat_cli/_internal/parsers.py new file mode 100644 index 0000000..2c2be54 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/parsers.py @@ -0,0 +1,73 @@ +# Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import json +import os.path +from typing import Any + +from attr.converters import to_bool +from cvat_sdk.core.proxies.tasks import ResourceType + + +def parse_resource_type(s: str) -> ResourceType: + try: + return ResourceType[s.upper()] + except KeyError: + return s + + +def parse_label_arg(s): + """If s is a file load it as JSON, otherwise parse s as JSON.""" + if os.path.exists(s): + with open(s, "r") as fp: + return json.load(fp) + else: + return json.loads(s) + + +def parse_function_parameter(s: str) -> tuple[str, Any]: + key, sep, type_and_value = s.partition("=") + + if not sep: + raise argparse.ArgumentTypeError("parameter value not specified") + + type_, sep, value = type_and_value.partition(":") + + if not sep: + raise argparse.ArgumentTypeError("parameter type not specified") + + if type_ == "int": + value = int(value) + elif type_ == "float": + value = float(value) + elif type_ == "str": + pass + elif type_ == "bool": + value = to_bool(value) + else: + raise argparse.ArgumentTypeError(f"unsupported parameter type {type_!r}") + + return (key, value) + + +def parse_threshold(s: str) -> float: + try: + value = float(s) + except ValueError as e: + raise argparse.ArgumentTypeError("must be a number") from e + + if not 0 <= value <= 1: + raise argparse.ArgumentTypeError("must be between 0 and 1") + return value + + +class BuildDictAction(argparse.Action): + def __init__(self, option_strings, dest, default=None, **kwargs): + super().__init__(option_strings, dest, default=default or {}, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + key, value = values + getattr(namespace, self.dest)[key] = value diff --git a/cvat-cli/src/cvat_cli/_internal/utils.py b/cvat-cli/src/cvat_cli/_internal/utils.py new file mode 100644 index 0000000..4a3d142 --- /dev/null +++ b/cvat-cli/src/cvat_cli/_internal/utils.py @@ -0,0 +1,9 @@ +# Copyright (C) CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + + +def popattr(obj, name): + value = getattr(obj, name) + delattr(obj, name) + return value diff --git a/cvat-cli/src/cvat_cli/version.py b/cvat-cli/src/cvat_cli/version.py new file mode 100644 index 0000000..ab4abe6 --- /dev/null +++ b/cvat-cli/src/cvat_cli/version.py @@ -0,0 +1 @@ +VERSION = "2.69.1" diff --git a/cvat-core/.dockerignore b/cvat-core/.dockerignore new file mode 100644 index 0000000..1ea915a --- /dev/null +++ b/cvat-core/.dockerignore @@ -0,0 +1,5 @@ +/dist +/docs +/node_modules +/reports + diff --git a/cvat-core/.eslintrc.cjs b/cvat-core/.eslintrc.cjs new file mode 100644 index 0000000..57e274b --- /dev/null +++ b/cvat-core/.eslintrc.cjs @@ -0,0 +1,28 @@ +// Copyright (C) 2018-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +const { join } = require('path'); + +module.exports = { + ignorePatterns: [ + '.eslintrc.cjs', + 'webpack.config.cjs', + 'node_modules/**', + 'dist/**', + 'tests/**/*.cjs', + ], + parserOptions: { + project: './tsconfig.json', + tsconfigRootDir: __dirname, + }, + rules: { + 'import/no-extraneous-dependencies': [ + 'error', + { + packageDir: [__dirname, join(__dirname, '../')] + }, + ], + } +}; diff --git a/cvat-core/README.md b/cvat-core/README.md new file mode 100644 index 0000000..39a7d2f --- /dev/null +++ b/cvat-core/README.md @@ -0,0 +1,42 @@ +# Module CVAT-CORE + +## Description + +This CVAT module is a client-side JavaScript library for management of objects, frames, logs, etc. +It contains the core logic of the Computer Vision Annotation Tool. + +### Commands + +- Dependencies installation + +```bash +yarn install --immutable +``` + + + +- Building the module from sources in the `dist` directory: + +```bash +yarn run build +yarn run build --mode=development # without a minification +``` + +- Running of tests: + +```bash +yarn run test +``` + +- Updating of a module version: + +```bash +yarn version --patch # updated after minor fixes +yarn version --minor # updated after major changes which don't affect API compatibility with previous versions +yarn version --major # updated after major changes which affect API compatibility with previous versions +``` + +Visual studio code configurations: + +- cvat.js debug starts debugging with entrypoint api.js +- cvat.js test builds library and runs entrypoint tests.js diff --git a/cvat-core/babel.config.json b/cvat-core/babel.config.json new file mode 100644 index 0000000..cbd256c --- /dev/null +++ b/cvat-core/babel.config.json @@ -0,0 +1,16 @@ +{ + "presets": [ + [ + "@babel/preset-env", + { + "targets": { + "node": "current" + } + } + ], + "@babel/preset-typescript" + ], + "plugins": [ + "babel-plugin-transform-import-meta" + ] +} diff --git a/cvat-core/docs/Interpolation.pdf b/cvat-core/docs/Interpolation.pdf new file mode 100644 index 0000000..51e7d5e Binary files /dev/null and b/cvat-core/docs/Interpolation.pdf differ diff --git a/cvat-core/package.json b/cvat-core/package.json new file mode 100644 index 0000000..fd28ee1 --- /dev/null +++ b/cvat-core/package.json @@ -0,0 +1,38 @@ +{ + "name": "cvat-core", + "version": "15.3.1", + "type": "module", + "description": "Part of Computer Vision Tool which presents an interface for client-side integration", + "main": "src/api.ts", + "scripts": { + "build": "webpack", + "type-check": "tsc --noEmit", + "type-check:watch": "yarn run type-check --watch" + }, + "author": "CVAT.ai", + "license": "MIT", + "browserslist": [ + "Chrome >= 99", + "Firefox >= 110", + "not IE 11", + "> 2%" + ], + "devDependencies": { + "@babel/preset-typescript": "^7.23.3", + "babel-plugin-transform-import-meta": "^2.2.1" + }, + "dependencies": { + "axios": "^1.15.2", + "axios-retry": "^4.0.0", + "cvat-data": "link:./../cvat-data", + "detect-browser": "^5.2.1", + "dompurify": "^3.4.11", + "error-stack-parser": "^2.0.2", + "form-data": "^4.0.6", + "js-cookie": "^3.0.7", + "json-logic-js": "^2.0.1", + "platform": "^1.3.5", + "quickhull": "^1.0.3", + "tus-js-client": "^3.0.1" + } +} diff --git a/cvat-core/src/about.ts b/cvat-core/src/about.ts new file mode 100644 index 0000000..1604f82 --- /dev/null +++ b/cvat-core/src/about.ts @@ -0,0 +1,41 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { SerializedAbout } from './server-response-types'; + +export default class AboutData { + #description: string; + #name: string; + #version: string; + #logoURL: string; + #subtitle: string; + + constructor(initialData: SerializedAbout) { + this.#description = initialData.description; + this.#name = initialData.name; + this.#version = initialData.version; + this.#logoURL = initialData.logo_url; + this.#subtitle = initialData.subtitle; + } + + get description(): string { + return this.#description; + } + + get name(): string { + return this.#name; + } + + get version(): string { + return this.#version; + } + + get logoURL(): string { + return this.#logoURL; + } + + get subtitle(): string { + return this.#subtitle; + } +} diff --git a/cvat-core/src/annotation-formats.ts b/cvat-core/src/annotation-formats.ts new file mode 100644 index 0000000..09b854a --- /dev/null +++ b/cvat-core/src/annotation-formats.ts @@ -0,0 +1,104 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { DimensionType } from './enums'; +import { + SerializedAnnotationExporter, + SerializedAnnotationFormats, + SerializedAnnotationImporter, +} from './server-response-types'; + +export class Loader { + public name: string; + public format: string; + public version: string; + public enabled: boolean; + public dimension: DimensionType; + + constructor(initialData: SerializedAnnotationImporter) { + const data = { + name: initialData.name, + format: initialData.ext, + version: initialData.version, + enabled: initialData.enabled, + dimension: initialData.dimension, + }; + + Object.defineProperties(this, { + name: { + get: () => data.name, + }, + format: { + get: () => data.format, + }, + version: { + get: () => data.version, + }, + enabled: { + get: () => data.enabled, + }, + dimension: { + get: () => data.dimension, + }, + }); + } +} + +export class Dumper { + public name: string; + public format: string; + public version: string; + public enabled: boolean; + public dimension: DimensionType; + + constructor(initialData: SerializedAnnotationExporter) { + const data = { + name: initialData.name, + format: initialData.ext, + version: initialData.version, + enabled: initialData.enabled, + dimension: initialData.dimension, + }; + + Object.defineProperties(this, { + name: { + get: () => data.name, + }, + format: { + get: () => data.format, + }, + version: { + get: () => data.version, + }, + enabled: { + get: () => data.enabled, + }, + dimension: { + get: () => data.dimension, + }, + }); + } +} + +export default class AnnotationFormats { + public loaders: Loader[]; + public dumpers: Dumper[]; + + constructor(initialData: SerializedAnnotationFormats) { + const data = { + exporters: initialData.exporters.map((el) => new Dumper(el)), + importers: initialData.importers.map((el) => new Loader(el)), + }; + + Object.defineProperties(this, { + loaders: { + get: () => [...data.importers], + }, + dumpers: { + get: () => [...data.exporters], + }, + }); + } +} diff --git a/cvat-core/src/annotations-actions/actions-worker.ts b/cvat-core/src/annotations-actions/actions-worker.ts new file mode 100644 index 0000000..f6dedde --- /dev/null +++ b/cvat-core/src/annotations-actions/actions-worker.ts @@ -0,0 +1,119 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +/* eslint-disable no-restricted-globals */ + +import { initializeOpenCVInWorker } from '../worker-utils'; +import type { OpenCVInterface } from '../opencv/opencv-interface'; + +export enum WorkerAction { + INITIALIZE = 'initialize', + SIMPLIFY_POLYGONS = 'simplify_polygons', +} + +export interface SimplifyShape { + clientID: number; + points: number[]; + shapeType: 'polygon' | 'polyline'; + frame: number; +} + +export interface WorkerRequest { + command: WorkerAction; + opencvPath?: string; + shapes?: SimplifyShape[]; + threshold?: number; +} + +export interface WorkerResponse { + shapes?: SimplifyShape[]; + error?: string; +} + +class ActionsWorkerManager { + private cvInterface: OpenCVInterface | null = null; + private initialized = false; + + public async initialize(opencvPath: string): Promise { + this.cvInterface = await initializeOpenCVInWorker(opencvPath); + this.initialized = true; + } + + public simplifyShapes(shapes: SimplifyShape[], threshold: number): SimplifyShape[] { + if (!this.initialized || !this.cvInterface) { + throw new Error('Worker not initialized'); + } + + return shapes.map((shape) => { + try { + const closed = shape.shapeType === 'polygon'; + const simplifiedPoints = this.cvInterface!.contours.simplifyPolygon( + shape.points, + threshold, + closed, + ); + return { + ...shape, + points: simplifiedPoints, + }; + } catch (error: unknown) { + // If simplification fails, return original shape + console.error(`Failed to simplify shape ${shape.clientID} on frame ${shape.frame}:`, error); + return shape; + } + }); + } + + public get isInitialized(): boolean { + return this.initialized; + } +} + +if ((self as any).importScripts) { + const manager = new ActionsWorkerManager(); + + onmessage = (event: MessageEvent) => { + const { command } = event.data; + + if (command === WorkerAction.INITIALIZE) { + const { opencvPath } = event.data; + + manager.initialize(opencvPath) + .then(() => { + postMessage({} as WorkerResponse); + }) + .catch((error: unknown) => { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + postMessage({ + error: `Could not initialize OpenCV in worker: ${errorMessage}`, + } as WorkerResponse); + }); + } else if (command === WorkerAction.SIMPLIFY_POLYGONS) { + const { shapes, threshold } = event.data; + + if (!shapes || threshold === undefined) { + postMessage({ + error: 'Missing shapes or threshold for simplification', + } as WorkerResponse); + return; + } + + try { + const simplifiedShapes = manager.simplifyShapes(shapes, threshold); + postMessage({ + shapes: simplifiedShapes, + } as WorkerResponse); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + postMessage({ + error: `Error during simplification: ${errorMessage}`, + } as WorkerResponse); + } + } else { + postMessage({ + error: `Unknown command: ${command}`, + } as WorkerResponse); + } + }; +} diff --git a/cvat-core/src/annotations-actions/annotations-actions.ts b/cvat-core/src/annotations-actions/annotations-actions.ts new file mode 100644 index 0000000..8be28d3 --- /dev/null +++ b/cvat-core/src/annotations-actions/annotations-actions.ts @@ -0,0 +1,124 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import ObjectState from '../object-state'; +import { ArgumentError } from '../exceptions'; +import { Job, Task } from '../session'; +import { BaseAction } from './base-action'; +import { + BaseShapesAction, run as runShapesAction, call as callShapesAction, +} from './base-shapes-action'; +import { + BaseCollectionAction, run as runCollectionAction, call as callCollectionAction, +} from './base-collection-action'; + +import { RemoveFilteredShapes } from './remove-filtered-shapes'; +import { PropagateShapes } from './propagate-shapes'; +import { PolySimplify } from './poly-simplify'; + +const registeredActions: BaseAction[] = []; + +export async function listActions(): Promise { + return [...registeredActions]; +} + +export async function registerAction(action: BaseAction): Promise { + if (!(action instanceof BaseAction)) { + throw new ArgumentError('Provided action must inherit one of base classes'); + } + + const { name } = action; + if (registeredActions.map((_action) => _action.name).includes(name)) { + throw new ArgumentError(`Action name must be unique. Name "${name}" is already exists`); + } + + registeredActions.push(action); +} + +export async function unregisterAction(action: BaseAction): Promise { + const index = registeredActions.indexOf(action); + if (index === -1) { + throw new ArgumentError(`Action "${action.name}" was not registered`); + } else { + registeredActions.splice(index, 1); + } +} + +registerAction(new RemoveFilteredShapes()); +registerAction(new PropagateShapes()); +registerAction(new PolySimplify()); + +export async function runAction( + instance: Job | Task, + action: BaseAction, + actionParameters: Record, + frameFrom: number, + frameTo: number, + filters: object[], + onProgress: (message: string, progress: number) => void, + cancelled: () => boolean, +): Promise { + if (action instanceof BaseShapesAction) { + return runShapesAction( + instance, + action, + actionParameters, + frameFrom, + frameTo, + filters, + onProgress, + cancelled, + ); + } + + if (action instanceof BaseCollectionAction) { + return runCollectionAction( + instance, + action, + actionParameters, + frameFrom, + filters, + onProgress, + cancelled, + ); + } + + return Promise.resolve(); +} + +export async function callAction( + instance: Job | Task, + action: BaseAction, + actionParameters: Record, + frame: number, + states: ObjectState[], + onProgress: (message: string, progress: number) => void, + cancelled: () => boolean, +): Promise { + if (action instanceof BaseShapesAction) { + return callShapesAction( + instance, + action, + actionParameters, + frame, + states, + onProgress, + cancelled, + ); + } + + if (action instanceof BaseCollectionAction) { + return callCollectionAction( + instance, + action, + actionParameters, + frame, + states, + onProgress, + cancelled, + ); + } + + return Promise.resolve(); +} diff --git a/cvat-core/src/annotations-actions/base-action.ts b/cvat-core/src/annotations-actions/base-action.ts new file mode 100644 index 0000000..31bc34f --- /dev/null +++ b/cvat-core/src/annotations-actions/base-action.ts @@ -0,0 +1,68 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { SerializedCollection } from '../server-response-types'; +import ObjectState from '../object-state'; +import { Job, Task } from '../session'; + +export enum ActionParameterType { + SELECT = 'select', + NUMBER = 'number', + CHECKBOX = 'checkbox', +} + +// For SELECT values should be a list of possible options +// For NUMBER values should be a list with [min, max, step], +// or a callback ({ instance }: { instance: Job | Task }) => [min, max, step] +export type ActionParameters = Record string[]); + defaultValue: string | (({ instance }: { instance: Job | Task }) => string); + tooltip?: { + type: string, + content: string | { + columns: Record[]; + data: Record[]; + } + }; +}>; + +export abstract class BaseAction { + public abstract init(sessionInstance: Job | Task, parameters: Record): Promise; + public abstract destroy(): Promise; + public abstract run(input: unknown): Promise; + public abstract applyFilter(input: unknown): unknown; + public abstract isApplicableForObject(objectState: ObjectState): boolean; + + public abstract get name(): string; + public abstract get parameters(): ActionParameters | null; +} + +export function prepareActionParameters(declared: ActionParameters, defined: object): Record { + if (!declared) { + return {}; + } + + return Object.entries(declared).reduce((acc, [name, { type, defaultValue }]) => { + if (type === ActionParameterType.NUMBER) { + acc[name] = +(Object.hasOwn(defined, name) ? defined[name] : defaultValue); + } else { + acc[name] = (Object.hasOwn(defined, name) ? defined[name] : defaultValue); + } + return acc; + }, {} as Record); +} + +export function validateClientIDs(collection: Partial): void { + [].concat( + collection.shapes ?? [], + collection.tracks ?? [], + collection.tags ?? [], + ).forEach((object) => { + // clientID is required to correct collection filtering and committing in annotations actions logic + if (typeof object.clientID !== 'number') { + throw new Error('ClientID is undefined when running annotations action, but required'); + } + }); +} diff --git a/cvat-core/src/annotations-actions/base-collection-action.ts b/cvat-core/src/annotations-actions/base-collection-action.ts new file mode 100644 index 0000000..dd922e4 --- /dev/null +++ b/cvat-core/src/annotations-actions/base-collection-action.ts @@ -0,0 +1,179 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { throttle } from 'lodash'; + +import ObjectState from '../object-state'; +import AnnotationsFilter from '../annotations-filter'; +import { Job, Task } from '../session'; +import { + SerializedCollection, SerializedShape, + SerializedTag, SerializedTrack, +} from '../server-response-types'; +import { EventScope, ObjectType } from '../enums'; +import { getCollection } from '../annotations'; +import { BaseAction, prepareActionParameters, validateClientIDs } from './base-action'; + +export interface CollectionActionInput { + onProgress(message: string, percent: number): void; + cancelled(): boolean; + collection: Pick; + frameData: { + width: number; + height: number; + number: number; + }; +} + +export interface CollectionActionOutput { + created: CollectionActionInput['collection']; + deleted: CollectionActionInput['collection']; +} + +export abstract class BaseCollectionAction extends BaseAction { + public abstract run(input: CollectionActionInput): Promise; + public abstract applyFilter( + input: Pick, + ): CollectionActionInput['collection']; +} + +export async function run( + instance: Job | Task, + action: BaseCollectionAction, + actionParameters: Record, + frame: number, + filters: object[], + onProgress: (message: string, progress: number) => void, + cancelled: () => boolean, +): Promise { + const event = await instance.logger.log(EventScope.annotationsAction, { + from: frame, + to: frame, + name: action.name, + }, true); + + const wrappedOnProgress = throttle(onProgress, 100, { leading: true, trailing: true }); + const showMessageWithPause = async (message: string, progress: number, duration: number): Promise => { + // wrapper that gives a chance to abort action + wrappedOnProgress(message, progress); + await new Promise((resolve) => setTimeout(resolve, duration)); + }; + + try { + await showMessageWithPause('Action initialization', 0, 500); + if (cancelled()) { + return; + } + + await action.init(instance, prepareActionParameters(action.parameters, actionParameters)); + + const frameData = await Object.getPrototypeOf(instance).frames + .get.implementation.call(instance, frame); + const exportedCollection = getCollection(instance).export(); + + // Apply action filter first + const filteredByAction = action.applyFilter({ collection: exportedCollection, frameData }); + validateClientIDs(filteredByAction); + + let mapID2Obj = [].concat(filteredByAction.shapes, filteredByAction.tags, filteredByAction.tracks) + .reduce((acc, object) => { + acc[object.clientID as number] = object; + return acc; + }, {}); + + // Then apply user filter + const annotationsFilter = new AnnotationsFilter( + instance instanceof Job ? instance.stopFrame : instance.size - 1, + ); + const filteredCollectionIDs = annotationsFilter + .filterSerializedCollection(filteredByAction, instance.labels, filters); + const filteredByUser = { + shapes: filteredCollectionIDs.shapes.map((clientID) => mapID2Obj[clientID]), + tags: filteredCollectionIDs.tags.map((clientID) => mapID2Obj[clientID]), + tracks: filteredCollectionIDs.tracks.map((clientID) => mapID2Obj[clientID]), + }; + mapID2Obj = [].concat(filteredByUser.shapes, filteredByUser.tags, filteredByUser.tracks) + .reduce((acc, object) => { + acc[object.clientID as number] = object; + return acc; + }, {}); + + const { created, deleted } = await action.run({ + collection: filteredByUser, + frameData: { + width: frameData.width, + height: frameData.height, + number: frameData.number, + }, + onProgress: wrappedOnProgress, + cancelled, + }); + + await instance.annotations.commit(created, deleted, frame); + event.close(); + } finally { + await action.destroy(); + } +} + +export async function call( + instance: Job | Task, + action: BaseCollectionAction, + actionParameters: Record, + frame: number, + states: ObjectState[], + onProgress: (message: string, progress: number) => void, + cancelled: () => boolean, +): Promise { + const event = await instance.logger.log(EventScope.annotationsAction, { + from: frame, + to: frame, + name: action.name, + }, true); + + const throttledOnProgress = throttle(onProgress, 100, { leading: true, trailing: true }); + try { + await action.init(instance, prepareActionParameters(action.parameters, actionParameters)); + const exportedStates = await Promise.all(states.map((state) => state.export())); + const exportedCollection = exportedStates.reduce((acc, value, idx) => { + if (states[idx].objectType === ObjectType.SHAPE) { + acc.shapes.push(value as SerializedShape); + } + + if (states[idx].objectType === ObjectType.TAG) { + acc.tags.push(value as SerializedTag); + } + + if (states[idx].objectType === ObjectType.TRACK) { + acc.tracks.push(value as SerializedTrack); + } + + return acc; + }, { shapes: [], tags: [], tracks: [] }); + + const frameData = await Object.getPrototypeOf(instance).frames.get.implementation.call(instance, frame); + const filteredByAction = action.applyFilter({ collection: exportedCollection, frameData }); + validateClientIDs(filteredByAction); + + const processedCollection = await action.run({ + onProgress: throttledOnProgress, + cancelled, + collection: filteredByAction, + frameData: { + width: frameData.width, + height: frameData.height, + number: frameData.number, + }, + }); + + await instance.annotations.commit( + processedCollection.created, + processedCollection.deleted, + frame, + ); + event.close(); + } finally { + await action.destroy(); + } +} diff --git a/cvat-core/src/annotations-actions/base-shapes-action.ts b/cvat-core/src/annotations-actions/base-shapes-action.ts new file mode 100644 index 0000000..b358b84 --- /dev/null +++ b/cvat-core/src/annotations-actions/base-shapes-action.ts @@ -0,0 +1,205 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { throttle, range } from 'lodash'; + +import ObjectState from '../object-state'; +import AnnotationsFilter from '../annotations-filter'; +import { Job, Task } from '../session'; +import { SerializedCollection, SerializedShape } from '../server-response-types'; +import { EventScope, ObjectType } from '../enums'; +import { getCollection } from '../annotations'; +import { BaseAction, prepareActionParameters, validateClientIDs } from './base-action'; + +export interface ShapesActionInput { + onProgress(message: string, percent: number): void; + cancelled(): boolean; + collection: Pick; + frameData: { + width: number; + height: number; + number: number; + }; +} + +export interface ShapesActionOutput { + created: ShapesActionInput['collection']; + deleted: ShapesActionInput['collection']; +} + +export abstract class BaseShapesAction extends BaseAction { + public abstract run(input: ShapesActionInput): Promise; + public abstract applyFilter( + input: Pick + ): ShapesActionInput['collection']; +} + +export async function run( + instance: Job | Task, + action: BaseShapesAction, + actionParameters: Record, + frameFrom: number, + frameTo: number, + filters: object[], + onProgress: (message: string, progress: number) => void, + cancelled: () => boolean, +): Promise { + const event = await instance.logger.log(EventScope.annotationsAction, { + from: frameFrom, + to: frameTo, + name: action.name, + }, true); + + const throttledOnProgress = throttle(onProgress, 100, { leading: true, trailing: true }); + const showMessageWithPause = async (message: string, progress: number, duration: number): Promise => { + // wrapper that gives a chance to abort action + throttledOnProgress(message, progress); + await new Promise((resolve) => { setTimeout(resolve, duration); }); + }; + + try { + await showMessageWithPause('Actions initialization', 0, 500); + if (cancelled()) { + return; + } + + await action.init(instance, prepareActionParameters(action.parameters, actionParameters)); + + const exportedCollection = getCollection(instance).export(); + validateClientIDs(exportedCollection); + + const annotationsFilter = new AnnotationsFilter( + instance instanceof Job ? instance.stopFrame : instance.size - 1, + ); + const filteredShapeIDs = annotationsFilter.filterSerializedCollection({ + shapes: exportedCollection.shapes, + tags: [], + tracks: [], + }, instance.labels, filters).shapes; + + const filteredShapesByFrame = exportedCollection.shapes.reduce((acc, shape) => { + if (shape.frame >= frameFrom && shape.frame <= frameTo && filteredShapeIDs.includes(shape.clientID)) { + acc[shape.frame] = acc[shape.frame] ?? []; + acc[shape.frame].push(shape); + } + return acc; + }, {} as Record); + + const totalUpdates = { created: { shapes: [] }, deleted: { shapes: [] } }; + // Iterate over frames + const rawFrameNumbers = instance instanceof Job ? + await instance.frames.frameNumbers() : range(0, instance.size); + const frameNumbers = rawFrameNumbers.filter((frame) => frame >= frameFrom && frame <= frameTo); + const totalFrames = frameNumbers.length; + if (totalFrames === 0) { + await showMessageWithPause('No frames to process', 100, 1500); + return; + } + for (let frameIdx = 0; frameIdx < totalFrames; frameIdx++) { + const frame = frameNumbers[frameIdx]; + const frameData = await Object.getPrototypeOf(instance).frames + .get.implementation.call(instance, frame); + + // Ignore deleted frames + if (!frameData.deleted) { + const frameShapes = filteredShapesByFrame[frame] ?? []; + if (!frameShapes.length) { + continue; + } + + // finally apply the own filter of the action + const filteredByAction = action.applyFilter({ + collection: { + shapes: frameShapes, + }, + frameData, + }); + validateClientIDs(filteredByAction); + + const { created, deleted } = await action.run({ + onProgress: throttledOnProgress, + cancelled, + collection: { shapes: filteredByAction.shapes }, + frameData: { + width: frameData.width, + height: frameData.height, + number: frameData.number, + }, + }); + + Array.prototype.push.apply(totalUpdates.created.shapes, created.shapes); + Array.prototype.push.apply(totalUpdates.deleted.shapes, deleted.shapes); + + const progress = Math.ceil(((frameIdx + 1) / totalFrames) * 100); + throttledOnProgress('Actions are running', progress); + if (cancelled()) { + return; + } + } + } + + await showMessageWithPause('Committing handled objects', 100, 1500); + if (cancelled()) { + return; + } + + await instance.annotations.commit( + { shapes: totalUpdates.created.shapes, tags: [], tracks: [] }, + { shapes: totalUpdates.deleted.shapes, tags: [], tracks: [] }, + frameNumbers[0], + ); + + event.close(); + } finally { + await action.destroy(); + } +} + +export async function call( + instance: Job | Task, + action: BaseShapesAction, + actionParameters: Record, + frame: number, + states: ObjectState[], + onProgress: (message: string, progress: number) => void, + cancelled: () => boolean, +): Promise { + const event = await instance.logger.log(EventScope.annotationsAction, { + from: frame, + to: frame, + name: action.name, + }, true); + + const throttledOnProgress = throttle(onProgress, 100, { leading: true, trailing: true }); + try { + await action.init(instance, prepareActionParameters(action.parameters, actionParameters)); + + const exported = await Promise.all(states.filter((state) => state.objectType === ObjectType.SHAPE) + .map((state) => state.export())) as SerializedShape[]; + const frameData = await Object.getPrototypeOf(instance).frames.get.implementation.call(instance, frame); + const filteredByAction = action.applyFilter({ collection: { shapes: exported }, frameData }); + validateClientIDs(filteredByAction); + + const processedCollection = await action.run({ + onProgress: throttledOnProgress, + cancelled, + collection: { shapes: filteredByAction.shapes }, + frameData: { + width: frameData.width, + height: frameData.height, + number: frameData.number, + }, + }); + + await instance.annotations.commit( + { shapes: processedCollection.created.shapes, tags: [], tracks: [] }, + { shapes: processedCollection.deleted.shapes, tags: [], tracks: [] }, + frame, + ); + + event.close(); + } finally { + await action.destroy(); + } +} diff --git a/cvat-core/src/annotations-actions/poly-simplify.ts b/cvat-core/src/annotations-actions/poly-simplify.ts new file mode 100644 index 0000000..a73e4d3 --- /dev/null +++ b/cvat-core/src/annotations-actions/poly-simplify.ts @@ -0,0 +1,188 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import ObjectState from '../object-state'; +import { ShapeType, ObjectType } from '../enums'; +import config from '../config'; + +import { ActionParameterType, ActionParameters } from './base-action'; +import { BaseShapesAction, ShapesActionInput, ShapesActionOutput } from './base-shapes-action'; +import { WorkerAction } from './actions-worker'; +import type { SimplifyShape, WorkerRequest, WorkerResponse } from './actions-worker'; + +export class PolySimplify extends BaseShapesAction { + #threshold = 1.0; + #worker: Worker | null = null; + + public async init(_instance: any, parameters: Record): Promise { + this.#threshold = parameters.Distance as number; + + return new Promise((resolve, reject) => { + this.#worker = new Worker(new URL('./actions-worker.ts', import.meta.url)); + + this.#worker.onmessage = (event: MessageEvent) => { + const result = event.data; + if (result.error) { + reject(new Error(result.error)); + } else { + resolve(); + } + }; + + this.#worker.onerror = (event: ErrorEvent) => { + reject(new Error(event.message)); + }; + + this.#worker.postMessage({ + command: WorkerAction.INITIALIZE, + opencvPath: config.opencvPath, + } as WorkerRequest); + }); + } + + public async destroy(): Promise { + if (this.#worker) { + this.#worker.terminate(); + this.#worker = null; + } + } + + public async run(input: ShapesActionInput): Promise { + const { onProgress, cancelled } = input; + + if (!this.#worker) { + throw new Error('Worker not initialized'); + } + + if (!input.collection.shapes.length) { + return { + created: { shapes: [] }, + deleted: { shapes: [] }, + }; + } + + return new Promise((resolve, reject) => { + if (cancelled()) { + resolve({ + created: { shapes: [] }, + deleted: { shapes: [] }, + }); + return; + } + + onProgress('Simplifying polygons', 0); + + this.#worker!.onmessage = (event: MessageEvent) => { + const result = event.data; + if (result.error) { + reject(new Error(result.error)); + } else if (result.shapes) { + onProgress('Simplifying polygons', 100); + + const simplifiedShapes = result.shapes.map((simplifiedShape) => { + const originalShape = input.collection.shapes.find( + (s) => s.clientID === simplifiedShape.clientID, + ); + return { + ...originalShape, + points: simplifiedShape.points, + }; + }); + + resolve({ + created: { shapes: simplifiedShapes }, + deleted: { shapes: input.collection.shapes }, + }); + } + }; + + this.#worker!.onerror = (event: ErrorEvent) => { + reject(new Error(event.message)); + }; + + const shapes: SimplifyShape[] = input.collection.shapes.map((shape) => ({ + clientID: shape.clientID, + points: shape.points, + shapeType: shape.type as ShapeType.POLYGON | ShapeType.POLYLINE, + frame: shape.frame, + })); + + this.#worker!.postMessage({ + command: WorkerAction.SIMPLIFY_POLYGONS, + shapes, + threshold: this.#threshold, + } as WorkerRequest); + }); + } + + public applyFilter(input: ShapesActionInput): ShapesActionInput['collection'] { + return { + shapes: input.collection.shapes.filter( + (shape) => shape.type === ShapeType.POLYGON || shape.type === ShapeType.POLYLINE, + ), + }; + } + + public isApplicableForObject(objectState: ObjectState): boolean { + return ( + objectState.objectType === ObjectType.SHAPE && + (objectState.shapeType === ShapeType.POLYGON || objectState.shapeType === ShapeType.POLYLINE) + ); + } + + public get name(): string { + return 'Simplify polygons and polylines'; + } + + public get parameters(): ActionParameters | null { + return { + Distance: { + type: ActionParameterType.NUMBER, + values: ['0.1', '64', '0.1'], // min, max, step + defaultValue: '1.0', + tooltip: { + type: 'table', + content: { + columns: [ + { + title: 'Distance', + dataIndex: 'distance', + key: 'distance', + }, + { + title: 'Detail', + dataIndex: 'detail', + key: 'detail', + }, + { + title: 'Reduction', + dataIndex: 'reduction', + key: 'reduction', + }, + { + title: 'Use Case', + dataIndex: 'useCase', + key: 'useCase', + }, + ], + data: [ + { + distance: '0.5', detail: 'High detail', reduction: '~10%', useCase: 'Precise imaging', + }, + { + distance: '1.0', detail: 'Balanced', reduction: '~30-50%', useCase: 'General annotations', + }, + { + distance: '2.0', detail: 'Simplified', reduction: '~60-80%', useCase: 'High optimization', + }, + { + distance: '5.0+', detail: 'Very simple', reduction: '~90%', useCase: 'Rough shapes', + }, + ], + }, + }, + }, + }; + } +} diff --git a/cvat-core/src/annotations-actions/propagate-shapes.ts b/cvat-core/src/annotations-actions/propagate-shapes.ts new file mode 100644 index 0000000..aeb82e7 --- /dev/null +++ b/cvat-core/src/annotations-actions/propagate-shapes.ts @@ -0,0 +1,85 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { range } from 'lodash'; + +import ObjectState from '../object-state'; +import { Job, Task } from '../session'; +import { SerializedShape } from '../server-response-types'; +import { propagateShapes } from '../object-utils'; +import { ObjectType } from '../enums'; + +import { ActionParameterType, ActionParameters } from './base-action'; +import { BaseCollectionAction, CollectionActionInput, CollectionActionOutput } from './base-collection-action'; + +export class PropagateShapes extends BaseCollectionAction { + #instance: Task | Job; + #targetFrame: number; + + public async init(instance: Job | Task, parameters): Promise { + this.#instance = instance; + this.#targetFrame = parameters['Target frame']; + } + + public async destroy(): Promise { + // nothing to destroy + } + + public async run(input: CollectionActionInput): Promise { + const { collection, frameData: { number } } = input; + if (number === this.#targetFrame) { + return { + created: { shapes: [], tags: [], tracks: [] }, + deleted: { shapes: [], tags: [], tracks: [] }, + }; + } + + const frameNumbers = this.#instance instanceof Job ? + await this.#instance.frames.frameNumbers() : range(0, this.#instance.size); + const propagatedShapes = propagateShapes( + collection.shapes, number, this.#targetFrame, frameNumbers, + ); + + return { + created: { shapes: propagatedShapes, tags: [], tracks: [] }, + deleted: { shapes: [], tags: [], tracks: [] }, + }; + } + + public applyFilter(input: CollectionActionInput): CollectionActionInput['collection'] { + return { + shapes: input.collection.shapes.filter((shape) => shape.frame === input.frameData.number), + tags: [], + tracks: [], + }; + } + + public isApplicableForObject(objectState: ObjectState): boolean { + return objectState.objectType === ObjectType.SHAPE; + } + + public get name(): string { + return 'Propagate shapes'; + } + + public get parameters(): ActionParameters | null { + return { + 'Target frame': { + type: ActionParameterType.NUMBER, + values: ({ instance }) => { + if (instance instanceof Job) { + return [instance.startFrame, instance.stopFrame, 1].map((val) => val.toString()); + } + return [0, instance.size - 1, 1].map((val) => val.toString()); + }, + defaultValue: ({ instance }) => { + if (instance instanceof Job) { + return instance.stopFrame.toString(); + } + return (instance.size - 1).toString(); + }, + }, + }; + } +} diff --git a/cvat-core/src/annotations-actions/remove-filtered-shapes.ts b/cvat-core/src/annotations-actions/remove-filtered-shapes.ts new file mode 100644 index 0000000..41617de --- /dev/null +++ b/cvat-core/src/annotations-actions/remove-filtered-shapes.ts @@ -0,0 +1,41 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { BaseShapesAction, ShapesActionInput, ShapesActionOutput } from './base-shapes-action'; +import { ActionParameters } from './base-action'; + +export class RemoveFilteredShapes extends BaseShapesAction { + public async init(): Promise { + // nothing to init + } + + public async destroy(): Promise { + // nothing to destroy + } + + public async run(input: ShapesActionInput): Promise { + return { + created: { shapes: [] }, + deleted: input.collection, + }; + } + + public applyFilter(input: ShapesActionInput): ShapesActionInput['collection'] { + const { collection } = input; + return collection; + } + + public isApplicableForObject(): boolean { + // remove action does not make sense when running on one object + return false; + } + + public get name(): string { + return 'Remove filtered shapes'; + } + + public get parameters(): ActionParameters | null { + return null; + } +} diff --git a/cvat-core/src/annotations-collection.ts b/cvat-core/src/annotations-collection.ts new file mode 100644 index 0000000..2902308 --- /dev/null +++ b/cvat-core/src/annotations-collection.ts @@ -0,0 +1,1795 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import _ from 'lodash'; +import { + shapeFactory, trackFactory, Track, Shape, Tag, + MaskShape, BasicInjection, SkeletonShape, + SkeletonTrack, PolygonShape, CuboidShape, + RectangleShape, PolylineShape, PointsShape, EllipseShape, + InterpolationNotPossibleError, AudioInterval, +} from './annotations-objects'; +import { + SerializedCollection, SerializedShape, SerializedTrack, +} from './server-response-types'; +import AnnotationsFilter from './annotations-filter'; +import { checkObjectType } from './common'; +import Statistics from './statistics'; +import { Attribute, Label } from './labels'; +import { ArgumentError } from './exceptions'; +import ObjectState from './object-state'; +import { cropMask } from './object-utils'; +import { AudioIntervalState } from './annotations-objects/audio-interval-state'; +import config from './config'; +import { + HistoryActions, ShapeType, ObjectType, colors, Source, DimensionType, JobType, +} from './enums'; +import AnnotationHistory from './annotations-history'; + +type AnnotationObject = Shape | Tag | Track | AudioInterval; +type AnnotationState = ObjectState | AudioIntervalState; + +const validateAttributesList = ( + attributes: { spec_id: number, value: string }[], +): { spec_id: number, value: string }[] => { + for (const { spec_id: specID, value } of attributes) { + checkObjectType('attribute id', specID, 'integer', null); + checkObjectType('attribute value', value, 'string', null); + } + return attributes; +}; + +const objectAttributesAsList = (state: AnnotationState): { spec_id: number, value: string }[] => ( + Object.entries(state.attributes).map(([key, value]) => ({ + spec_id: +key, + value, + })) +); + +type LayerPlacement = { exact: number } | { before: number } | { after: number }; +type LayerPlacementData = + { kind: 'exact'; zOrder: number } | + { kind: 'before'; zOrder: number } | + { kind: 'after'; zOrder: number }; + +function isLayerState(state: ObjectState): boolean { + return [ObjectType.SHAPE, ObjectType.TRACK].includes(state.objectType); +} + +function parseLayerPlacement(placement: LayerPlacement): LayerPlacementData { + checkObjectType('placement', placement, null, { cls: Object, name: 'Object' }); + + const hasExact = Object.hasOwn(placement, 'exact'); + const hasBefore = Object.hasOwn(placement, 'before'); + const hasAfter = Object.hasOwn(placement, 'after'); + const specifiedCount = Number(hasExact) + Number(hasBefore) + Number(hasAfter); + + if (specifiedCount !== 1) { + throw new ArgumentError('Exactly one of "exact", "before", or "after" must be specified'); + } + + if (hasExact) { + const { exact: zOrder } = placement as { exact: number }; + checkObjectType('placement exact', zOrder, 'integer', null); + return { kind: 'exact', zOrder }; + } + + if (hasBefore) { + const { before: zOrder } = placement as { before: number }; + checkObjectType('placement before', zOrder, 'integer', null); + return { kind: 'before', zOrder }; + } + + const { after: zOrder } = placement as { after: number }; + checkObjectType('placement after', zOrder, 'integer', null); + return { kind: 'after', zOrder }; +} + +const labelAttributesAsDict = (label: Label): Record => ( + label.attributes.reduce((accumulator, attribute) => { + accumulator[attribute.id] = attribute; + return accumulator; + }, {}) +); + +export default class Collection { + public flush: boolean; + private stopFrame: number; + private labels: Record; + private annotationsFilter: AnnotationsFilter; + private history: AnnotationHistory; + private shapes: Record; + private tags: Record; + private tracks: Track[]; + private intervals: AudioInterval[]; + private objects: Record; + private groupsInfo: BasicInjection['groupsInfo']; + private injection: BasicInjection; + + constructor(data: { + labels: Label[]; + history: AnnotationHistory; + stopFrame: number; + dimension: DimensionType; + framesInfo: BasicInjection['framesInfo']; + jobType: JobType; + replicasCount?: number; + }) { + this.stopFrame = data.stopFrame; + + this.labels = data.labels.reduce((labelAccumulator, label) => { + // eslint-disable-next-line no-param-reassign + labelAccumulator[label.id] = label; + (label?.structure?.sublabels || []).forEach((sublabel) => { + // eslint-disable-next-line no-param-reassign + labelAccumulator[sublabel.id] = sublabel; + }); + + return labelAccumulator; + }, {}); + + this.annotationsFilter = new AnnotationsFilter(this.stopFrame); + this.history = data.history; + this.shapes = {}; // key is a frame + this.tags = {}; // key is a frame + this.tracks = []; + this.intervals = []; + this.objects = {}; // key is a client id + this.flush = false; + this.groupsInfo = { + max: 0, + colors: {}, + }; // it is an object to we can pass it as an argument by a reference + + this.injection = { + labels: this.labels, + groupsInfo: this.groupsInfo, + framesInfo: data.framesInfo, + history: this.history, + dimension: data.dimension, + jobType: data.jobType, + nextClientID: () => ++config.globalObjectsCounter, + getMasksOnFrame: (frame: number) => (this.shapes[frame] as MaskShape[]) + .filter((object) => object instanceof MaskShape), + replicasCount: data.replicasCount, + }; + } + + private _captureZOrderRestore(object: Shape | Track, frame: number): () => void { + if (object instanceof Track) { + const wasKeyframe = frame in object.shapes; + const shape = wasKeyframe ? object.shapes[frame] : undefined; + const { source } = object; + + return (): void => { + object.source = source; + object.updated = Date.now(); + if (shape) { + object.shapes[frame] = shape; + } else { + delete object.shapes[frame]; + } + }; + } + + const { zOrder, source } = object; + return (): void => { + object.source = source; + object.updated = Date.now(); + object.zOrder = zOrder; + }; + } + + private _applyZOrderUpdates(frame: number, zOrders: Map): ObjectState[] { + const updatedStates: ObjectState[] = []; + const snapshots: { + clientID: number; + undo: () => void; + redo: () => void; + }[] = []; + + // Prevent each individual object.save() from creating its own history item. + this.history.freeze(true); + + try { + for (const [clientID, zOrder] of zOrders) { + const object = this.objects[clientID]; + if (!(object instanceof Shape || object instanceof Track) || object.removed || object.lock) { + throw new Error('Only non-removed and non-locked shapes and tracks can be reordered'); + } + + let currentState: ObjectState; + try { + currentState = new ObjectState(object.get(frame)); + } catch (error: unknown) { + if (error instanceof InterpolationNotPossibleError) { + continue; + } + throw error; + } + + const previousZOrder = currentState.zOrder; + if (previousZOrder === zOrder) { + continue; + } + + const undo = this._captureZOrderRestore(object, frame); + currentState.zOrder = zOrder; + object.save(frame, currentState); + const redo = this._captureZOrderRestore(object, frame); + + const updatedState = new ObjectState(object.get(frame)); + snapshots.push({ clientID: object.clientID, undo, redo }); + updatedStates.push(updatedState); + } + } catch (error: unknown) { + snapshots.forEach(({ undo }) => undo()); + throw error; + } finally { + this.history.freeze(false); + } + + if (snapshots.length) { + // Store the whole layer operation as one undo/redo item after all objects are updated. + this.history.do( + HistoryActions.CHANGED_ZORDER, + () => { + snapshots.forEach(({ undo }) => undo()); + }, + () => { + snapshots.forEach(({ redo }) => redo()); + }, + snapshots.map(({ clientID }) => clientID), + frame, + ); + } + + return updatedStates; + } + + public import(data: Partial): { + tags: Tag[]; + shapes: Shape[]; + tracks: Track[]; + intervals: AudioInterval[]; + } { + const result = { + tags: [], + shapes: [], + tracks: [], + intervals: [], + }; + + for (const tag of data.tags ?? []) { + const clientID = this.injection.nextClientID(); + const color = colors[clientID % colors.length]; + const tagModel = new Tag(tag, clientID, color, this.injection); + this.tags[tagModel.frame] = this.tags[tagModel.frame] || []; + this.tags[tagModel.frame].push(tagModel); + this.objects[clientID] = tagModel; + + result.tags.push(tagModel); + } + + for (const shape of data.shapes ?? []) { + const clientID = this.injection.nextClientID(); + const shapeModel = shapeFactory(shape, clientID, this.injection); + this.shapes[shapeModel.frame] = this.shapes[shapeModel.frame] || []; + this.shapes[shapeModel.frame].push(shapeModel); + this.objects[clientID] = shapeModel; + + result.shapes.push(shapeModel); + } + + for (const track of data.tracks ?? []) { + const clientID = this.injection.nextClientID(); + const trackModel = trackFactory(track, clientID, this.injection); + // The function can return null if track doesn't have any shapes. + // In this case a corresponded message will be sent to the console + if (trackModel) { + this.tracks.push(trackModel); + result.tracks.push(trackModel); + this.objects[clientID] = trackModel; + } + } + + for (const interval of data.intervals ?? []) { + const clientID = this.injection.nextClientID(); + const color = colors[clientID % colors.length]; + const intervalModel = new AudioInterval(interval, clientID, color, this.injection); + this.intervals.push(intervalModel); + this.objects[clientID] = intervalModel; + result.intervals.push(intervalModel); + } + + return result; + } + + public commit( + appended: Partial, + removed: Partial, + frame: number | null, + ): void { + const removedObjects = [].concat( + removed.shapes ?? [], + removed.tags ?? [], + removed.tracks ?? [], + removed.intervals ?? [], + ); + + const allRemovedObjectsPresented = removedObjects.every( + (object) => typeof object.clientID === 'number' && + Object.hasOwn(this.objects, object.clientID), + ); + + if (!allRemovedObjectsPresented) { + throw new ArgumentError('Objects required to be deleted were not found in the collection'); + } + + const removedCollection: AnnotationObject[] = removedObjects + .map((object) => this.objects[object.clientID]); + + const imported = this.import(appended); + const appendedCollection = ([] as (AnnotationObject)[]).concat( + imported.shapes, + imported.tags, + imported.tracks, + imported.intervals, + ); + + if (appendedCollection.length === 0 && removedCollection.length === 0) { + // nothing to commit + return; + } + + let prevRemoved: boolean[] = []; + removedCollection.forEach((collectionObject) => { + prevRemoved.push(collectionObject.removed); + collectionObject.removed = true; + }); + + this.history.do( + HistoryActions.COMMIT_ANNOTATIONS, + () => { + removedCollection.forEach((collectionObject, idx) => { + collectionObject.removed = prevRemoved[idx]; + }); + prevRemoved = []; + appendedCollection.forEach((collectionObject) => { + collectionObject.removed = true; + }); + }, + () => { + removedCollection.forEach((collectionObject) => { + prevRemoved.push(collectionObject.removed); + collectionObject.removed = true; + }); + appendedCollection.forEach((collectionObject) => { + collectionObject.removed = false; + }); + }, + [].concat( + removedCollection.map((object) => object.clientID), + appendedCollection.map((object) => object.clientID), + ), + frame, + ); + } + + public getAllIntervals(filters: object[]): AudioIntervalState[] { + const intervals = this.intervals + .filter((interval) => !interval.removed) + .map((interval) => interval.get()); + + const filtered = this.annotationsFilter.filterAudioIntervalStates(intervals, filters); + return intervals.filter((interval) => !filters.length || filtered.includes(interval.clientID as number)); + } + + public export(): SerializedCollection { + const data = { + tracks: this.tracks.filter((track) => !track.removed) + .map((track) => track.toJSON() as SerializedTrack), + shapes: Object.values(this.shapes) + .reduce((accumulator, frameShapes) => { + accumulator.push(...frameShapes); + return accumulator; + }, []) + .filter((shape) => !shape.removed) + .map((shape) => shape.toJSON() as SerializedShape), + tags: Object.values(this.tags) + .reduce((accumulator, frameTags) => { + accumulator.push(...frameTags); + return accumulator; + }, []) + .filter((tag) => !tag.removed) + .map((tag) => tag.toJSON()), + intervals: this.intervals + .filter((interval) => !interval.removed) + .map((interval) => interval.toJSON()), + }; + + return data; + } + + public get(frame: number, allTracks: boolean, filters: object[]): ObjectState[] { + if (this.injection.framesInfo.isFrameDeleted(frame)) { + return []; + } + + const { tracks } = this; + const shapes = this.shapes[frame] ?? []; + const tags = this.tags[frame] ?? []; + + const objects = [].concat(tracks, shapes, tags); + const visible = []; + + for (const object of objects) { + if (object.removed) { + continue; + } + + try { + const stateData = object.get(frame); + if (stateData.outside && !stateData.keyframe && !allTracks && object instanceof Track) { + continue; + } + visible.push(stateData); + } catch (error: unknown) { + if (!(error instanceof InterpolationNotPossibleError)) { + throw error; + } + } + } + + const objectStates = []; + const filtered = this.annotationsFilter.filterSerializedObjectStates(visible, filters); + + visible.forEach((stateData) => { + if (!filters.length || filtered.includes(stateData.clientID)) { + const objectState = new ObjectState(stateData); + objectStates.push(objectState); + } + }); + + return objectStates; + } + + private _mergeInternal(objectsForMerge: (Track | Shape)[], shapeType: ShapeType, label: Label): SerializedTrack { + const keyframes: Record = {}; // frame: position + const elements = {}; // element_sublabel_id: [element], each sublabel will be merged recursively + + if (!Object.values(ShapeType).includes(shapeType)) { + throw new ArgumentError(`Got unknown shapeType "${shapeType}"`); + } + + const labelAttributes = labelAttributesAsDict(label); + for (let i = 0; i < objectsForMerge.length; i++) { + // For each state get corresponding object + const object = objectsForMerge[i]; + if (object.label.id !== label.id) { + throw new ArgumentError( + `All object labels are expected to be "${label.name}", but got "${object.label.name}"`, + ); + } + + if (object.shapeType !== shapeType) { + throw new ArgumentError( + `All shapes are expected to be "${shapeType}", but got "${object.shapeType}"`, + ); + } + + // If this object is shape, get it position and save as a keyframe + if (object instanceof Shape) { + // Frame already saved and it is not outside + if (object.frame in keyframes && !keyframes[object.frame].outside) { + throw new ArgumentError('Expected only one visible shape per frame'); + } + + keyframes[object.frame] = { + type: shapeType, + frame: object.frame, + points: object.shapeType === ShapeType.SKELETON ? undefined : [...object.points], + occluded: object.occluded, + rotation: object.rotation, + z_order: object.zOrder, + outside: false, + attributes: Object.keys(object.attributes).reduce((accumulator, attrID) => { + // We save only mutable attributes inside a keyframe + if (attrID in labelAttributes && labelAttributes[attrID].mutable) { + accumulator.push({ + spec_id: +attrID, + value: object.attributes[attrID], + }); + } + return accumulator; + }, []), + }; + + // Push outside shape after each annotation shape + // Any not outside shape will rewrite it later + if (!(object.frame + 1 in keyframes) && object.frame + 1 <= this.stopFrame) { + keyframes[object.frame + 1] = JSON.parse(JSON.stringify(keyframes[object.frame])); + keyframes[object.frame + 1].outside = true; + keyframes[object.frame + 1].frame++; + keyframes[object.frame + 1].attributes = []; + ((keyframes[object.frame + 1] as any).elements || []).forEach((el) => { + el.outside = keyframes[object.frame + 1].outside; + el.frame = keyframes[object.frame + 1].frame; + }); + } + } else if (object instanceof Track) { + // If this object is a track, iterate through all its + // keyframes and push copies to new keyframes + const attributes = {}; // id:value + const trackShapes = object.shapes; + for (const keyframe of Object.keys(trackShapes)) { + const shape = trackShapes[keyframe]; + // Frame already saved and it is not outside + if (keyframe in keyframes && !keyframes[keyframe].outside) { + // This shape is outside and non-outside shape already exists + if (shape.outside) { + continue; + } + + throw new ArgumentError('Expected only one visible shape per frame'); + } + + // We do not save an attribute if it has the same value + // We save only updates + let updatedAttributes = false; + for (const attrID in shape.attributes) { + if (!(attrID in attributes) || attributes[attrID] !== shape.attributes[attrID]) { + updatedAttributes = true; + attributes[attrID] = shape.attributes[attrID]; + } + } + + keyframes[keyframe] = { + type: shapeType, + frame: +keyframe, + points: object.shapeType === ShapeType.SKELETON ? undefined : [...shape.points], + rotation: shape.rotation, + occluded: shape.occluded, + outside: shape.outside, + z_order: shape.zOrder, + attributes: updatedAttributes ? Object.keys(attributes).reduce((accumulator, attrID) => { + accumulator.push({ + spec_id: +attrID, + value: attributes[attrID], + }); + + return accumulator; + }, []) : [], + }; + } + } else { + throw new ArgumentError( + 'Trying to merge unknown object type. Only shapes and tracks are expected.', + ); + } + + if (object.shapeType === ShapeType.SKELETON) { + for (const element of (object as unknown as SkeletonShape | SkeletonTrack).elements) { + // for each track/shape element get its first objectState and keep it + elements[element.label.id] = [ + ...(elements[element.label.id] || []), element, + ]; + } + } + } + + const mergedElements = []; + if (shapeType === ShapeType.SKELETON) { + for (const sublabel of label.structure.sublabels) { + if (!(sublabel.id in elements)) { + throw new ArgumentError( + `Merged skeleton is absent some of its elements (sublabel id: ${sublabel.id})`, + ); + } + + try { + mergedElements.push(this._mergeInternal( + elements[sublabel.id], elements[sublabel.id][0].shapeType, sublabel, + )); + } catch (error) { + throw new ArgumentError( + `Could not merge some skeleton parts (sublabel id: ${sublabel.id}). + Original error is ${error.toString()}`, + ); + } + } + } + + let firstNonOutside = false; + for (const frame of Object.keys(keyframes).sort((a, b) => +a - +b)) { + // Remove all outside frames at the begin + firstNonOutside = firstNonOutside || keyframes[frame].outside; + if (!firstNonOutside && keyframes[frame].outside) { + delete keyframes[frame]; + } else { + break; + } + } + + const track = { + frame: Math.min.apply( + null, + Object.keys(keyframes).map((frame) => +frame), + ), + shapes: Object.values(keyframes), + elements: shapeType === ShapeType.SKELETON ? mergedElements : undefined, + group: 0, + source: Source.MANUAL, + label_id: label.id, + attributes: Object.keys(objectsForMerge[0].attributes).reduce((accumulator, attrID) => { + if (!labelAttributes[attrID].mutable) { + accumulator.push({ + spec_id: +attrID, + value: objectsForMerge[0].attributes[attrID], + }); + } + + return accumulator; + }, []), + }; + + return track; + } + + public merge(objectStates: ObjectState[]): void { + checkObjectType('shapes to merge', objectStates, null, { cls: Array, name: 'Array' }); + if (!objectStates.length) return; + const objectsForMerge = objectStates.map((state) => { + checkObjectType('object state', state, null, { cls: ObjectState, name: 'ObjectState' }); + const object = this.objects[state.clientID]; + if (typeof object === 'undefined') { + throw new ArgumentError( + 'The object is not in collection yet. Call ObjectState.put([state]) before you can merge it', + ); + } + + if (state.shapeType === ShapeType.MASK) { + throw new ArgumentError( + 'Merging for masks is not supported', + ); + } + return object; + }); + + const { label, shapeType } = objectStates[0]; + if (!(label.id in this.labels)) { + throw new ArgumentError(`Unknown label for the task: ${label.id}`); + } + + const track = this._mergeInternal(objectsForMerge as (Shape | Track)[], shapeType, label); + const imported = this.import({ tracks: [track] }); + + // Remove other shapes + for (const object of objectsForMerge) { + object.removed = true; + } + + const [importedTrack] = imported.tracks; + this.history.do( + HistoryActions.MERGED_OBJECTS, + () => { + importedTrack.removed = true; + for (const object of objectsForMerge) { + object.removed = false; + } + }, + () => { + importedTrack.removed = false; + for (const object of objectsForMerge) { + object.removed = true; + } + }, + [...objectsForMerge.map((object) => object.clientID), importedTrack.clientID], + objectStates[0].frame, + ); + } + + private _splitInternal(objectState: ObjectState, object: Track, frame: number): SerializedTrack[] { + const labelAttributes = labelAttributesAsDict(object.label); + // first clear all server ids which may exist in the object being splitted + const copy = trackFactory(object.toJSON(), -1, this.injection); + copy.clearServerId(); + const exported = copy.toJSON(); + + // then create two copies, before this frame and after this frame + const prev = { + frame: exported.frame, + group: 0, + label_id: exported.label_id, + attributes: exported.attributes, + shapes: [], + source: Source.MANUAL, + elements: [], + }; + + // after this frame copy is almost the same, except of starting frame + const next = JSON.parse(JSON.stringify(prev)); + next.frame = frame; + + // get position of the object on a frame where user does split and push it to next shape + const position = { + type: objectState.shapeType, + points: objectState.shapeType === ShapeType.SKELETON ? undefined : [...objectState.points], + rotation: objectState.rotation, + occluded: objectState.occluded, + outside: objectState.outside, + z_order: objectState.zOrder, + attributes: Object.keys(objectState.attributes).reduce((accumulator, attrID) => { + if (labelAttributes[attrID].mutable) { + accumulator.push({ + spec_id: +attrID, + value: objectState.attributes[attrID], + }); + } + + return accumulator; + }, []), + frame, + }; + next.shapes.push(JSON.parse(JSON.stringify(position))); + // split all shapes of an initial object into two groups (before/after the frame) + exported.shapes.forEach((shape) => { + if (shape.frame < frame) { + prev.shapes.push(JSON.parse(JSON.stringify(shape))); + } else if (shape.frame > frame) { + next.shapes.push(JSON.parse(JSON.stringify(shape))); + } + }); + prev.shapes.push(JSON.parse(JSON.stringify(position))); + prev.shapes[prev.shapes.length - 1].outside = true; + + // do the same recursively for all object elements if there are any + + if (object instanceof SkeletonTrack) { + objectState.elements.forEach((elementState, idx) => { + const elementObject = object.elements[idx]; + const [prevEl, nextEl] = this._splitInternal(elementState, elementObject, frame); + prev.elements.push(prevEl); + next.elements.push(nextEl); + }); + } + + return [prev, next]; + } + + public split(objectState: ObjectState, frame: number): void { + checkObjectType('object state', objectState, null, { cls: ObjectState, name: 'ObjectState' }); + checkObjectType('frame', frame, 'integer', null); + + const object = this.objects[objectState.clientID] as Track; + if (typeof object === 'undefined') { + throw new ArgumentError('The object has not been saved yet. Call annotations.put([state]) before'); + } + + if (objectState.objectType !== ObjectType.TRACK) return; + const keyframes = Object.keys(object.shapes).sort((a, b) => +a - +b); + if (frame <= +keyframes[0]) return; + + const [prev, next] = this._splitInternal(objectState, object, frame); + const imported = this.import({ tracks: [prev, next] }); + + // Remove source object + object.removed = true; + + const [prevImported, nextImported] = imported.tracks; + this.history.do( + HistoryActions.SPLITTED_TRACK, + () => { + object.removed = false; + prevImported.removed = true; + nextImported.removed = true; + }, + () => { + object.removed = true; + prevImported.removed = false; + nextImported.removed = false; + }, + [object.clientID, prevImported.clientID, nextImported.clientID], + frame, + ); + } + + public group(objectStates: ObjectState[], reset: boolean): number { + checkObjectType('shapes to group', objectStates, null, { cls: Array, name: 'Array' }); + + const objectsForGroup = objectStates.map((state) => { + checkObjectType('object state', state, null, { cls: ObjectState, name: 'ObjectState' }); + const object = this.objects[state.clientID]; + if (typeof object === 'undefined') { + throw new ArgumentError('The object has not been saved yet. Call annotations.put([state]) before'); + } + return object; + }); + + const groupIdx = reset ? 0 : ++this.groupsInfo.max; + const undoGroups = objectsForGroup.map((object) => object.group); + for (const object of objectsForGroup) { + object.group = groupIdx; + object.updated = Date.now(); + } + const redoGroups = objectsForGroup.map((object) => object.group); + + this.history.do( + HistoryActions.GROUPED_OBJECTS, + () => { + objectsForGroup.forEach((object, idx) => { + object.group = undoGroups[idx]; + object.updated = Date.now(); + }); + }, + () => { + objectsForGroup.forEach((object, idx) => { + object.group = redoGroups[idx]; + object.updated = Date.now(); + }); + }, + objectsForGroup.map((object) => object.clientID), + objectStates[0].frame, + ); + + return groupIdx; + } + + public join(objectStates: ObjectState[], points: number[][]): void { + checkObjectType('shapes to join', objectStates, null, { cls: Array, name: 'Array' }); + + if (objectStates.some((state, idx) => idx && state.frame !== objectStates[idx - 1].frame)) { + throw new ArgumentError('All joined objects must be placed on the same frame'); + } + if (objectStates.some((state, idx) => idx && state.label.id !== objectStates[idx - 1].label.id)) { + throw new ArgumentError('All the objects must have the same label'); + } + + const objectsToJoin = objectStates.map((state): Shape => { + checkObjectType('object state', state, null, { cls: ObjectState, name: 'ObjectState' }); + + const object = this.objects[state.clientID]; + if (typeof object === 'undefined') { + throw new ArgumentError('The object has not been saved yet. Call annotations.put([state]) before'); + } + + if (!(object instanceof Shape)) { + throw new ArgumentError('Only shapes can be joined'); + } + + return object; + }); + + const isPolygonJoin = objectsToJoin[0] instanceof PolygonShape; + const isMaskJoin = objectsToJoin[0] instanceof MaskShape; + + if (!isPolygonJoin && !isMaskJoin) { + throw new ArgumentError('Only polygons and masks can be joined'); + } + + if (isPolygonJoin && objectsToJoin.some((obj) => !(obj instanceof PolygonShape))) { + throw new ArgumentError('Cannot join polygons with other shape types'); + } + + if (isMaskJoin && objectsToJoin.some((obj) => !(obj instanceof MaskShape))) { + throw new ArgumentError('Cannot join masks with other shape types'); + } + + if (objectsToJoin.length > 1) { + const labelAttributes = labelAttributesAsDict(objectsToJoin[0].label); + const attrValues = validateAttributesList(objectAttributesAsList(objectStates[0])); + for (const attr of attrValues) { + if (objectStates.some((state) => state.attributes[attr.spec_id] !== attr.value)) { + attr.value = labelAttributes[attr.spec_id].defaultValue; + } + } + + const shapesToCreate = []; + + for (const shapePoints of points) { + checkObjectType('joined shape points', shapePoints, null, { cls: Array, name: 'Array' }); + const shapeType = isMaskJoin ? ShapeType.MASK : ShapeType.POLYGON; + + shapesToCreate.push({ + attributes: attrValues, + frame: objectsToJoin[0].frame, + group: 0, + label_id: objectsToJoin[0].label.id, + outside: false, + occluded: objectsToJoin.some((object: any) => object.occluded), + points: shapePoints, + rotation: 0, + type: shapeType, + z_order: Math.max(...objectsToJoin.map((object: any) => object.zOrder)), + source: Source.MANUAL, + elements: [], + }); + } + + // Append newly created object(s) to the collection + const imported = this.import({ shapes: shapesToCreate }); + + // and remove joined shapes + for (const object of objectsToJoin) { + object.removed = true; + } + + // handle history actions + const importedShapes = imported.shapes; + this.history.do( + HistoryActions.JOINED_OBJECTS, + () => { + for (const importedShape of importedShapes) { + importedShape.removed = true; + } + for (const object of objectsToJoin) { + object.removed = false; + } + }, + () => { + for (const importedShape of importedShapes) { + importedShape.removed = false; + } + for (const object of objectsToJoin) { + object.removed = true; + } + }, + [ + ...objectsToJoin.map((object) => object.clientID), + ...importedShapes.map((shape) => shape.clientID), + ], + objectsToJoin[0].frame, + ); + } + } + + public slice(state: ObjectState, results: number[][]): void { + if (results.length !== 2) { + throw new Error('Not supported slicing count'); + } + + const [points1, points2] = results; + checkObjectType('sliced object', state, null, { cls: ObjectState, name: 'ObjectState' }); + checkObjectType('first slicing contour', points1, null, { cls: Array, name: 'Array' }); + checkObjectType('second slicing contour', points2, null, { cls: Array, name: 'Array' }); + + points1.forEach( + (el: number) => checkObjectType('first slicing contour element', el, 'number'), + ); + points2.forEach( + (el: number) => checkObjectType('second slicing contour element', el, 'number'), + ); + + const slicedObject = this.objects[state.clientID]; + if (!(slicedObject instanceof PolygonShape || slicedObject instanceof MaskShape)) { + throw new ArgumentError('Only polygon shape or mask shape can be sliced'); + } + + const { width, height } = this.injection.framesInfo[slicedObject.frame]; + if (slicedObject instanceof MaskShape) { + points1.push(slicedObject.left, slicedObject.top, slicedObject.right, slicedObject.bottom); + points2.push(slicedObject.left, slicedObject.top, slicedObject.right, slicedObject.bottom); + } + + const imported = this.import({ + shapes: [{ + attributes: validateAttributesList(objectAttributesAsList(state)), + frame: slicedObject.frame, + group: slicedObject.group, + label_id: slicedObject.label.id, + outside: false, + occluded: slicedObject.occluded, + points: slicedObject.shapeType === ShapeType.POLYGON ? + points1 : cropMask(points1, width, height), + rotation: 0, + type: slicedObject.shapeType, + z_order: slicedObject.zOrder, + source: Source.MANUAL, + elements: [], + }, { + attributes: validateAttributesList(objectAttributesAsList(state)), + frame: slicedObject.frame, + group: slicedObject.group, + label_id: slicedObject.label.id, + outside: false, + occluded: slicedObject.occluded, + points: slicedObject.shapeType === ShapeType.POLYGON ? + points2 : cropMask(points2, width, height), + rotation: 0, + type: slicedObject.shapeType, + z_order: slicedObject.zOrder, + source: Source.MANUAL, + elements: [], + }], + }); + slicedObject.removed = true; + + this.history.do( + HistoryActions.SLICED_OBJECT, + () => { + slicedObject.removed = false; + imported.shapes.forEach((shape) => { + shape.removed = true; + }); + }, + () => { + slicedObject.removed = true; + imported.shapes.forEach((shape) => { + shape.removed = false; + }); + }, + [...imported.shapes.map((object) => object.clientID), slicedObject.clientID], + slicedObject.frame, + ); + } + + public clear(options?: { + from?: number; + to?: number; + delTrackKeyframesOnly?: boolean; + }): void { + const { from, to, delTrackKeyframesOnly } = options ?? {}; + + if (typeof from === 'undefined' && typeof to === 'undefined') { + this.shapes = {}; + this.tags = {}; + this.tracks = []; + this.intervals = []; + this.objects = {}; + + this.flush = true; + } else { + const start = from ?? 0; + const stop = to ?? this.stopFrame; + + if (this.injection.dimension === DimensionType.DIMENSION_1D) { + this.intervals.slice(0).forEach((interval) => { + const intervalStop = interval.stop ?? this.stopFrame; + if (interval.start <= stop && intervalStop >= start) { + this.intervals.splice(this.intervals.indexOf(interval), 1); + delete this.objects[interval.clientID]; + } + }); + return; + } + + // If only a range of annotations need to be cleared + for (let frame = start; frame <= stop; frame++) { + this.shapes[frame] = []; + this.tags[frame] = []; + } + + this.tracks.slice(0).forEach((track) => { + if (track.frame <= stop) { + if (delTrackKeyframesOnly) { + for (const keyframe of Object.keys(track.shapes)) { + if (+keyframe >= start && +keyframe <= stop) { + // eslint-disable-next-line no-param-reassign + delete track.shapes[keyframe]; + if (track instanceof SkeletonTrack) { + track.elements.forEach((element) => { + if (keyframe in element.shapes) { + delete element.shapes[keyframe]; + element.updated = Date.now(); + } + }); + } + // eslint-disable-next-line no-param-reassign + track.updated = Date.now(); + } + } + + if (Object.keys(track.shapes).length === 0) { + this.tracks.splice(this.tracks.indexOf(track), 1); + } + } else if (track.frame >= from) { + this.tracks.splice(this.tracks.indexOf(track), 1); + } + } + }); + } + } + + public statistics(): Statistics { + const labels = {}; + const body = { + rectangle: { shape: 0, track: 0 }, + polygon: { shape: 0, track: 0 }, + polyline: { shape: 0, track: 0 }, + points: { shape: 0, track: 0 }, + ellipse: { shape: 0, track: 0 }, + cuboid: { shape: 0, track: 0 }, + skeleton: { shape: 0, track: 0 }, + mask: { shape: 0 }, + tag: 0, + interval: { + count: 0, + duration: 0, + coverage: 0, + }, + manually: 0, + interpolated: 0, + total: 0, + }; + + const sep = '{{cvat.skeleton.lbl.sep}}'; + const fillBody = (spec, prefix = ''): void => { + const pref = prefix ? `${prefix}${sep}` : ''; + for (const label of spec) { + const { name } = label; + labels[`${pref}${name}`] = _.cloneDeep(body); + + if (label?.structure?.sublabels) { + fillBody(label.structure.sublabels, `${pref}${name}`); + } + } + }; + + const total = _.cloneDeep(body); + fillBody(Object.values(this.labels).filter((label) => !label.hasParent)); + + const scanTrack = (track, prefix = ''): void => { + const countInterpolatedFrames = (start: number, stop: number, lastIsKeyframe: boolean): number => { + let count = stop - start; + if (lastIsKeyframe) { + count -= 1; + } + for (let i = start + 1; lastIsKeyframe ? i < stop : i <= stop; i++) { + if (this.injection.framesInfo.isFrameDeleted(i)) { + count--; + } + } + return count; + }; + + const pref = prefix ? `${prefix}${sep}` : ''; + const label = `${pref}${track.label.name}`; + labels[label][track.shapeType].track++; + const keyframes = Object.keys(track.shapes) + .sort((a, b) => +a - +b) + .map((el) => +el) + .filter((frame) => !this.injection.framesInfo.isFrameDeleted(frame)); + + if (!keyframes.length) { + return; + } + + let prevKeyframe = keyframes[0]; + let visible = false; + for (const keyframe of keyframes) { + if (visible) { + const interpolated = countInterpolatedFrames(prevKeyframe, keyframe, true); + labels[label].interpolated += interpolated; + labels[label].total += interpolated; + } + visible = !track.shapes[keyframe].outside; + prevKeyframe = keyframe; + + if (visible) { + labels[label].manually++; + labels[label].total++; + } + } + + let lastKey = keyframes[keyframes.length - 1]; + if (track.shapeType === ShapeType.SKELETON) { + track.elements.forEach((element) => { + scanTrack(element, label); + lastKey = Math.max(lastKey, ...Object.keys(element.shapes).map((key) => +key)); + }); + } + + if (lastKey !== this.stopFrame && !track.get(lastKey).outside) { + const interpolated = countInterpolatedFrames(lastKey, this.stopFrame, false); + labels[label].interpolated += interpolated; + labels[label].total += interpolated; + } + }; + + for (const object of Object.values(this.objects)) { + if (object.removed) { + continue; + } + + if (!( + object instanceof Shape || + object instanceof Track || + object instanceof Tag || + object instanceof AudioInterval + )) { + continue; + } + + const labelName = object.label.name; + if (object instanceof AudioInterval) { + const stop = object.stop ?? this.stopFrame; + labels[labelName].interval.count++; + labels[labelName].interval.duration += Math.max(0, stop - object.start); + labels[labelName].manually++; + labels[labelName].total++; + } else if (object instanceof Tag && !this.injection.framesInfo.isFrameDeleted(object.frame)) { + labels[labelName].tag++; + labels[labelName].manually++; + labels[labelName].total++; + } else if (object instanceof Track) { + scanTrack(object); + } else if (object instanceof Shape && !this.injection.framesInfo.isFrameDeleted(object.frame)) { + const { shapeType } = object; + labels[labelName][shapeType].shape++; + labels[labelName].manually++; + labels[labelName].total++; + + if (shapeType === ShapeType.SKELETON) { + (object as unknown as SkeletonShape).elements.forEach((element) => { + const combinedName = [labelName, element.label.name].join(sep); + labels[combinedName][element.shapeType].shape++; + labels[combinedName].manually++; + labels[combinedName].total++; + }); + } + } + } + + for (const label of Object.keys(labels)) { + labels[label].interval.coverage = labels[label].interval.duration / this.stopFrame; + } + + for (const label of Object.keys(labels)) { + for (const shapeType of Object.keys(labels[label])) { + if (typeof labels[label][shapeType] === 'object') { + for (const objectType of Object.keys(labels[label][shapeType])) { + total[shapeType][objectType] += labels[label][shapeType][objectType]; + } + } else { + total[shapeType] += labels[label][shapeType]; + } + } + } + + total.interval.coverage = total.interval.duration / this.stopFrame; + return new Statistics(labels, total); + } + + public put(annotationStates: AnnotationState[]): number[] { + checkObjectType('shapes for put', annotationStates, null, { cls: Array, name: 'Array' }); + const constructed = { + shapes: [], + tracks: [], + tags: [], + intervals: [], + }; + + for (const state of annotationStates) { + if (!(state instanceof ObjectState || state instanceof AudioIntervalState)) { + throw new ArgumentError('annotationState must be an ObjectState or AudioIntervalState'); + } + + if (state.clientID !== null) { + throw new ArgumentError('ObjectState.clientID must be null when adding new objects'); + } + + checkObjectType('state attributes', state.attributes, null, { cls: Object, name: 'Object' }); + checkObjectType('state label', state.label, null, { cls: Label, name: 'Label' }); + + const attributes = validateAttributesList(objectAttributesAsList(state)); + const labelAttributes = state.label.attributes.reduce((accumulator, attribute) => { + accumulator[attribute.id] = attribute; + return accumulator; + }, {}); + + // Construct whole objects from states + if (state instanceof AudioIntervalState) { + constructed.intervals.push({ + attributes, + start: state.start, + stop: Math.min(state.stop ?? this.stopFrame, this.stopFrame), + label_id: state.label.id, + group: 0, + source: state.source, + score: state.score, + }); + } else if (state.objectType === 'tag') { + constructed.tags.push({ + attributes, + frame: state.frame, + label_id: state.label.id, + group: 0, + source: state.source, + }); + } else { + checkObjectType('state rotation', state.rotation ?? 0, 'number'); + checkObjectType('state occluded', state.occluded, 'boolean'); + checkObjectType('state points', state.points, null, { cls: Array, name: 'Array' }); + checkObjectType('state zOrder', state.zOrder, 'integer'); + checkObjectType('state descriptions', state.descriptions, null, { cls: Array, name: 'Array' }); + state.descriptions.forEach((desc) => checkObjectType('state description', desc, 'string')); + + for (const coord of state.points) { + checkObjectType('point coordinate', coord, 'number'); + } + + if (!Object.values(ShapeType).includes(state.shapeType)) { + throw new ArgumentError( + `Object shape must be one of: ${JSON.stringify(Object.values(ShapeType))}`, + ); + } + + if (state.shapeType === 'mask' && state.points.length < 6) { + throw new ArgumentError('Could not create empty mask'); + } + + if (state.objectType === 'shape') { + constructed.shapes.push({ + attributes, + descriptions: state.descriptions, + frame: state.frame, + group: 0, + label_id: state.label.id, + outside: state.outside || false, + occluded: state.occluded || false, + points: state.shapeType === 'mask' ? (() => { + const { width, height } = this.injection.framesInfo[state.frame]; + return cropMask(state.points, width, height); + })() : state.points, + rotation: state.rotation || 0, + type: state.shapeType, + z_order: state.zOrder, + source: state.source, + elements: state.shapeType === 'skeleton' ? state.elements.map((element) => ({ + attributes: validateAttributesList(objectAttributesAsList(element)), + frame: element.frame, + group: 0, + label_id: element.label.id, + points: [...element.points], + rotation: 0, + type: element.shapeType, + z_order: 0, + outside: element.outside || false, + occluded: element.occluded || false, + })) : undefined, + }); + } else if (state.objectType === 'track') { + constructed.tracks.push({ + attributes: attributes.filter((attr) => !labelAttributes[attr.spec_id].mutable), + descriptions: state.descriptions, + frame: state.frame, + group: 0, + source: state.source, + label_id: state.label.id, + shapes: [ + { + attributes: attributes.filter((attr) => labelAttributes[attr.spec_id].mutable), + frame: state.frame, + occluded: false, + outside: false, + points: [...state.points], + rotation: state.rotation || 0, + type: state.shapeType, + z_order: state.zOrder, + }, + ], + elements: state.shapeType === 'skeleton' ? state.elements.map((element) => { + const elementAttrValues = validateAttributesList(objectAttributesAsList(state)); + const elementAttributes = element.label.attributes.reduce((accumulator, attribute) => { + accumulator[attribute.id] = attribute; + return accumulator; + }, {}); + + return ({ + attributes: elementAttrValues + .filter((attr) => !elementAttributes[attr.spec_id].mutable), + frame: state.frame, + group: 0, + label_id: element.label.id, + shapes: [{ + frame: state.frame, + type: element.shapeType, + points: [...element.points], + z_order: state.zOrder, + outside: element.outside || false, + occluded: element.occluded || false, + rotation: element.rotation || 0, + attributes: elementAttrValues + .filter((attr) => !elementAttributes[attr.spec_id].mutable), + }], + }); + }) : undefined, + }); + } else { + throw new ArgumentError('Object type must be one of SHAPE, TRACK, or TAG'); + } + } + } + + // Add constructed objects to a collection + const imported = this.import(constructed); + const importedArray = ([] as AnnotationObject[]).concat( + imported.tags, + imported.tracks, + imported.shapes, + imported.intervals, + ); + const additionalUndo = []; + const additionalRedo = []; + const additionalClientIDs = []; + let globalEmptyMaskOccurred = false; + for (const object of importedArray) { + if (object instanceof MaskShape && config.removeUnderlyingMaskPixels.enabled) { + const { + clientIDs, + emptyMaskOccurred, + undo: undoWithUnderlyingPixels, + redo: redoWithUnderlyingPixels, + } = object.removeUnderlyingPixels(object.frame); + + additionalUndo.push(undoWithUnderlyingPixels); + additionalRedo.push(redoWithUnderlyingPixels); + additionalClientIDs.push(clientIDs); + globalEmptyMaskOccurred = emptyMaskOccurred || globalEmptyMaskOccurred; + } + } + + if (config.removeUnderlyingMaskPixels.enabled && globalEmptyMaskOccurred) { + config.removeUnderlyingMaskPixels?.onEmptyMaskOccurrence(); + } + + if (annotationStates.length) { + const frame = annotationStates.find((state) => state instanceof ObjectState)?.frame ?? null; + this.history.do( + HistoryActions.CREATED_OBJECTS, + () => { + importedArray.forEach((object) => { + object.removed = true; + }); + additionalUndo.forEach((undo) => { + undo(); + }); + }, + () => { + importedArray.forEach((object) => { + object.removed = false; + object.serverId = undefined; + }); + + additionalRedo.forEach((redo) => { + redo(); + }); + }, + [...importedArray.map((object) => object.clientID), ...additionalClientIDs.flat()], + frame, + ); + } + + return importedArray.map((value) => value.clientID); + } + + public updateLayer(frame: number, placement: LayerPlacement, objectStates: ObjectState[]): ObjectState[] { + const parsedPlacement = parseLayerPlacement(placement); + // Validate the public inputs before reading collection state or applying any changes. + checkObjectType('frame', frame, 'integer', null); + checkObjectType('object states', objectStates, null, { cls: Array, name: 'Array' }); + objectStates.forEach((state) => { + checkObjectType('object state', state, null, { cls: ObjectState, name: 'ObjectState' }); + if (state.frame !== frame) { + throw new ArgumentError('Object state frame must match the requested frame'); + } + }); + + // Resolve requested IDs against the whole collection on the frame + // Ignore objects which cannot be moved (e.g. tags or locked) + // And perform the grouping by clientID and by layer + const { + clientId: visibleStatesByClientID, + layer: visibleStatesByLayer, + } = this.get(frame, false, []).reduce( + (accumulator, state) => { + if (!isLayerState(state) || state.lock) { + return accumulator; + } + + accumulator.clientId.set(state.clientID, state); + accumulator.layer.set(state.zOrder, accumulator.layer.get(state.zOrder) ?? []); + accumulator.layer.get(state.zOrder)?.push(state); + return accumulator; + }, { + clientId: new Map(), + layer: new Map(), + }, + ); + + // Filter the requested states to move by visibility and existence on the frame + const requestedStatesClientIds = new Set( + objectStates.map((state) => state.clientID) + .filter((clientID): clientID is number => ( + Number.isInteger(clientID) && visibleStatesByClientID.has(clientID) + )), + ); + + const requestedStates = Array.from(requestedStatesClientIds) + .map((clientID) => visibleStatesByClientID.get(clientID)); + if (!requestedStates.length) { + return []; + } + + if (parsedPlacement.kind === 'exact') { + const exactUpdates = new Map(); + requestedStates.forEach((state) => { + exactUpdates.set(state.clientID, parsedPlacement.zOrder); + }); + return this._applyZOrderUpdates(frame, exactUpdates); + } + + const updates = new Map(); + const scheduleMove = (states: ObjectState[], zOrder: number): void => { + // Find the objects already occupying the target layer, excluding the current move batch. + const movingClientIDs = new Set(states.map((state) => state.clientID)); + const displacedStates = (visibleStatesByLayer.get(zOrder) ?? []).filter((state) => ( + !movingClientIDs.has(state.clientID) && !requestedStatesClientIds.has(state.clientID) + )); + + if (displacedStates.length) { + // First make room deeper in the stack, then place this batch into the freed layer. + scheduleMove(displacedStates, zOrder + 1); + } + + // Record the planned move after deeper layers are scheduled, but before any mutation occurs. + states.forEach((state) => { + updates.set(state.clientID as number, zOrder); + }); + }; + + if (parsedPlacement.kind === 'before') { + scheduleMove(requestedStates, parsedPlacement.zOrder - 1); + } else if (parsedPlacement.kind === 'after') { + scheduleMove(requestedStates, parsedPlacement.zOrder + 1); + } + + // Apply all scheduled changes at once to preserve a single batched undo/redo action. + return this._applyZOrderUpdates(frame, updates); + } + + public compactLayers(frame: number): ObjectState[] { + checkObjectType('frame', frame, 'integer', null); + + const allStates = this.get(frame, false, []).filter((state) => isLayerState(state) && !state.lock); + const zOrderMap = new Map( + Array.from(new Set(allStates.map((state: ObjectState): number => state.zOrder))) + .sort((left: number, right: number): number => left - right) + .map((zOrder: number, index: number): [number, number] => [zOrder, index]), + ); + + const zOrders = new Map(); + allStates.forEach((state) => { + const newZOrder = zOrderMap.get(state.zOrder) as number; + if (newZOrder !== state.zOrder) { + zOrders.set(state.clientID, newZOrder); + } + }); + + return this._applyZOrderUpdates(frame, zOrders); + } + + public select(objectStates: ObjectState[], x: number, y: number): { + state: ObjectState, + distance: number | null, + } { + checkObjectType('shapes for select', objectStates, null, { cls: Array, name: 'Array' }); + checkObjectType('x coordinate', x, 'number', null); + checkObjectType('y coordinate', y, 'number', null); + + let minimumDistance = null; + let minimumState = null; + for (const state of objectStates) { + checkObjectType('object state', state, null, { cls: ObjectState, name: 'ObjectState' }); + if (state.outside || state.hidden || state.objectType === ObjectType.TAG) { + continue; + } + + let distanceMetric: typeof RectangleShape['distance'] | null = null; + switch (state.shapeType) { + case ShapeType.CUBOID: + distanceMetric = CuboidShape.distance; + break; + case ShapeType.ELLIPSE: + distanceMetric = EllipseShape.distance; + break; + case ShapeType.MASK: + distanceMetric = MaskShape.distance; + break; + case ShapeType.POINTS: + distanceMetric = PointsShape.distance; + break; + case ShapeType.POLYGON: + distanceMetric = PolygonShape.distance; + break; + case ShapeType.POLYLINE: + distanceMetric = PolylineShape.distance; + break; + case ShapeType.RECTANGLE: + distanceMetric = RectangleShape.distance; + break; + case ShapeType.SKELETON: + distanceMetric = SkeletonShape.distance; + break; + default: + throw new ArgumentError(`Unknown shape type "${state.shapeType}"`); + } + + let points = []; + if (state.shapeType === ShapeType.SKELETON) { + points = state.elements.filter((el) => !el.outside && !el.hidden).map((el) => el.points).flat(); + } else { + points = state.points; + } + const distance = distanceMetric(points, x, y, state.rotation); + if (distance !== null && (minimumDistance === null || distance < minimumDistance)) { + minimumDistance = distance; + minimumState = state; + } + } + + return { + state: minimumState, + distance: minimumDistance, + }; + } + + public selectInterval(intervalStates: AudioIntervalState[], position: number): { + state: AudioIntervalState | null, + distance: number | null, + } { + checkObjectType('intervals for select', intervalStates, null, { cls: Array, name: 'Array' }); + checkObjectType('position', position, 'number', null); + + let minimumDistance = null; + let minimumState = null; + for (const state of intervalStates) { + checkObjectType('interval state', state, null, { cls: AudioIntervalState, name: 'AudioIntervalState' }); + if (state.hidden) { + continue; + } + + const distance = AudioInterval.distance(state.start, state.stop ?? this.stopFrame, position); + if (distance !== null && (minimumDistance === null || distance < minimumDistance)) { + minimumDistance = distance; + minimumState = state; + } + } + + return { + state: minimumState, + distance: minimumDistance, + }; + } + + private _searchEmpty( + frameFrom: number, + frameTo: number, + searchParameters: { + allowDeletedFrames: boolean; + }, + ): number | null { + const { allowDeletedFrames } = searchParameters; + const sign = Math.sign(frameTo - frameFrom); + const predicate = sign > 0 ? (frame) => frame <= frameTo : (frame) => frame >= frameTo; + const update = sign > 0 ? (frame) => frame + 1 : (frame) => frame - 1; + for (let frame = frameFrom; predicate(frame); frame = update(frame)) { + if (!allowDeletedFrames && this.injection.framesInfo.isFrameDeleted(frame)) { + continue; + } + + if (frame in this.shapes && this.shapes[frame].some((shape) => !shape.removed)) { + continue; + } + + if (frame in this.tags && this.tags[frame].some((tag) => !tag.removed)) { + continue; + } + + const filteredTracks = this.tracks.filter((track) => !track.removed); + let found = false; + for (const track of filteredTracks) { + const keyframes = track.boundedKeyframes(frame); + const { prev, first } = keyframes; + const last = prev === null ? first : prev; + const lastShape = track.shapes[last]; + const isKeyfame = frame in track.shapes; + if (first <= frame && (!lastShape.outside || isKeyfame)) { + found = true; + break; + } + } + + if (found) continue; + + return frame; + } + + return null; + } + + public search( + frameFrom: number, + frameTo: number, + searchParameters: { + allowDeletedFrames: boolean; + annotationsFilters?: object[]; + generalFilters?: { + isEmptyFrame?: boolean; + }; + }, + ): number | null { + const { allowDeletedFrames } = searchParameters; + let { annotationsFilters } = searchParameters; + + if ('generalFilters' in searchParameters) { + // if we are looking for en empty frame, run a dedicated algorithm + if (searchParameters.generalFilters.isEmptyFrame) { + return this._searchEmpty(frameFrom, frameTo, { allowDeletedFrames }); + } + + // not empty frames corresponds to default behaviour of the function with empty annotation filters + annotationsFilters = []; + } + + const sign = Math.sign(frameTo - frameFrom); + const predicate = sign > 0 ? (frame) => frame <= frameTo : (frame) => frame >= frameTo; + const update = sign > 0 ? (frame) => frame + 1 : (frame) => frame - 1; + + // if not looking for an empty frame nor frame with annotations, return the next frame + // check if deleted frames are allowed additionally + if (!annotationsFilters) { + let frame = frameFrom; + while (predicate(frame)) { + if (!allowDeletedFrames && this.injection.framesInfo.isFrameDeleted(frame)) { + frame = update(frame); + continue; + } + + return frame; + } + + return null; + } + + const filtersStr = JSON.stringify(annotationsFilters); + const linearSearch = filtersStr.match(/"var":"width"/) || + filtersStr.match(/"var":"height"/) || + filtersStr.match(/"var":"rotation"/) || + filtersStr.match(/"var":"zOrder"/); + + for (let frame = frameFrom; predicate(frame); frame = update(frame)) { + if (!allowDeletedFrames && this.injection.framesInfo.isFrameDeleted(frame)) { + continue; + } + + // First prepare all data for the frame + // Consider all shapes, tags, and not outside tracks that have keyframe here + // In particular consider first and last frame as keyframes for all tracks + const statesData = [].concat( + (frame in this.shapes ? this.shapes[frame] : []) + .filter((shape) => !shape.removed) + .map((shape) => shape.get(frame)), + (frame in this.tags ? this.tags[frame] : []) + .filter((tag) => !tag.removed) + .map((tag) => tag.get(frame)), + ); + const tracks = Object.values(this.tracks) + .filter((track) => ( + frame in track.shapes || frame === frameFrom || + frame === frameTo || linearSearch)) + .filter((track) => !track.removed); + statesData.push(...tracks.map((track) => track.get(frame)).filter((state) => !state.outside)); + + // Filtering + const filtered = this.annotationsFilter.filterSerializedObjectStates(statesData, annotationsFilters); + if (filtered.length) { + return frame; + } + } + + return null; + } +} diff --git a/cvat-core/src/annotations-filter.ts b/cvat-core/src/annotations-filter.ts new file mode 100644 index 0000000..fc0e93b --- /dev/null +++ b/cvat-core/src/annotations-filter.ts @@ -0,0 +1,493 @@ +// Copyright (C) 2020-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import jsonLogic from 'json-logic-js'; +import { SerializedData } from './object-state'; +import { AttributeType, ObjectType, ShapeType } from './enums'; +import { SerializedCollection } from './server-response-types'; +import { Attribute, Label } from './labels'; +import type { AudioIntervalState } from './annotations-objects/audio-interval-state'; + +function adjustName(name): string { + return name.replace(/\./g, '\u2219'); +} + +type Dimensions = { + width: number | null; + height: number | null; +}; + +type ConvertedAttributeValue = string | number | boolean; + +interface ConvertedAttributes { + [key: string]: ConvertedAttributeValue | ConvertedAttributes; +} + +function getDimensions( + points: number[], + shapeType: ShapeType, +): Dimensions { + let [width, height]: (number | null)[] = [null, null]; + if (!points.length) { + return { width, height }; + } + + if (shapeType === ShapeType.MASK) { + const [xtl, ytl, xbr, ybr] = points.slice(-4); + [width, height] = [xbr - xtl + 1, ybr - ytl + 1]; + } else if (shapeType === ShapeType.ELLIPSE) { + const [cx, cy, rightX, topY] = points; + width = Math.abs(rightX - cx) * 2; + height = Math.abs(cy - topY) * 2; + } else { + let xtl = Number.MAX_SAFE_INTEGER; + let xbr = Number.MIN_SAFE_INTEGER; + let ytl = Number.MAX_SAFE_INTEGER; + let ybr = Number.MIN_SAFE_INTEGER; + + points.forEach((coord, idx) => { + if (idx % 2) { + // y + ytl = Math.min(ytl, coord); + ybr = Math.max(ybr, coord); + } else { + // x + xtl = Math.min(xtl, coord); + xbr = Math.max(xbr, coord); + } + }); + [width, height] = [xbr - xtl, ybr - ytl]; + } + + return { + width, + height, + }; +} + +function convertAttributes( + attributes: SerializedData['attributes'] | SerializedCollection['shapes'][0]['attributes'], + attributesSpec: Record, +): ConvertedAttributes { + const entries: [number, string][] = Array.isArray(attributes) ? + attributes.map(({ spec_id, value }) => [spec_id, value]) : + Object.keys(attributes).map((key) => [+key, attributes[key]]); + + return entries.reduce((acc, [id, value]) => { + const spec = attributesSpec[id]; + const name = adjustName(spec.name); + if (spec.inputType === AttributeType.NUMBER) { + acc[name] = +value; + } else if (spec.inputType === AttributeType.CHECKBOX) { + acc[name] = value === 'true'; + } else { + acc[name] = value; + } + + return acc; + }, {} as Record); +} + +function buildAttributeMap(attributes: Attribute[]): Record { + return attributes.reduce((acc, attribute) => { + if (typeof attribute.id === 'number') { + acc[attribute.id] = attribute; + } + return acc; + }, {} as Record); +} + +function buildLabelMaps(labels: Label[]): { + labelByID: Record; + attributeByID: Record; +} { + const labelByID: Record = {}; + const attributeByID: Record = {}; + + const registerLabel = (label: Label): void => { + if (typeof label.id === 'number') { + labelByID[label.id] = label; + } + + label.attributes.forEach((attribute) => { + if (typeof attribute.id === 'number') { + attributeByID[attribute.id] = attribute; + } + }); + }; + + labels.forEach((label) => { + registerLabel(label); + label.structure?.sublabels.forEach(registerLabel); + }); + + return { labelByID, attributeByID }; +} + +interface BaseConvertedData { + width: number | null; + height: number | null; + rotation: number | null; + attr: Record; + label: string; + type: ObjectType | null; + shape: ShapeType | null; + occluded: boolean | null; + score: number | null; + votes: number | null; + zOrder: number | null; +} + +interface ConvertedElementData extends BaseConvertedData { + objectID: number | null; +} + +interface ConvertedObjectData extends BaseConvertedData { + serverID: number | null; + objectID: number | null; + elements: ConvertedElementData[]; +} + +interface ConvertedAudioIntervalData { + clientID: number | null; + attr: Record; + duration: number; + end: number; + label: string; + serverID: number | null; + source: string | null; + start: number; +} + +function getRotation(shapeType: ShapeType, rotation?: number | null): number | null { + return shapeType === ShapeType.RECTANGLE || shapeType === ShapeType.ELLIPSE ? rotation ?? null : null; +} + +function isEmptyFilter(filter: object | undefined): boolean { + return !filter || !Object.keys(filter).length; +} + +function getMatchingIDs( + entries: ConvertedObjectData[], + objectFilter?: object, + keypointFilter?: object, +): number[] { + const matchingIDs = new Set(); + entries.forEach((entry) => { + const objectMatches = isEmptyFilter(objectFilter) || jsonLogic.apply(objectFilter, entry); + const keypointsMatch = isEmptyFilter(keypointFilter) || + entry.elements.some((element) => jsonLogic.apply(keypointFilter, element)); + if (typeof entry.objectID === 'number' && objectMatches && keypointsMatch) { + matchingIDs.add(entry.objectID); + } + }); + return [...matchingIDs]; +} + +export default class AnnotationsFilter { + private lastPosition: number | null; + + constructor(lastPosition: number | null) { + this.lastPosition = lastPosition; + } + + private _convertAudioIntervalStates( + intervalsData: AudioIntervalState[], + ): ConvertedAudioIntervalData[] { + if (this.lastPosition === null) { + throw new Error('Last position is required to filter audio intervals'); + } + + return intervalsData.map((interval) => { + const labelAttributes = buildAttributeMap(interval.label.attributes); + const attributes = convertAttributes(interval.attributes, labelAttributes); + const end = interval.stop ?? this.lastPosition; + + return { + clientID: interval.clientID, + attr: { + [adjustName(interval.label.name)]: attributes, + }, + duration: end - interval.start, + end, + label: interval.label.name, + serverID: interval.serverID ?? null, + source: interval.source ?? null, + start: interval.start, + }; + }); + } + + private _convertSerializedObjectStates(statesData: SerializedData[]): ConvertedObjectData[] { + return statesData.map((state) => { + const labelAttributes = buildAttributeMap(state.label.attributes); + + let dimensions: Dimensions = { width: null, height: null }; + let rotation: number | null = null; + if (state.objectType !== ObjectType.TAG) { + const points = + state.shapeType === ShapeType.SKELETON ? + (state.elements ?? []) + .map((element) => element.points ?? []) + .flat() : + state.points; + + dimensions = getDimensions(points ?? [], state.shapeType as ShapeType); + rotation = getRotation(state.shapeType, state.rotation); + } + + const attributes = convertAttributes(state.attributes || {}, labelAttributes); + const elements: ConvertedElementData[] = state.shapeType === ShapeType.SKELETON && state.elements ? + state.elements.map((element) => { + const elementLabelAttributes = buildAttributeMap(element.label.attributes); + const sublabelName = `${state.label.name} / ${element.label.name}`; + const elementAttributes = convertAttributes(element.attributes || {}, elementLabelAttributes); + + return { + width: null, + height: null, + rotation: null, + attr: { + [adjustName(sublabelName)]: elementAttributes, + }, + label: sublabelName, + objectID: element.clientID ?? null, + type: null, + shape: null, + occluded: element.occluded ?? false, + score: null, + votes: null, + zOrder: null, + }; + }) : + []; + + return { + width: dimensions.width, + height: dimensions.height, + rotation, + attr: { + [adjustName(state.label.name)]: attributes, + }, + label: state.label.name, + serverID: state.serverID ?? null, + objectID: state.clientID ?? null, + type: state.objectType, + shape: state.shapeType ?? null, + occluded: state.occluded ?? null, + score: state.score ?? null, + votes: state.votes ?? null, + zOrder: state.zOrder ?? null, + elements, + }; + }); + } + + private _convertSerializedCollection( + collection: Pick, + labelsSpec: Label[], + ): { + shapes: ConvertedObjectData[]; + tags: ConvertedObjectData[]; + tracks: ConvertedObjectData[]; + } { + const { labelByID, attributeByID } = buildLabelMaps(labelsSpec); + + return { + shapes: collection.shapes.map((shape) => { + const label = labelByID[shape.label_id]; + const points = + shape.type === ShapeType.SKELETON ? + shape.elements.map((el) => el.points ?? []).flat() : + shape.points; + const dimensions = getDimensions(points ?? [], shape.type); + const attributes = convertAttributes(shape.attributes, attributeByID); + + const elements: ConvertedElementData[] = shape.type === ShapeType.SKELETON && shape.elements ? + shape.elements.flatMap((element) => { + const elementLabel = labelByID[element.label_id]; + if (!elementLabel) { + return []; + } + + const sublabelName = `${label.name} / ${elementLabel.name}`; + const elementAttributes = convertAttributes(element.attributes, attributeByID); + + return [{ + width: null, + height: null, + rotation: null, + attr: { + [adjustName(sublabelName)]: elementAttributes, + }, + label: sublabelName, + objectID: null, + type: null, + shape: null, + occluded: element.occluded ?? false, + score: null, + votes: null, + zOrder: null, + }]; + }) : + []; + + return { + width: dimensions.width, + height: dimensions.height, + rotation: getRotation(shape.type, shape.rotation), + attr: { + [adjustName(label.name)]: attributes, + }, + label: label.name, + serverID: shape.id ?? null, + objectID: shape.clientID ?? null, + type: ObjectType.SHAPE, + shape: shape.type, + occluded: shape.occluded, + score: shape.score ?? null, + votes: null, + zOrder: shape.z_order, + elements, + }; + }), + tags: collection.tags.map((tag) => { + const label = labelByID[tag.label_id]; + const attributes = convertAttributes(tag.attributes, attributeByID); + + return { + width: null, + height: null, + rotation: null, + attr: { + [adjustName(label.name)]: attributes, + }, + label: label.name, + serverID: tag.id ?? null, + objectID: tag.clientID ?? null, + type: ObjectType.TAG, + shape: null, + occluded: false, + score: null, + votes: null, + zOrder: 0, + elements: [], + }; + }), + tracks: collection.tracks.map((track) => { + const label = labelByID[track.label_id]; + const attributes = convertAttributes(track.attributes, attributeByID); + + let elements: ConvertedElementData[] = []; + if (track.shapes[0]?.type === ShapeType.SKELETON && track.elements) { + elements = track.elements.flatMap((element) => { + const elementLabel = labelByID[element.label_id]; + if (!elementLabel) { + return []; + } + + const sublabelName = `${label.name} / ${elementLabel.name}`; + const elementAttributes = convertAttributes(element.attributes, attributeByID); + + return [{ + width: null, + height: null, + rotation: null, + attr: { + [adjustName(sublabelName)]: elementAttributes, + }, + label: sublabelName, + objectID: null, + type: null, + shape: null, + occluded: null, + score: null, + votes: null, + zOrder: null, + }]; + }); + } + + return { + width: null, + height: null, + rotation: null, + attr: { + [adjustName(label.name)]: attributes, + }, + label: label.name, + serverID: track.id ?? null, + objectID: track.clientID ?? null, + type: ObjectType.TRACK, + shape: track.shapes[0]?.type ?? null, + occluded: null, + score: null, + votes: null, + zOrder: track.shapes[0]?.z_order ?? null, + elements, + }; + }), + }; + } + + public filterSerializedObjectStates(statesData: SerializedData[], filters: object[]): number[] { + if (isEmptyFilter(filters[0]) && isEmptyFilter(filters[1])) { + return statesData.map((stateData): number => stateData.clientID); + } + + const converted = this._convertSerializedObjectStates(statesData); + return getMatchingIDs(converted, filters[0], filters[1]); + } + + public filterAudioIntervalStates(audioIntervalStates: AudioIntervalState[], filters: object[]): number[] { + if (isEmptyFilter(filters[0])) { + return audioIntervalStates.map((intervalData): number => intervalData.clientID!); + } + + const converted = this._convertAudioIntervalStates(audioIntervalStates); + return converted.filter((interval) => jsonLogic.apply(filters[0], interval)) + .map((interval) => interval.clientID); + } + + public filterSerializedCollection( + collection: Pick, + labelsSpec: Label[], + filters: object[], + ): { shapes: number[]; tags: number[]; tracks: number[]; } { + if (isEmptyFilter(filters[0]) && isEmptyFilter(filters[1])) { + return { + shapes: collection.shapes.map((shape) => shape.clientID), + tags: collection.tags.map((tag) => tag.clientID), + tracks: collection.tracks.map((track) => track.clientID), + }; + } + + const converted = this._convertSerializedCollection(collection, labelsSpec); + + return { + shapes: getMatchingIDs(converted.shapes, filters[0], filters[1]), + tags: getMatchingIDs(converted.tags, filters[0], filters[1]), + tracks: getMatchingIDs(converted.tracks, filters[0], filters[1]), + }; + } + + public filterSerializedSkeletonElements(statesData: SerializedData[], filters: object[]): Record { + if (isEmptyFilter(filters[1])) { + return {}; + } + + const filter = filters[1]; + const converted = this._convertSerializedObjectStates(statesData); + return converted.reduce((acc, entry) => { + if (entry.shape === ShapeType.SKELETON && typeof entry.objectID === 'number') { + acc[entry.objectID] = entry.elements + .filter((element) => typeof element.objectID === 'number' && jsonLogic.apply(filter, element)) + .map((element) => element.objectID as number); + } + + return acc; + }, {} as Record); + } +} diff --git a/cvat-core/src/annotations-history.ts b/cvat-core/src/annotations-history.ts new file mode 100644 index 0000000..17f76cf --- /dev/null +++ b/cvat-core/src/annotations-history.ts @@ -0,0 +1,100 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { HistoryActions } from './enums'; + +const MAX_HISTORY_LENGTH = 32; + +interface ActionItem { + action: HistoryActions; + clientIds: number[]; + frame: number; + undo: () => void; + redo: () => void; +} + +export default class AnnotationHistory { + private frozen: boolean; + private _undo: ActionItem[]; + private _redo: ActionItem[]; + + constructor() { + this.frozen = false; + this.clear(); + } + + public freeze(frozen: boolean): void { + this.frozen = frozen; + } + + public get(): { + undo: [HistoryActions, number | null][], + redo: [HistoryActions, number | null][], + } { + return { + undo: this._undo.map((undo) => [undo.action, undo.frame]), + redo: this._redo.map((redo) => [redo.action, redo.frame]), + }; + } + + public do( + action: HistoryActions, + undo: () => void, + redo: () => void, + clientIds: number[], + frame: number | null, + ): void { + if (this.frozen) return; + + const actionItem: ActionItem = { + clientIds, + action, + undo, + redo, + frame, + }; + + this._undo = this._undo.slice(-MAX_HISTORY_LENGTH + 1); + this._undo.push(actionItem); + this._redo = []; + } + + public async undo(count: number): Promise { + const affectedObjects = []; + for (let i = 0; i < count; i++) { + const action = this._undo.pop(); + if (action) { + await action.undo(); + this._redo.push(action); + affectedObjects.push(...action.clientIds); + } else { + break; + } + } + + return affectedObjects; + } + + public async redo(count: number): Promise { + const affectedObjects = []; + for (let i = 0; i < count; i++) { + const action = this._redo.pop(); + if (action) { + await action.redo(); + this._undo.push(action); + affectedObjects.push(...action.clientIds); + } else { + break; + } + } + + return affectedObjects; + } + + public clear(): void { + this._undo = []; + this._redo = []; + } +} diff --git a/cvat-core/src/annotations-objects/annotation-common.ts b/cvat-core/src/annotations-objects/annotation-common.ts new file mode 100644 index 0000000..da52caa --- /dev/null +++ b/cvat-core/src/annotations-objects/annotation-common.ts @@ -0,0 +1,268 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { type Label } from '../labels'; +import { + colors, Source, HistoryActions, +} from '../enums'; +import { AnnotationContext } from './annotation-context'; +import type { AnnotationInjection, CommonUpdateFlags } from './types'; +import { computeNewSource, defaultGroupColor, deserializeAttributes } from './utils'; + +// Stores common annotation identity/state and field history mutations. +export class AnnotationBase extends AnnotationContext { + private _removed: boolean; + + protected _serverId?: number; + protected _parentId?: number; + protected color: string; + protected readOnlyFields: string[]; + protected groupObject: { + color: string; + readonly id: number; + }; + + public readonly clientID: number; + public group: number; + public label: Label; + public lock: boolean; + public hidden: boolean; + public source: Source; + public updated: number; + public attributes: Map; + + constructor(data, clientID: number, color: string, injection: AnnotationInjection) { + super(injection); + this._removed = false; + + this._serverId = data.id ?? undefined; + this._parentId = injection.parentId ?? undefined; + this.color = color; + this.readOnlyFields = []; + this.groupObject = Object.defineProperties( + {}, { + color: { + get: () => { + if (this.group) { + return this.groupsInfo.colors[this.group] || colors[this.group % colors.length]; + } + return defaultGroupColor; + }, + set: (newColor) => { + if (this.group && typeof newColor === 'string' && /^#[0-9A-F]{6}$/i.test(newColor)) { + this.groupsInfo.colors[this.group] = newColor; + this.updated = Date.now(); + } + }, + }, + id: { + get: () => this.group, + }, + }, + ) as AnnotationBase['groupObject']; + + this.clientID = clientID; + this.group = data.group; + this.label = this.labels[data.label_id]; + this.lock = false; + this.hidden = false; + this.source = data.source; + this.updated = Date.now(); + this.attributes = deserializeAttributes(data.attributes); + + this.appendDefaultAttributes(this.label); + // eslint-disable-next-line no-param-reassign + injection.groupsInfo.max = Math.max(injection.groupsInfo.max, this.group); + } + + protected saveLock(lock: boolean, frame: number | null): void { + const undoLock = this.lock; + const redoLock = lock; + + this.history.do( + HistoryActions.CHANGED_LOCK, + () => { + this.lock = undoLock; + this.updated = Date.now(); + }, + () => { + this.lock = redoLock; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.lock = lock; + } + + protected saveHidden(hidden: boolean, frame: number | null): void { + const undoHidden = this.hidden; + const redoHidden = hidden; + + this.history.do( + HistoryActions.CHANGED_HIDDEN, + () => { + this.hidden = undoHidden; + this.updated = Date.now(); + }, + () => { + this.hidden = redoHidden; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.hidden = hidden; + } + + protected saveColor(color: string, frame: number | null): void { + const undoColor = this.color; + const redoColor = color; + + this.history.do( + HistoryActions.CHANGED_COLOR, + () => { + this.color = undoColor; + this.updated = Date.now(); + }, + () => { + this.color = redoColor; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.color = color; + } + + protected saveLabel(label: Label, frame: number | null): void { + const undoLabel = this.label; + const redoLabel = label; + const undoAttributes = new Map(this.attributes); + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + this.label = label; + this.source = redoSource; + this.attributes = new Map(); + this.appendDefaultAttributes(label); + const redoAttributes = new Map(this.attributes); + + this.history.do( + HistoryActions.CHANGED_LABEL, + () => { + this.label = undoLabel; + this.attributes = undoAttributes; + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + this.label = redoLabel; + this.attributes = redoAttributes; + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + } + + protected saveAttributes(attributes: Record, frame: number | null): void { + const undoAttributes = new Map(this.attributes); + for (const [id, value] of Object.entries(attributes)) { + this.attributes.set(+id, value); + } + const redoAttributes = new Map(this.attributes); + + this.history.do( + HistoryActions.CHANGED_ATTRIBUTES, + () => { + this.attributes = undoAttributes; + this.updated = Date.now(); + }, + () => { + this.attributes = redoAttributes; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + } + + protected appendDefaultAttributes(label: Label): void { + const labelAttributes = label.attributes; + for (const attribute of labelAttributes) { + if (typeof attribute.id === 'number' && !this.attributes.has(attribute.id)) { + this.attributes.set(attribute.id, attribute.defaultValue); + } + } + } + + public clearServerId(): void { + this._serverId = undefined; + } + + protected updateTimestamp(updated: T): void { + const anyChanges = Object.values(updated).some((value) => value); + if (anyChanges) { + this.updated = Date.now(); + } + } + + protected withContext(): { + delete: AnnotationBase['delete']; + } { + return { + delete: this.delete.bind(this), + }; + } + + public delete(frame: number | null, force: boolean): boolean { + if (!this.lock || force) { + this.removed = true; + this.history.do( + HistoryActions.REMOVED_OBJECT, + () => { + this.removed = false; + this.updated = Date.now(); + }, + () => { + this.removed = true; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + } + + return this.removed; + } + + get removed(): boolean { + return this._removed; + } + + set removed(value: boolean) { + if (value) { + this.clearServerId(); + } + + this._removed = value; + } + + get clientId(): number { + return this.clientID; + } + + get serverId(): number | undefined { + return this._serverId; + } + + set serverId(id: number | undefined) { + this._serverId = id; + } +} diff --git a/cvat-core/src/annotations-objects/annotation-context.ts b/cvat-core/src/annotations-objects/annotation-context.ts new file mode 100644 index 0000000..12d9ece --- /dev/null +++ b/cvat-core/src/annotations-objects/annotation-context.ts @@ -0,0 +1,31 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type { AnnotationInjection } from './types'; + +// Stores shared annotation infrastructure injected by the collection. +// It intentionally has no behavior so derived classes can access context without owning object state. +export class AnnotationContext { + protected labels: AnnotationInjection['labels']; + protected groupsInfo: AnnotationInjection['groupsInfo']; + protected framesInfo: AnnotationInjection['framesInfo']; + protected history: AnnotationInjection['history']; + protected dimension: AnnotationInjection['dimension']; + protected jobType: AnnotationInjection['jobType']; + protected replicasCount: AnnotationInjection['replicasCount']; + protected nextClientID: AnnotationInjection['nextClientID']; + protected getMasksOnFrame: AnnotationInjection['getMasksOnFrame']; + + constructor(injection: AnnotationInjection) { + this.labels = injection.labels; + this.groupsInfo = injection.groupsInfo; + this.framesInfo = injection.framesInfo; + this.history = injection.history; + this.dimension = injection.dimension; + this.jobType = injection.jobType; + this.replicasCount = injection.replicasCount; + this.nextClientID = injection.nextClientID; + this.getMasksOnFrame = injection.getMasksOnFrame; + } +} diff --git a/cvat-core/src/annotations-objects/audio-interval-state.ts b/cvat-core/src/annotations-objects/audio-interval-state.ts new file mode 100644 index 0000000..821a5af --- /dev/null +++ b/cvat-core/src/annotations-objects/audio-interval-state.ts @@ -0,0 +1,338 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import PluginRegistry from '../plugins'; +import { ArgumentError } from '../exceptions'; +import { Label } from '../labels'; +import { ObjectType, Source } from '../enums'; +import type { SerializedInterval } from '../server-response-types'; +import type { AudioIntervalUpdateFlags } from './types'; + +export interface SerializedAudioIntervalState { + objectType: ObjectType.INTERVAL; + label: Label; + clientID: number | null; + serverID: number | null; + start: number; + stop: number | null; + color: string; + lock: boolean; + updated: number; + source: Source; + score: number; + votes: number; + hidden: boolean; + attributes: Record; + __internal?: { + save: (audioIntervalState: AudioIntervalState) => AudioIntervalState; + delete: (frame: number | null, force: boolean) => boolean; + export: () => SerializedInterval; + }; +} + +export class AudioIntervalState { + private readonly __internal: SerializedAudioIntervalState['__internal']; + + public readonly updateFlags: AudioIntervalUpdateFlags; + public readonly objectType: ObjectType.INTERVAL; + public readonly source: Source; + public readonly clientID: number | null; + public readonly serverID: number | null; + public readonly updated: number; + public readonly score: number; + public readonly votes: number; + public label: Label; + public start: number; + public stop: number | null; + public color: string; + public hidden: boolean; + public lock: boolean; + public attributes: Record; + + constructor(serialized: SerializedAudioIntervalState) { + if (serialized.objectType !== ObjectType.INTERVAL) { + throw new ArgumentError( + `AudioIntervalState must be provided correct ObjectType (INTERVAL), got ${serialized.objectType}`, + ); + } + + if (!(serialized.label instanceof Label)) { + throw new ArgumentError( + `AudioIntervalState must be provided correct Label, got wrong value ${serialized.label}`, + ); + } + + if (typeof serialized.start !== 'number') { + throw new ArgumentError( + `AudioIntervalState must be provided correct start, got wrong value ${serialized.start}`, + ); + } + + if (serialized.stop !== null && typeof serialized.stop !== 'number') { + throw new ArgumentError( + `AudioIntervalState must be provided correct stop, got wrong value ${serialized.stop}`, + ); + } + + const updateFlags: AudioIntervalUpdateFlags = { + reset() { + delete this.label; + delete this.attributes; + delete this.position; + delete this.lock; + delete this.color; + delete this.hidden; + }, + }; + + Object.defineProperty(updateFlags, 'reset', { + enumerable: false, + writable: false, + }); + + const data = { + label: serialized.label, + attributes: {}, + start: serialized.start, + stop: serialized.stop, + lock: serialized.lock, + color: serialized.color, + hidden: serialized.hidden, + source: serialized.source, + updated: serialized.updated, + clientID: serialized.clientID, + serverID: serialized.serverID, + objectType: serialized.objectType, + updateFlags, + score: serialized.score, + votes: serialized.votes, + }; + + Object.defineProperties( + this, + Object.freeze({ + updateFlags: { + get: () => data.updateFlags, + }, + objectType: { + get: () => data.objectType, + }, + source: { + get: () => data.source, + }, + score: { + get: () => data.score, + }, + votes: { + get: () => data.votes, + }, + clientID: { + get: () => data.clientID, + }, + serverID: { + get: () => data.serverID, + }, + label: { + get: () => data.label, + set: (label) => { + data.updateFlags.label = true; + if (!(label instanceof Label)) { + throw new ArgumentError( + `Label must be an instance of Label class, got ${typeof label}`, + ); + } + + if (label.id! === data.label.id) { + return; + } + + data.label = label; + }, + }, + start: { + get: () => data.start, + set: (start) => { + if (typeof start !== 'number') { + throw new ArgumentError('Start is expected to be a number.'); + } + + if (start === data.start) { + return; + } + + data.updateFlags.position = true; + data.start = start; + }, + }, + stop: { + get: () => data.stop, + set: (stop) => { + if (stop !== null && typeof stop !== 'number') { + throw new ArgumentError('Stop is expected to be a number or null.'); + } + + if (stop === data.stop) { + return; + } + + data.updateFlags.position = true; + data.stop = stop; + }, + }, + color: { + get: () => data.color, + set: (color) => { + data.updateFlags.color = true; + data.color = color; + }, + }, + hidden: { + get: () => data.hidden, + set: (hidden) => { + if (typeof hidden !== 'boolean') { + throw new ArgumentError('Hidden is expected to be a boolean.'); + } + + if (hidden === data.hidden) { + return; + } + + data.updateFlags.hidden = true; + data.hidden = hidden; + }, + }, + lock: { + get: () => data.lock, + set: (lock) => { + if (typeof lock !== 'boolean') { + throw new ArgumentError('Lock is expected to be a boolean.'); + } + + if (lock === data.lock) { + return; + } + + data.updateFlags.lock = true; + data.lock = lock; + }, + }, + updated: { + get: () => data.updated, + }, + attributes: { + get: () => data.attributes, + set: (attributes) => { + if (typeof attributes !== 'object') { + throw new ArgumentError( + 'Attributes are expected to be an object ' + + `but got ${ + typeof attributes === 'object' ? + attributes.constructor.name : + typeof attributes + }`, + ); + } + + for (const attrID of Object.keys(attributes)) { + data.updateFlags.attributes = true; + data.attributes[attrID] = attributes[attrID]; + } + }, + }, + }), + ); + + if (typeof serialized.hidden === 'boolean') { + data.hidden = serialized.hidden; + } + if (typeof serialized.color === 'string') { + data.color = serialized.color; + } + if (typeof serialized.attributes === 'object') { + data.attributes = serialized.attributes; + } + + /* eslint-disable-next-line no-underscore-dangle */ + if (serialized.__internal) { + /* eslint-disable-next-line no-underscore-dangle */ + this.__internal = serialized.__internal; + } + } + + async save(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, AudioIntervalState.prototype.save); + return result; + } + + async delete(force = false): Promise { + const result = await PluginRegistry.apiWrapper.call(this, AudioIntervalState.prototype.delete, force); + return result; + } + + async export(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, AudioIntervalState.prototype.export); + return result; + } + + /** + * Creates a new unsaved audio interval state from user input. + * The returned state is intended to be passed to AnnotationCollection.put(). + */ + static create(body: { + label: Label; + start: number; + stop: number | null; + source: Source; + }): AudioIntervalState { + return new AudioIntervalState({ + objectType: ObjectType.INTERVAL, + clientID: null, + serverID: null, + label: body.label, + start: body.start, + stop: body.stop, + color: body.label.color!, + lock: false, + updated: Date.now(), + source: body.source, + score: 1, + votes: 0, + hidden: false, + attributes: {}, + }); + } +} + +Object.defineProperty(AudioIntervalState.prototype.save, 'implementation', { + value: function saveImplementation(): AudioIntervalState { + if (this.__internal && this.__internal.save) { + return this.__internal.save(this); + } + + throw new Error('Could not save audio interval state. Context is not provided.'); + }, + writable: false, +}); + +Object.defineProperty(AudioIntervalState.prototype.export, 'implementation', { + value: function exportImplementation(): SerializedInterval | AudioIntervalState { + if (this.__internal && this.__internal.export) { + return this.__internal.export(); + } + + throw new Error('Could not export audio interval state. Context is not provided.'); + }, + writable: false, +}); + +Object.defineProperty(AudioIntervalState.prototype.delete, 'implementation', { + value: function deleteImplementation(force: boolean): boolean { + if (this.__internal && this.__internal.delete) { + return this.__internal.delete(null, force); + } + + throw new Error('Could not delete audio interval state. Context is not provided.'); + }, + writable: false, +}); diff --git a/cvat-core/src/annotations-objects/audio-interval.ts b/cvat-core/src/annotations-objects/audio-interval.ts new file mode 100644 index 0000000..8babd30 --- /dev/null +++ b/cvat-core/src/annotations-objects/audio-interval.ts @@ -0,0 +1,149 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type { SerializedInterval } from '../server-response-types'; +import { HistoryActions, ObjectType } from '../enums'; +import { AnnotationBase } from './annotation-common'; +import { AudioIntervalState } from './audio-interval-state'; +import type { AnnotationInjection } from './types'; +import { ScoredMixin } from './scored'; +import { serializeAttributes } from './utils'; + +export class AudioInterval extends ScoredMixin(AnnotationBase) { + public start: number; + public stop: number | null; + + public static distance(start: number, stop: number, position: number): number | null { + if (position < start || position > stop) { + return null; + } + + return Math.min(Math.abs(position - start), Math.abs(position - stop)); + } + + constructor(data: SerializedInterval, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.start = data.start; + this.stop = data.stop; + } + + protected withContext(): ReturnType & { + save: (data: AudioIntervalState) => AudioIntervalState; + export: () => SerializedInterval; + } { + return { + delete: this.delete.bind(this), + save: this.save.bind(this), + export: this.toJSON.bind(this), + }; + } + + protected savePosition(start: number, stop: number | null): void { + const undoStart = this.start; + const undoStop = this.stop; + this.start = start; + this.stop = stop; + + this.history.do( + HistoryActions.CHANGED_AUDIO_POSITION, + () => { + this.start = undoStart; + this.stop = undoStop; + this.updated = Date.now(); + }, + () => { + this.start = start; + this.stop = stop; + this.updated = Date.now(); + }, + [this.clientID], + null, + ); + } + + public toJSON(): SerializedInterval { + const result: SerializedInterval = { + clientID: this.clientID, + label_id: this.label.id, + start: this.start, + stop: this.stop, + group: this.groupObject.id, + source: this.source, + score: this.score, + attributes: serializeAttributes(this.attributes), + }; + + if (typeof this._serverId === 'number') { + result.id = this._serverId; + } + + return result; + } + + public get(): AudioIntervalState { + return new AudioIntervalState({ + objectType: ObjectType.INTERVAL, + clientID: this.clientID, + serverID: this._serverId ?? null, + label: this.label, + start: this.start, + stop: this.stop, + color: this.color, + lock: this.lock, + hidden: this.hidden, + updated: this.updated, + source: this.source, + score: this.score, + votes: this.votes, + attributes: Object.fromEntries(this.attributes), + __internal: this.withContext(), + }); + } + + public updateFromServerResponse(body: { id: number }): void { + this._serverId = body.id; + } + + public save(data: AudioIntervalState): AudioIntervalState { + if (this.lock && data.lock) { + return this.get(); + } + + const updated = data.updateFlags; + for (const readOnlyField of this.readOnlyFields) { + delete updated[readOnlyField]; + } + + // this.validateStateBeforeSave(data, updated); + + if (updated.label) { + this.saveLabel(data.label, null); + } + + if (updated.attributes) { + this.saveAttributes(data.attributes, null); + } + + if (updated.lock) { + this.saveLock(data.lock, null); + } + + if (updated.color) { + this.saveColor(data.color, null); + } + + if (updated.position) { + this.savePosition(data.start, data.stop); + } + + if (updated.hidden) { + this.saveHidden(data.hidden, null); + } + + this.updateTimestamp(updated); + updated.reset(); + + return this.get(); + } +} diff --git a/cvat-core/src/annotations-objects/cuboid-shape.ts b/cvat-core/src/annotations-objects/cuboid-shape.ts new file mode 100644 index 0000000..571973f --- /dev/null +++ b/cvat-core/src/annotations-objects/cuboid-shape.ts @@ -0,0 +1,133 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { checkNumberOfPoints } from '../object-utils'; +import { Shape } from './shape'; +import type { AnnotationInjection, Point2D } from './types'; + +export class CuboidShape extends Shape { + constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.rotation = 0; + this.shapeType = ShapeType.CUBOID; + this.pinned = false; + checkNumberOfPoints(this.shapeType, this.points); + } + + static makeHull(geoPoints: Point2D[]): Point2D[] { + // Returns the convex hull, assuming that each points[i] <= points[i + 1]. + function makeHullPresorted(points: Point2D[]): Point2D[] { + if (points.length <= 1) return points.slice(); + + // Andrew's monotone chain algorithm. Positive y coordinates correspond to 'up' + // as per the mathematical convention, instead of 'down' as per the computer + // graphics convention. This doesn't affect the correctness of the result. + + const upperHull = []; + for (let i = 0; i < points.length; i += 1) { + const p = points[`${i}`]; + while (upperHull.length >= 2) { + const q = upperHull[upperHull.length - 1]; + const r = upperHull[upperHull.length - 2]; + if ((q.x - r.x) * (p.y - r.y) >= (q.y - r.y) * (p.x - r.x)) upperHull.pop(); + else break; + } + upperHull.push(p); + } + upperHull.pop(); + + const lowerHull = []; + for (let i = points.length - 1; i >= 0; i -= 1) { + const p = points[`${i}`]; + while (lowerHull.length >= 2) { + const q = lowerHull[lowerHull.length - 1]; + const r = lowerHull[lowerHull.length - 2]; + if ((q.x - r.x) * (p.y - r.y) >= (q.y - r.y) * (p.x - r.x)) lowerHull.pop(); + else break; + } + lowerHull.push(p); + } + lowerHull.pop(); + + if ( + upperHull.length === 1 && + lowerHull.length === 1 && + upperHull[0].x === lowerHull[0].x && + upperHull[0].y === lowerHull[0].y + ) return upperHull; + return upperHull.concat(lowerHull); + } + + function pointsComparator(a, b): number { + if (a.x < b.x) return -1; + if (a.x > b.x) return +1; + if (a.y < b.y) return -1; + if (a.y > b.y) return +1; + return 0; + } + + const newPoints = geoPoints.slice(); + newPoints.sort(pointsComparator); + return makeHullPresorted(newPoints); + } + + static contain(shapePoints, x, y): boolean { + function isLeft(P0, P1, P2): number { + return (P1.x - P0.x) * (P2.y - P0.y) - (P2.x - P0.x) * (P1.y - P0.y); + } + + const points = CuboidShape.makeHull(shapePoints); + let wn = 0; + for (let i = 0; i < points.length; i += 1) { + const p1 = points[`${i}`]; + const p2 = points[i + 1] || points[0]; + + if (p1.y <= y) { + if (p2.y > y) { + if (isLeft(p1, p2, { x, y }) > 0) { + wn += 1; + } + } + } else if (p2.y < y) { + if (isLeft(p1, p2, { x, y }) < 0) { + wn -= 1; + } + } + } + + return wn !== 0; + } + + static distance(actualPoints: number[], x: number, y: number): number | null { + const points = []; + + for (let i = 0; i < 16; i += 2) { + points.push({ x: actualPoints[i], y: actualPoints[i + 1] }); + } + + if (!CuboidShape.contain(points, x, y)) return null; + + let minDistance = Number.MAX_SAFE_INTEGER; + for (let i = 0; i < points.length; i += 1) { + const p1 = points[`${i}`]; + const p2 = points[i + 1] || points[0]; + + // perpendicular from point to straight length + const distance = Math.abs((p2.y - p1.y) * x - (p2.x - p1.x) * y + p2.x * p1.y - p2.y * p1.x) / + Math.sqrt((p2.y - p1.y) ** 2 + (p2.x - p1.x) ** 2); + + // check if perpendicular belongs to the straight segment + const a = (p1.x - x) ** 2 + (p1.y - y) ** 2; + const b = (p2.x - p1.x) ** 2 + (p2.y - p1.y) ** 2; + const c = (p2.x - x) ** 2 + (p2.y - y) ** 2; + if (distance < minDistance && a + b - c >= 0 && c + b - a >= 0) { + minDistance = distance; + } + } + return minDistance; + } +} diff --git a/cvat-core/src/annotations-objects/cuboid-track.ts b/cvat-core/src/annotations-objects/cuboid-track.ts new file mode 100644 index 0000000..c8b726b --- /dev/null +++ b/cvat-core/src/annotations-objects/cuboid-track.ts @@ -0,0 +1,71 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { DimensionType, ShapeType } from '../enums'; +import type { SerializedTrack } from '../server-response-types'; +import { checkNumberOfPoints, findAngleDiff } from '../object-utils'; +import { Track } from './track'; +import type { AnnotationInjection, InterpolatedPosition } from './types'; +import { CuboidShape } from './cuboid-shape'; + +export class CuboidTrack extends Track { + constructor(data: SerializedTrack, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.CUBOID; + this.pinned = false; + for (const shape of Object.values(this.shapes)) { + checkNumberOfPoints(this.shapeType, shape.points); + shape.rotation = 0; // is not supported + } + } + + protected interpolatePosition(leftPosition, rightPosition, offset): InterpolatedPosition { + const positionOffset = leftPosition.points.map((point, index) => rightPosition.points[index] - point); + const result = { + points: leftPosition.points.map((point, index) => point + positionOffset[index] * offset), + rotation: leftPosition.rotation, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + + if (this.dimension === DimensionType.DIMENSION_3D) { + // for 3D cuboids angle for different axies stored as a part of points array + // we need to apply interpolation using the shortest arc for each angle + + const [ + angleX, angleY, angleZ, + ] = leftPosition.points.slice(3, 6).concat(rightPosition.points.slice(3, 6)) + .map((_angle: number) => { + if (_angle < 0) { + return _angle + Math.PI * 2; + } + + return _angle; + }) + .map((_angle) => _angle * (180 / Math.PI)) + .reduce((acc: number[], angleBefore: number, index: number, arr: number[]) => { + if (index < 3) { + const angleAfter = arr[index + 3]; + let angle = (angleBefore + findAngleDiff(angleAfter, angleBefore) * offset) * (Math.PI / 180); + if (angle > Math.PI) { + angle -= Math.PI * 2; + } + acc.push(angle); + } + + return acc; + }, []); + + result.points[3] = angleX; + result.points[4] = angleY; + result.points[5] = angleZ; + } + + return result; + } +} + +Object.defineProperty(CuboidTrack, 'distance', { value: CuboidShape.distance }); diff --git a/cvat-core/src/annotations-objects/drawn.ts b/cvat-core/src/annotations-objects/drawn.ts new file mode 100644 index 0000000..437810e --- /dev/null +++ b/cvat-core/src/annotations-objects/drawn.ts @@ -0,0 +1,95 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type ObjectState from '../object-state'; +import { checkObjectType } from '../common'; +import { clamp } from '../opencv/math-utils'; +import { ShapeType, HistoryActions, DimensionType } from '../enums'; +import { + checkNumberOfPoints, checkShapeArea, +} from '../object-utils'; +import { ImageObject } from './image-object'; +import type { AnnotationInjection } from './types'; +import { isChildObject } from './utils'; + +export class Drawn extends ImageObject { + protected descriptions: string[]; + protected pinned: boolean; + public shapeType: ShapeType; + + constructor(data, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.descriptions = data.descriptions || []; + this.pinned = true; + this.shapeType = null; + } + + protected saveDescriptions(descriptions: string[]): void { + this.descriptions = [...descriptions]; + } + + protected savePinned(pinned: boolean, frame: number): void { + const undoPinned = this.pinned; + const redoPinned = pinned; + + this.history.do( + HistoryActions.CHANGED_PINNED, + () => { + this.pinned = undoPinned; + this.updated = Date.now(); + }, + () => { + this.pinned = redoPinned; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.pinned = pinned; + } + + private fitPoints(points: number[], rotation: number, maxX: number, maxY: number): number[] { + const { shapeType, _parentId } = this; + checkObjectType('rotation', rotation, 'number'); + points.forEach((coordinate) => checkObjectType('coordinate', coordinate, 'number')); + + if (isChildObject(_parentId) || shapeType === ShapeType.CUBOID || + shapeType === ShapeType.ELLIPSE || !!rotation) { + // cuboids, rotated bounding boxes, and skeleton elements cannot be fitted + return points; + } + + const fittedPoints = []; + + for (let i = 0; i < points.length - 1; i += 2) { + const x = points[i]; + const y = points[i + 1]; + const clampedX = clamp(x, 0, maxX); + const clampedY = clamp(y, 0, maxY); + fittedPoints.push(clampedX, clampedY); + } + + return fittedPoints; + } + + protected validateStateBeforeSave(data: ObjectState, updated: ObjectState['updateFlags'], frame?: number): number[] { + super.validateStateBeforeSave(data, updated); + + let fittedPoints = []; + if (updated.points && Number.isInteger(frame)) { + checkObjectType('points', data.points, null, { cls: Array, name: 'Array' }); + checkNumberOfPoints(this.shapeType, data.points); + // cut points + const { width, height } = this.framesInfo[frame]; + fittedPoints = this.fitPoints(data.points, data.rotation, width, height); + if (this.dimension === DimensionType.DIMENSION_2D && !checkShapeArea(this.shapeType, fittedPoints)) { + fittedPoints = []; + } + } + + return fittedPoints; + } +} diff --git a/cvat-core/src/annotations-objects/ellipse-shape.ts b/cvat-core/src/annotations-objects/ellipse-shape.ts new file mode 100644 index 0000000..8698d8c --- /dev/null +++ b/cvat-core/src/annotations-objects/ellipse-shape.ts @@ -0,0 +1,66 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { checkNumberOfPoints, rotatePoint } from '../object-utils'; +import { Shape } from './shape'; +import type { AnnotationInjection } from './types'; + +export class EllipseShape extends Shape { + constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.ELLIPSE; + this.pinned = false; + checkNumberOfPoints(this.shapeType, this.points); + } + + static distance(points: number[], x: number, y: number, angle: number): number | null { + const [cx, cy, rightX, topY] = points; + const [rx, ry] = [rightX - cx, cy - topY]; + const [rotX, rotY] = rotatePoint(x, y, -angle, cx, cy); + // https://math.stackexchange.com/questions/76457/check-if-a-point-is-within-an-ellipse + const pointWithinEllipse = (_x: number, _y: number): boolean => ( + ((_x - cx) ** 2) / rx ** 2) + (((_y - cy) ** 2) / ry ** 2 + ) <= 1; + + if (!pointWithinEllipse(rotX, rotY)) { + // Cursor is outside of an ellipse + return null; + } + + if (Math.abs(x - cx) < Number.EPSILON && Math.abs(y - cy) < Number.EPSILON) { + // cursor is near to the center, just return minimum of height, width + return Math.min(rx, ry); + } + + // ellipse equation is x^2/rx^2 + y^2/ry^2 = 1 + // from this equation: + // x^2 = ((rx * ry)^2 - (y * rx)^2) / ry^2 + // y^2 = ((rx * ry)^2 - (x * ry)^2) / rx^2 + + // we have one point inside the ellipse, let's build two lines (horizontal and vertical) through the point + // and find their interception with ellipse + const x2Equation = (_y: number): number => (((rx * ry) ** 2) - ((_y * rx) ** 2)) / (ry ** 2); + const y2Equation = (_x: number): number => (((rx * ry) ** 2) - ((_x * ry) ** 2)) / (rx ** 2); + + // shift x,y to the ellipse coordinate system to compute equation correctly + // y axis is inverted + const [shiftedX, shiftedY] = [x - cx, cy - y]; + const [x1, x2] = [Math.sqrt(x2Equation(shiftedY)), -Math.sqrt(x2Equation(shiftedY))]; + const [y1, y2] = [Math.sqrt(y2Equation(shiftedX)), -Math.sqrt(y2Equation(shiftedX))]; + + // found two points on ellipse edge + const ellipseP1X = shiftedX >= 0 ? x1 : x2; // ellipseP1Y is shiftedY + const ellipseP2Y = shiftedY >= 0 ? y1 : y2; // ellipseP1X is shiftedX + + // found diffs between two points on edges and target point + const diff1X = ellipseP1X - shiftedX; + const diff2Y = ellipseP2Y - shiftedY; + + // return minimum, get absolute value because we need distance, not diff + return Math.min(Math.abs(diff1X), Math.abs(diff2Y)); + } +} diff --git a/cvat-core/src/annotations-objects/ellipse-track.ts b/cvat-core/src/annotations-objects/ellipse-track.ts new file mode 100644 index 0000000..5669c95 --- /dev/null +++ b/cvat-core/src/annotations-objects/ellipse-track.ts @@ -0,0 +1,39 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedTrack } from '../server-response-types'; +import { checkNumberOfPoints, findAngleDiff } from '../object-utils'; +import { Track } from './track'; +import type { AnnotationInjection, InterpolatedPosition } from './types'; +import { EllipseShape } from './ellipse-shape'; + +export class EllipseTrack extends Track { + constructor(data: SerializedTrack, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.ELLIPSE; + this.pinned = false; + for (const shape of Object.values(this.shapes)) { + checkNumberOfPoints(this.shapeType, shape.points); + } + } + + interpolatePosition(leftPosition, rightPosition, offset): InterpolatedPosition { + const positionOffset = leftPosition.points.map((point, index) => rightPosition.points[index] - point); + + return { + points: leftPosition.points.map((point, index) => point + positionOffset[index] * offset), + rotation: + (leftPosition.rotation + findAngleDiff( + rightPosition.rotation, leftPosition.rotation, + ) * offset + 360) % 360, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + } +} + +Object.defineProperty(EllipseTrack, 'distance', { value: EllipseShape.distance }); diff --git a/cvat-core/src/annotations-objects/image-object.ts b/cvat-core/src/annotations-objects/image-object.ts new file mode 100644 index 0000000..b33024f --- /dev/null +++ b/cvat-core/src/annotations-objects/image-object.ts @@ -0,0 +1,103 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type ObjectState from '../object-state'; +import { checkObjectType } from '../common'; +import { ArgumentError } from '../exceptions'; +import { Label } from '../labels'; +import { + attrsAsAnObject, validateAttributeValue, +} from '../object-utils'; +import { AnnotationBase } from './annotation-common'; +import type { AnnotationInjection, TrackedShape } from './types'; + +export class InterpolationNotPossibleError extends Error {} + +export class ImageObject extends AnnotationBase { + public frame: number; + + constructor(data, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.frame = data.frame; + } + + protected validateStateBeforeSave(data: ObjectState, updated: ObjectState['updateFlags']): void { + if (updated.label) { + checkObjectType('label', data.label, null, { cls: Label, name: 'Label' }); + } + + const labelAttributes = attrsAsAnObject(data.label.attributes); + if (updated.attributes) { + for (const [id, value] of Object.entries(data.attributes)) { + if (id in labelAttributes) { + if (!validateAttributeValue(value, labelAttributes[id])) { + throw new ArgumentError( + `Trying to save an attribute attribute with id ${id} and invalid value ${value}`, + ); + } + } else { + throw new ArgumentError( + `The label of the object doesn't have the attribute with id ${id}`, + ); + } + } + } + + if (updated.descriptions) { + if (!Array.isArray(data.descriptions) || data.descriptions.some((desc) => typeof desc !== 'string')) { + throw new ArgumentError( + `Descriptions are expected to be an array of strings but got ${data.descriptions}`, + ); + } + } + + if (updated.occluded) { + checkObjectType('occluded', data.occluded, 'boolean'); + } + + if (updated.outside) { + checkObjectType('outside', data.outside, 'boolean'); + } + + if (updated.zOrder) { + checkObjectType('zOrder', data.zOrder, 'integer'); + } + + if (updated.lock) { + checkObjectType('lock', data.lock, 'boolean'); + } + + if (updated.pinned) { + checkObjectType('pinned', data.pinned, 'boolean'); + } + + if (updated.color) { + checkObjectType('color', data.color, 'string'); + if (!/^#[0-9A-F]{6}$/i.test(data.color)) { + throw new ArgumentError(`Got invalid color value: "${data.color}"`); + } + } + + if (updated.hidden) { + checkObjectType('hidden', data.hidden, 'boolean'); + } + + if (updated.keyframe) { + checkObjectType('keyframe', data.keyframe, 'boolean'); + const tracksShapeContext = this as ImageObject & { shapes?: Record }; + if ( + tracksShapeContext.shapes && + Object.keys(tracksShapeContext.shapes).length === 1 && + data.frame in tracksShapeContext.shapes && + !data.keyframe + ) { + throw new ArgumentError( + `Can not remove the latest keyframe of an object "${data.label.name}".` + + 'Consider removing the object instead', + ); + } + } + } +} diff --git a/cvat-core/src/annotations-objects/index.ts b/cvat-core/src/annotations-objects/index.ts new file mode 100644 index 0000000..5e0a8e2 --- /dev/null +++ b/cvat-core/src/annotations-objects/index.ts @@ -0,0 +1,128 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { DataError } from '../exceptions'; +import { colors, ShapeType } from '../enums'; +import type { SerializedShape, SerializedTrack } from '../server-response-types'; +import { Shape } from './shape'; +import { Track } from './track'; +import type { AnnotationInjection } from './types'; +import { RectangleShape } from './rectangle-shape'; +import { EllipseShape } from './ellipse-shape'; +import { PolygonShape } from './polygon-shape'; +import { PolylineShape } from './polyline-shape'; +import { PointsShape } from './points-shape'; +import { CuboidShape } from './cuboid-shape'; +import { MaskShape } from './mask-shape'; +// eslint-disable-next-line import/no-cycle +import { SkeletonShape } from './skeleton-shape'; +import { RectangleTrack } from './rectangle-track'; +import { EllipseTrack } from './ellipse-track'; +import { PolygonTrack } from './polygon-track'; +import { PolylineTrack } from './polyline-track'; +import { PointsTrack } from './points-track'; +import { CuboidTrack } from './cuboid-track'; +// eslint-disable-next-line import/no-cycle +import { SkeletonTrack } from './skeleton-track'; +import { AudioInterval } from './audio-interval'; + +export type { BasicInjection, InterpolatedPosition } from './types'; +export { InterpolationNotPossibleError } from './image-object'; +export { Shape }; +export { Track }; +export { Tag } from './tag'; +export { AudioInterval }; +export { + RectangleShape, EllipseShape, PolygonShape, PolylineShape, PointsShape, + CuboidShape, SkeletonShape, MaskShape, +}; +export { + RectangleTrack, EllipseTrack, PolygonTrack, PolylineTrack, PointsTrack, + CuboidTrack, SkeletonTrack, +}; + +export function shapeFactory( + data: SerializedShape | SerializedShape['elements'][0], + clientID: number, + injection: AnnotationInjection, +): Shape { + const { type } = data; + const color = colors[clientID % colors.length]; + + let shapeModel = null; + switch (type) { + case ShapeType.RECTANGLE: + shapeModel = new RectangleShape(data as SerializedShape, clientID, color, injection); + break; + case ShapeType.POLYGON: + shapeModel = new PolygonShape(data as SerializedShape, clientID, color, injection); + break; + case ShapeType.POLYLINE: + shapeModel = new PolylineShape(data as SerializedShape, clientID, color, injection); + break; + case ShapeType.POINTS: + shapeModel = new PointsShape(data, clientID, color, injection); + break; + case ShapeType.ELLIPSE: + shapeModel = new EllipseShape(data as SerializedShape, clientID, color, injection); + break; + case ShapeType.CUBOID: + shapeModel = new CuboidShape(data as SerializedShape, clientID, color, injection); + break; + case ShapeType.MASK: + shapeModel = new MaskShape(data as SerializedShape, clientID, color, injection); + break; + case ShapeType.SKELETON: + shapeModel = new SkeletonShape(data as SerializedShape, clientID, color, injection); + break; + default: + throw new DataError(`An unexpected type of shape "${type}"`); + } + + return shapeModel; +} + +export function trackFactory( + trackData: SerializedTrack | SerializedTrack['elements'][0], + clientID: number, + injection: AnnotationInjection, +): Track { + if (trackData.shapes.length) { + const { type } = trackData.shapes[0]; + const color = colors[clientID % colors.length]; + + let trackModel = null; + switch (type) { + case ShapeType.RECTANGLE: + trackModel = new RectangleTrack(trackData as SerializedTrack, clientID, color, injection); + break; + case ShapeType.POLYGON: + trackModel = new PolygonTrack(trackData as SerializedTrack, clientID, color, injection); + break; + case ShapeType.POLYLINE: + trackModel = new PolylineTrack(trackData as SerializedTrack, clientID, color, injection); + break; + case ShapeType.POINTS: + trackModel = new PointsTrack(trackData, clientID, color, injection); + break; + case ShapeType.ELLIPSE: + trackModel = new EllipseTrack(trackData as SerializedTrack, clientID, color, injection); + break; + case ShapeType.CUBOID: + trackModel = new CuboidTrack(trackData as SerializedTrack, clientID, color, injection); + break; + case ShapeType.SKELETON: + trackModel = new SkeletonTrack(trackData as SerializedTrack, clientID, color, injection); + break; + default: + throw new DataError(`An unexpected type of track "${type}"`); + } + + return trackModel; + } + + console.warn('The track without any shapes had been found. It was ignored.'); + return null; +} diff --git a/cvat-core/src/annotations-objects/mask-shape.ts b/cvat-core/src/annotations-objects/mask-shape.ts new file mode 100644 index 0000000..d442c5d --- /dev/null +++ b/cvat-core/src/annotations-objects/mask-shape.ts @@ -0,0 +1,235 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import config from '../config'; +import type ObjectState from '../object-state'; +import { ArgumentError } from '../exceptions'; +import { ShapeType, HistoryActions } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { mask2Rle, rle2Mask } from '../rle-utils'; +import { cropMask } from '../object-utils'; +import { Shape } from './shape'; +import { computeNewSource } from './utils'; +import type { AnnotationInjection } from './types'; + +export class MaskShape extends Shape { + public left: number; + public top: number; + public right: number; + public bottom: number; + + constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + const [left, top, right, bottom] = this.points.slice(-4); + const { width, height } = this.framesInfo[this.frame]; + if (left < 0 || top < 0 || right >= width || bottom >= height) { + this.points = cropMask(this.points, width, height); + } + [this.left, this.top, this.right, this.bottom] = this.points.splice(-4, 4); + this.pinned = true; + this.shapeType = ShapeType.MASK; + } + + protected validateStateBeforeSave(data: ObjectState, updated: ObjectState['updateFlags'], frame?: number): number[] { + super.validateStateBeforeSave(data, updated, frame); + if (updated.points) { + const { width, height } = this.framesInfo[frame]; + return cropMask(data.points, width, height); + } + return []; + } + + public removeUnderlyingPixels(frame: number): + { + clientIDs: number[], + undo: () => void, + redo: () => void, + emptyMaskOccurred: boolean, + } { + if (frame !== this.frame) { + throw new ArgumentError( + `Wrong "frame" attribute: is not equal to the shape frame (${frame} vs ${this.frame})`, + ); + } + + const others = this.getMasksOnFrame(frame) + .filter((mask: MaskShape) => mask.clientID !== this.clientID && !mask.removed); + const width = this.right - this.left + 1; + const height = this.bottom - this.top + 1; + const updatedObjects: Record = {}; + + let masks = {}; + const currentMask = rle2Mask(this.points, width, height); + for (let i = 0; i < currentMask.length; i++) { + if (currentMask[i]) { + const imageX = (i % width) + this.left; + const imageY = Math.trunc(i / width) + this.top; + for (const other of others) { + const box = { + left: other.left, + top: other.top, + right: other.right, + bottom: other.bottom, + }; + const translatedX = imageX - box.left; + const translatedY = imageY - box.top; + const [otherWidth, otherHeight] = [box.right - box.left + 1, box.bottom - box.top + 1]; + if (translatedX >= 0 && translatedX < otherWidth && + translatedY >= 0 && translatedY < otherHeight) { + masks[other.clientID] = masks[other.clientID] || + rle2Mask(other.points, otherWidth, otherHeight); + const j = translatedY * otherWidth + translatedX; + masks[other.clientID][j] = 0; + updatedObjects[other.clientID] = other; + } + } + } + } + + const wrapper = { + stashedPoints: Object.values(updatedObjects).map((object) => object.points), + stashedRemoved: Object.values(updatedObjects).map((object) => object.removed), + }; + + let emptyMaskOccurred = false; + for (const object of Object.values(updatedObjects)) { + const points = mask2Rle(masks[object.clientID]); + if (points.length < 2) { + object.removed = true; + emptyMaskOccurred = true; + } else { + object.points = points; + object.updated = Date.now(); + } + } + masks = null; + + const undo = (): void => { + const updatedStashedPoints = Object.values(updatedObjects).map((object) => object.points); + const updatedStashedRemoved = Object.values(updatedObjects).map((object) => object.removed); + for (const [index, object] of Object.values(updatedObjects).entries()) { + object.points = wrapper.stashedPoints[index]; + object.removed = wrapper.stashedRemoved[index]; + object.updated = Date.now(); + } + wrapper.stashedPoints = updatedStashedPoints; + wrapper.stashedRemoved = updatedStashedRemoved; + }; + + const redo = undo; + return { + clientIDs: Object.keys(updatedObjects).map((clientID) => +clientID), + emptyMaskOccurred, + undo, + redo, + }; + } + + protected savePoints(maskPoints: number[], frame: number): void { + const undoPoints = this.points; + const undoLeft = this.left; + const undoRight = this.right; + const undoTop = this.top; + const undoBottom = this.bottom; + const undoSource = this.source; + + const [redoLeft, redoTop, redoRight, redoBottom] = maskPoints.splice(-4); + const points = maskPoints; + + const redoPoints = points; + const redoSource = computeNewSource(this.source); + + const undo = (): void => { + this.points = undoPoints; + this.source = undoSource; + this.left = undoLeft; + this.top = undoTop; + this.right = undoRight; + this.bottom = undoBottom; + this.updated = Date.now(); + }; + + const redo = (): void => { + this.points = redoPoints; + this.source = redoSource; + this.left = redoLeft; + this.top = redoTop; + this.right = redoRight; + this.bottom = redoBottom; + this.updated = Date.now(); + }; + + redo(); + if (config.removeUnderlyingMaskPixels.enabled) { + const { + clientIDs, + emptyMaskOccurred, + undo: undoWithUnderlyingPixels, + redo: redoWithUnderlyingPixels, + } = this.removeUnderlyingPixels(frame); + if (emptyMaskOccurred) { + config.removeUnderlyingMaskPixels?.onEmptyMaskOccurrence(); + } + this.history.do( + HistoryActions.CHANGED_POINTS, + () => { + undoWithUnderlyingPixels(); + undo(); + }, + () => { + redoWithUnderlyingPixels(); + redo(); + }, + [this.clientID, ...clientIDs], + frame, + ); + } else { + this.history.do( + HistoryActions.CHANGED_POINTS, + undo, + redo, + [this.clientID], + frame, + ); + } + } + + static distance(rle: number[], x: number, y: number): number | null { + const [left, top, right, bottom] = rle.slice(-4); + const [width, height] = [right - left + 1, bottom - top + 1]; + const [translatedX, translatedY] = [x - left, y - top]; + if (translatedX < 0 || translatedX >= width || translatedY < 0 || translatedY >= height) { + return null; + } + + const offset = Math.floor(translatedY) * width + Math.floor(translatedX); + let sum = 0; + let value = 0; + + for (const count of rle) { + sum += count; + if (sum > offset) { + return value || null; + } + value = Math.abs(value - 1); + } + + return null; + } +} + +MaskShape.prototype.toJSON = function () { + const result = Shape.prototype.toJSON.call(this); + result.points = this.points.slice(); + result.points.push(this.left, this.top, this.right, this.bottom); + return result; +}; + +MaskShape.prototype.get = function (frame) { + const result = Shape.prototype.get.call(this, frame); + result.points = this.points.slice(0); + result.points.push(this.left, this.top, this.right, this.bottom); + return result; +}; diff --git a/cvat-core/src/annotations-objects/points-shape.ts b/cvat-core/src/annotations-objects/points-shape.ts new file mode 100644 index 0000000..e5531b0 --- /dev/null +++ b/cvat-core/src/annotations-objects/points-shape.ts @@ -0,0 +1,39 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { checkNumberOfPoints } from '../object-utils'; +import type { AnnotationInjection } from './types'; +import { PolyShape } from './poly-shape'; +import { isChildObject } from './utils'; + +export class PointsShape extends PolyShape { + constructor( + data: SerializedShape | SerializedShape['elements'][0], + clientID: number, + color: string, + injection: AnnotationInjection, + ) { + super(data, clientID, color, injection); + if (isChildObject(this._parentId)) { + this.readOnlyFields = ['group', 'zOrder', 'source', 'rotation']; + } + this.shapeType = ShapeType.POINTS; + checkNumberOfPoints(this.shapeType, this.points); + } + + static distance(points: number[], x: number, y: number): number { + const distances = []; + for (let i = 0; i < points.length; i += 2) { + const x1 = points[i]; + const y1 = points[i + 1]; + + distances.push(Math.sqrt((x1 - x) ** 2 + (y1 - y) ** 2)); + } + + return Math.min.apply(null, distances); + } +} diff --git a/cvat-core/src/annotations-objects/points-track.ts b/cvat-core/src/annotations-objects/points-track.ts new file mode 100644 index 0000000..6185f30 --- /dev/null +++ b/cvat-core/src/annotations-objects/points-track.ts @@ -0,0 +1,55 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedTrack } from '../server-response-types'; +import { checkNumberOfPoints } from '../object-utils'; +import type { AnnotationInjection, InterpolatedPosition } from './types'; +import { PolyTrack } from './poly-track'; +import { PointsShape } from './points-shape'; +import { isChildObject } from './utils'; + +export class PointsTrack extends PolyTrack { + constructor( + data: SerializedTrack | SerializedTrack['elements'][0], + clientID: number, + color: string, + injection: AnnotationInjection, + ) { + super(data, clientID, color, injection); + if (isChildObject(this._parentId)) { + this.readOnlyFields = ['group', 'zOrder', 'source', 'rotation']; + } + this.shapeType = ShapeType.POINTS; + for (const shape of Object.values(this.shapes)) { + checkNumberOfPoints(this.shapeType, shape.points); + } + } + + protected interpolatePosition(leftPosition, rightPosition, offset): InterpolatedPosition { + // interpolate only when one point in both left and right positions + if (leftPosition.points.length === 2 && rightPosition.points.length === 2) { + return { + points: leftPosition.points.map( + (value, index) => value + (rightPosition.points[index] - value) * offset, + ), + rotation: leftPosition.rotation, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + } + + return { + points: [...leftPosition.points], + rotation: leftPosition.rotation, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + } +} + +Object.defineProperty(PointsTrack, 'distance', { value: PointsShape.distance }); diff --git a/cvat-core/src/annotations-objects/poly-shape.ts b/cvat-core/src/annotations-objects/poly-shape.ts new file mode 100644 index 0000000..61bb4cd --- /dev/null +++ b/cvat-core/src/annotations-objects/poly-shape.ts @@ -0,0 +1,20 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type { SerializedShape } from '../server-response-types'; +import { Shape } from './shape'; +import type { AnnotationInjection } from './types'; + +export class PolyShape extends Shape { + constructor( + data: SerializedShape | SerializedShape['elements'][0], + clientID: number, + color: string, + injection: AnnotationInjection, + ) { + super(data, clientID, color, injection); + this.rotation = 0; // is not supported + } +} diff --git a/cvat-core/src/annotations-objects/poly-track.ts b/cvat-core/src/annotations-objects/poly-track.ts new file mode 100644 index 0000000..46436fa --- /dev/null +++ b/cvat-core/src/annotations-objects/poly-track.ts @@ -0,0 +1,289 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type { SerializedTrack } from '../server-response-types'; +import { Track } from './track'; +import type { AnnotationInjection, InterpolatedPosition, Point2D } from './types'; + +export class PolyTrack extends Track { + constructor( + data: SerializedTrack | SerializedTrack['elements'][0], + clientID: number, + color: string, + injection: AnnotationInjection, + ) { + super(data, clientID, color, injection); + for (const shape of Object.values(this.shapes)) { + shape.rotation = 0; // is not supported + } + } + + protected interpolatePosition(leftPosition, rightPosition, offset): InterpolatedPosition { + if (offset === 0) { + return { + points: [...leftPosition.points], + rotation: leftPosition.rotation, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + } + + function toArray(points: { x: number; y: number; }[]): number[] { + return points.reduce((acc, val) => { + acc.push(val.x, val.y); + return acc; + }, []); + } + + function toPoints(array: number[]): { x: number; y: number; }[] { + return array.reduce((acc, _, index) => { + if (index % 2) { + acc.push({ + x: array[index - 1], + y: array[index], + }); + } + + return acc; + }, []); + } + + function curveLength(points: { x: number; y: number; }[]): number { + return points.slice(1).reduce((acc, _, index) => { + const dx = points[index + 1].x - points[index].x; + const dy = points[index + 1].y - points[index].y; + return acc + Math.sqrt(dx ** 2 + dy ** 2); + }, 0); + } + + function curveToOffsetVec(points: { x: number; y: number; }[], length: number): number[] { + const offsetVector = [0]; // with initial value + let accumulatedLength = 0; + + points.slice(1).forEach((_, index) => { + const dx = points[index + 1].x - points[index].x; + const dy = points[index + 1].y - points[index].y; + accumulatedLength += Math.sqrt(dx ** 2 + dy ** 2); + offsetVector.push(accumulatedLength / length); + }); + + return offsetVector; + } + + function findNearestPair(value: number, curve: number[]): number { + let minimum = [0, Math.abs(value - curve[0])]; + for (let i = 1; i < curve.length; i++) { + const distance = Math.abs(value - curve[i]); + if (distance < minimum[1]) { + minimum = [i, distance]; + } + } + + return minimum[0]; + } + + function matchLeftRight(leftCurve: number[], rightCurve: number[]): Record { + const matching: Record = {}; + for (let i = 0; i < leftCurve.length; i++) { + matching[i] = [findNearestPair(leftCurve[i], rightCurve)]; + } + + return matching; + } + + function matchRightLeft( + leftCurve: number[], + rightCurve: number[], + leftRightMatching: Record, + ): Record { + const matchedRightPoints = Object.values(leftRightMatching).flat(); + const unmatchedRightPoints = rightCurve + .map((_, index) => index) + .filter((index) => !matchedRightPoints.includes(index)); + const updatedMatching = { ...leftRightMatching }; + + for (const rightPoint of unmatchedRightPoints) { + const leftPoint = findNearestPair(rightCurve[rightPoint], leftCurve); + updatedMatching[leftPoint].push(rightPoint); + } + + for (const key of Object.keys(updatedMatching)) { + const sortedRightIndexes = updatedMatching[key].sort((a, b) => a - b); + updatedMatching[key] = sortedRightIndexes; + } + + return updatedMatching; + } + + function reduceInterpolation( + interpolatedPoints: Point2D[], + matching: Record, + leftPoints: Point2D[], + rightPoints: Point2D[], + ): Point2D[] { + function averagePoint(points: Point2D[]): Point2D { + let sumX = 0; + let sumY = 0; + for (const point of points) { + sumX += point.x; + sumY += point.y; + } + + return { + x: sumX / points.length, + y: sumY / points.length, + }; + } + + function computeDistance(point1: Point2D, point2: Point2D): number { + return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2); + } + + function minimizeSegment(baseLength: number, N: number, startInterpolated, stopInterpolated): Point2D[] { + const threshold = baseLength / (2 * N); + const minimized = [interpolatedPoints[startInterpolated]]; + let latestPushed = startInterpolated; + for (let i = startInterpolated + 1; i < stopInterpolated; i++) { + const distance = computeDistance(interpolatedPoints[latestPushed], interpolatedPoints[i]); + + if (distance >= threshold) { + minimized.push(interpolatedPoints[i]); + latestPushed = i; + } + } + + minimized.push(interpolatedPoints[stopInterpolated]); + + if (minimized.length === 2) { + const distance = computeDistance( + interpolatedPoints[startInterpolated], + interpolatedPoints[stopInterpolated], + ); + + if (distance < threshold) { + return [averagePoint(minimized)]; + } + } + + return minimized; + } + + const reduced: Point2D[] = []; + const interpolatedIndexes: Record = {}; + let accumulated = 0; + for (let i = 0; i < leftPoints.length; i++) { + // eslint-disable-next-line + interpolatedIndexes[i] = matching[i].map(() => accumulated++); + } + + function leftSegment(start, stop): void { + const startInterpolated = interpolatedIndexes[start][0]; + const stopInterpolated = interpolatedIndexes[stop][0]; + + if (startInterpolated === stopInterpolated) { + reduced.push(interpolatedPoints[startInterpolated]); + return; + } + + const baseLength = curveLength(leftPoints.slice(start, stop + 1)); + const N = stop - start + 1; + + reduced.push(...minimizeSegment(baseLength, N, startInterpolated, stopInterpolated)); + } + + function rightSegment(leftPoint): void { + const start = matching[leftPoint][0]; + const [stop] = matching[leftPoint].slice(-1); + const startInterpolated = interpolatedIndexes[leftPoint][0]; + const [stopInterpolated] = interpolatedIndexes[leftPoint].slice(-1); + const baseLength = curveLength(rightPoints.slice(start, stop + 1)); + const N = stop - start + 1; + + reduced.push(...minimizeSegment(baseLength, N, startInterpolated, stopInterpolated)); + } + + let previousOpened: number | null = null; + for (let i = 0; i < leftPoints.length; i++) { + if (matching[i].length === 1) { + // check if left segment is opened + if (previousOpened !== null) { + // check if we should continue the left segment + if (matching[i][0] === matching[previousOpened][0]) { + continue; + } else { + // left segment found + const start = previousOpened; + const stop = i - 1; + leftSegment(start, stop); + + // start next left segment + previousOpened = i; + } + } else { + // start next left segment + previousOpened = i; + } + } else { + // check if left segment is opened + if (previousOpened !== null) { + // left segment found + const start = previousOpened; + const stop = i - 1; + leftSegment(start, stop); + + previousOpened = null; + } + + // right segment found + rightSegment(i); + } + } + + // check if there is an opened segment + if (previousOpened !== null) { + leftSegment(previousOpened, leftPoints.length - 1); + } + + return reduced; + } + + // the algorithm below is based on fact that both left and right + // polyshapes have the same start point and the same draw direction + const leftPoints = toPoints(leftPosition.points); + const rightPoints = toPoints(rightPosition.points); + const leftOffsetVec = curveToOffsetVec(leftPoints, curveLength(leftPoints)); + const rightOffsetVec = curveToOffsetVec(rightPoints, curveLength(rightPoints)); + + const matching = matchLeftRight(leftOffsetVec, rightOffsetVec); + const completedMatching = matchRightLeft(leftOffsetVec, rightOffsetVec, matching); + + const interpolatedPoints = Object.keys(completedMatching) + .map((leftPointIdx) => +leftPointIdx) + .sort((a, b) => a - b) + .reduce((acc, leftPointIdx) => { + const leftPoint = leftPoints[leftPointIdx]; + for (const rightPointIdx of completedMatching[leftPointIdx]) { + const rightPoint = rightPoints[rightPointIdx]; + acc.push({ + x: leftPoint.x + (rightPoint.x - leftPoint.x) * offset, + y: leftPoint.y + (rightPoint.y - leftPoint.y) * offset, + }); + } + + return acc; + }, []); + + const reducedPoints = reduceInterpolation(interpolatedPoints, completedMatching, leftPoints, rightPoints); + + return { + points: toArray(reducedPoints), + rotation: leftPosition.rotation, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + } +} diff --git a/cvat-core/src/annotations-objects/polygon-shape.ts b/cvat-core/src/annotations-objects/polygon-shape.ts new file mode 100644 index 0000000..32c7e8c --- /dev/null +++ b/cvat-core/src/annotations-objects/polygon-shape.ts @@ -0,0 +1,81 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { checkNumberOfPoints } from '../object-utils'; +import type { AnnotationInjection } from './types'; +import { PolyShape } from './poly-shape'; + +export class PolygonShape extends PolyShape { + constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.POLYGON; + checkNumberOfPoints(this.shapeType, this.points); + } + + static distance(points: number[], x: number, y: number): number | null { + function position(x1, y1, x2, y2): number { + return (x2 - x1) * (y - y1) - (x - x1) * (y2 - y1); + } + + let wn = 0; + const distances = []; + + for (let i = 0, j = points.length - 2; i < points.length - 1; j = i, i += 2) { + // Current point + const x1 = points[j]; + const y1 = points[j + 1]; + + // Next point + const x2 = points[i]; + const y2 = points[i + 1]; + + // Check if a point is inside a polygon + // with a winding numbers algorithm + // https://en.wikipedia.org/wiki/Point_in_polygon#Winding_number_algorithm + if (y1 <= y) { + if (y2 > y) { + if (position(x1, y1, x2, y2) > 0) { + wn++; + } + } + } else if (y2 <= y) { + if (position(x1, y1, x2, y2) < 0) { + wn--; + } + } + + // Find the shortest distance from point to an edge + // Get an equation of a line in general + const aCoef = y1 - y2; + const bCoef = x2 - x1; + + // Vector (aCoef, bCoef) is a perpendicular to line + // Now find the point where two lines + // (edge and its perpendicular through the point (x,y)) are cross + const xCross = x - aCoef; + const yCross = y - bCoef; + + if ((xCross - x1) * (x2 - xCross) >= 0 && (yCross - y1) * (y2 - yCross) >= 0) { + // Cross point is on segment between p1(x1,y1) and p2(x2,y2) + distances.push(Math.sqrt((x - xCross) ** 2 + (y - yCross) ** 2)); + } else { + distances.push( + Math.min( + Math.sqrt((x1 - x) ** 2 + (y1 - y) ** 2), + Math.sqrt((x2 - x) ** 2 + (y2 - y) ** 2), + ), + ); + } + } + + if (wn !== 0) { + return Math.min.apply(null, distances); + } + + return null; + } +} diff --git a/cvat-core/src/annotations-objects/polygon-track.ts b/cvat-core/src/annotations-objects/polygon-track.ts new file mode 100644 index 0000000..3be75c3 --- /dev/null +++ b/cvat-core/src/annotations-objects/polygon-track.ts @@ -0,0 +1,42 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedTrack } from '../server-response-types'; +import { checkNumberOfPoints } from '../object-utils'; +import type { AnnotationInjection, InterpolatedPosition } from './types'; +import { PolyTrack } from './poly-track'; +import { PolygonShape } from './polygon-shape'; + +export class PolygonTrack extends PolyTrack { + constructor(data: SerializedTrack, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.POLYGON; + for (const shape of Object.values(this.shapes)) { + checkNumberOfPoints(this.shapeType, shape.points); + } + } + + protected interpolatePosition(leftPosition, rightPosition, offset): InterpolatedPosition { + const copyLeft = { + ...leftPosition, + points: [...leftPosition.points, leftPosition.points[0], leftPosition.points[1]], + }; + + const copyRight = { + ...rightPosition, + points: [...rightPosition.points, rightPosition.points[0], rightPosition.points[1]], + }; + + const result = super.interpolatePosition(copyLeft, copyRight, offset); + + return { + ...result, + points: result.points.slice(0, -2), + }; + } +} + +Object.defineProperty(PolygonTrack, 'distance', { value: PolygonShape.distance }); diff --git a/cvat-core/src/annotations-objects/polyline-shape.ts b/cvat-core/src/annotations-objects/polyline-shape.ts new file mode 100644 index 0000000..d34a73d --- /dev/null +++ b/cvat-core/src/annotations-objects/polyline-shape.ts @@ -0,0 +1,63 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { checkNumberOfPoints } from '../object-utils'; +import type { AnnotationInjection } from './types'; +import { PolyShape } from './poly-shape'; + +export class PolylineShape extends PolyShape { + constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.POLYLINE; + checkNumberOfPoints(this.shapeType, this.points); + } + + static distance(points: number[], x: number, y: number): number { + const distances = []; + for (let i = 0; i < points.length - 2; i += 2) { + // Current point + const x1 = points[i]; + const y1 = points[i + 1]; + + // Next point + const x2 = points[i + 2]; + const y2 = points[i + 3]; + + // Find the shortest distance from point to an edge + // using perpendicular or by the distance to the nearest point + + // Get coordinate vectors + const AB = [x2 - x1, y2 - y1]; + const BM = [x - x2, y - y2]; + const AM = [x - x1, y - y1]; + + // scalar products have different signs for two pairs of vectors + // it means that perpendicular projection lies on the edge + if (Math.sign(AB[0] * BM[0] + AB[1] * BM[1]) !== Math.sign(AB[0] * AM[0] + AB[1] * AM[1])) { + // Find the length of a perpendicular + // https://en.wikipedia.org/wiki/Distance_from_a_point_to_a_line + distances.push( + Math.abs((y2 - y1) * x - (x2 - x1) * y + x2 * y1 - y2 * x1) / + Math.sqrt((y2 - y1) ** 2 + (x2 - x1) ** 2), + ); + } else { + // The link below works for lines (which have infinite length) + // There is a case when perpendicular doesn't cross the edge + // In this case we don't use the computed distance + // Instead we use just distance to the nearest point + distances.push( + Math.min( + Math.sqrt((x1 - x) ** 2 + (y1 - y) ** 2), + Math.sqrt((x2 - x) ** 2 + (y2 - y) ** 2), + ), + ); + } + } + + return Math.min.apply(null, distances); + } +} diff --git a/cvat-core/src/annotations-objects/polyline-track.ts b/cvat-core/src/annotations-objects/polyline-track.ts new file mode 100644 index 0000000..d9276c5 --- /dev/null +++ b/cvat-core/src/annotations-objects/polyline-track.ts @@ -0,0 +1,23 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedTrack } from '../server-response-types'; +import { checkNumberOfPoints } from '../object-utils'; +import type { AnnotationInjection } from './types'; +import { PolyTrack } from './poly-track'; +import { PolylineShape } from './polyline-shape'; + +export class PolylineTrack extends PolyTrack { + constructor(data: SerializedTrack, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.POLYLINE; + for (const shape of Object.values(this.shapes)) { + checkNumberOfPoints(this.shapeType, shape.points); + } + } +} + +Object.defineProperty(PolylineTrack, 'distance', { value: PolylineShape.distance }); diff --git a/cvat-core/src/annotations-objects/rectangle-shape.ts b/cvat-core/src/annotations-objects/rectangle-shape.ts new file mode 100644 index 0000000..740e6d0 --- /dev/null +++ b/cvat-core/src/annotations-objects/rectangle-shape.ts @@ -0,0 +1,34 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { checkNumberOfPoints, rotatePoint } from '../object-utils'; +import { Shape } from './shape'; +import type { AnnotationInjection } from './types'; + +export class RectangleShape extends Shape { + constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.RECTANGLE; + this.pinned = false; + checkNumberOfPoints(this.shapeType, this.points); + } + + static distance(points: number[], x: number, y: number, angle: number): number | null { + const [xtl, ytl, xbr, ybr] = points; + const cx = xtl + (xbr - xtl) / 2; + const cy = ytl + (ybr - ytl) / 2; + const [rotX, rotY] = rotatePoint(x, y, -angle, cx, cy); + + if (!(rotX >= xtl && rotX <= xbr && rotY >= ytl && rotY <= ybr)) { + // Cursor is outside of a box + return null; + } + + // The shortest distance from point to an edge + return Math.min.apply(null, [rotX - xtl, rotY - ytl, xbr - rotX, ybr - rotY]); + } +} diff --git a/cvat-core/src/annotations-objects/rectangle-track.ts b/cvat-core/src/annotations-objects/rectangle-track.ts new file mode 100644 index 0000000..c4ef3ee --- /dev/null +++ b/cvat-core/src/annotations-objects/rectangle-track.ts @@ -0,0 +1,38 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ShapeType } from '../enums'; +import type { SerializedTrack } from '../server-response-types'; +import { checkNumberOfPoints, findAngleDiff } from '../object-utils'; +import { Track } from './track'; +import type { AnnotationInjection, InterpolatedPosition } from './types'; +import { RectangleShape } from './rectangle-shape'; + +export class RectangleTrack extends Track { + constructor(data: SerializedTrack, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.RECTANGLE; + this.pinned = false; + for (const shape of Object.values(this.shapes)) { + checkNumberOfPoints(this.shapeType, shape.points); + } + } + + protected interpolatePosition(leftPosition, rightPosition, offset): InterpolatedPosition { + const positionOffset = leftPosition.points.map((point, index) => rightPosition.points[index] - point); + return { + points: leftPosition.points.map((point, index) => point + positionOffset[index] * offset), + rotation: + (leftPosition.rotation + findAngleDiff( + rightPosition.rotation, leftPosition.rotation, + ) * offset + 360) % 360, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + }; + } +} + +Object.defineProperty(RectangleTrack, 'distance', { value: RectangleShape.distance }); diff --git a/cvat-core/src/annotations-objects/scored.ts b/cvat-core/src/annotations-objects/scored.ts new file mode 100644 index 0000000..9dd5455 --- /dev/null +++ b/cvat-core/src/annotations-objects/scored.ts @@ -0,0 +1,30 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type { AnnotationContext } from './annotation-context'; + +type Constructor = new (...args: any[]) => T; +type AnnotationContextConstructor = Constructor; + +export type Scored = { + score: number; + votes: number; +}; + +export function ScoredMixin( + Base: TBase, +): TBase & Constructor & Scored> { + return class ScoredAnnotation extends Base { + public score: number; + public votes: number; + + constructor(...args: any[]) { + super(...args); + const [data] = args as [{ score: number }]; + const { replicasCount } = this; + this.score = data.score ?? 1; + this.votes = replicasCount !== undefined ? Math.round(this.score * replicasCount) : 0; + } + } as TBase & Constructor & Scored>; +} diff --git a/cvat-core/src/annotations-objects/shape.ts b/cvat-core/src/annotations-objects/shape.ts new file mode 100644 index 0000000..e75f79b --- /dev/null +++ b/cvat-core/src/annotations-objects/shape.ts @@ -0,0 +1,321 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { omit } from 'lodash'; +import ObjectState, { type SerializedData } from '../object-state'; +import { ScriptingError } from '../exceptions'; +import { ObjectType, HistoryActions } from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { computeNewSource, isChildObject, serializeAttributes } from './utils'; +import { Drawn } from './drawn'; +import type { AnnotationInjection } from './types'; +import { ScoredMixin } from './scored'; + +export class Shape extends ScoredMixin(Drawn) { + public points: number[]; + public occluded: boolean; + public outside: boolean; + public rotation: number; + public zOrder: number; + + constructor( + data: SerializedShape | SerializedShape['elements'][0], + clientID: number, + color: string, + injection: AnnotationInjection, + ) { + super(data, clientID, color, injection); + this.points = data.points; + this.rotation = +(data.rotation ?? 0).toFixed(5); + this.occluded = data.occluded || false; + this.outside = data.outside || false; + this.zOrder = data.z_order; + } + + protected withContext(): ReturnType & { + save: (data: ObjectState) => ObjectState; + export: () => SerializedShape; + } { + return { + ...super.withContext(), + save: this.save.bind(this), + export: this.toJSON.bind(this), + }; + } + + // Method is used to export data to the server + public toJSON(): SerializedShape | SerializedShape['elements'][0] { + const result: SerializedShape = { + type: this.shapeType, + clientID: this.clientID, + occluded: this.occluded, + outside: this.outside, + z_order: this.zOrder, + points: this.points.slice(), + rotation: this.rotation, + attributes: serializeAttributes(this.attributes), + elements: [], + frame: this.frame, + label_id: this.label.id, + group: this.group, + source: this.source, + score: this.score, + }; + + if (typeof this._serverId === 'number') { + result.id = this._serverId; + } + + if (isChildObject(this._parentId)) { + return omit(result, 'elements'); + } + + return result; + } + + public get(frame): { outside?: boolean } & Omit, 'keyframe' | 'keyframes' | 'elements' | 'outside'> { + if (frame !== this.frame) { + throw new ScriptingError('Received frame is not equal to the frame of the shape'); + } + + const result: ReturnType = { + objectType: ObjectType.SHAPE, + shapeType: this.shapeType, + clientID: this.clientID, + serverID: this._serverId ?? null, + parentID: this._parentId ?? null, + occluded: this.occluded, + lock: this.lock, + zOrder: this.zOrder, + points: this.points.slice(), + rotation: this.rotation, + attributes: Object.fromEntries(this.attributes), + descriptions: [...this.descriptions], + label: this.label, + group: this.groupObject, + color: this.color, + hidden: this.hidden, + updated: this.updated, + pinned: this.pinned, + frame, + source: this.source, + score: this.score, + votes: this.votes, + __internal: this.withContext(), + }; + + if (typeof this.outside !== 'undefined') { + result.outside = this.outside; + } + + return result; + } + + public updateFromServerResponse(body: { id: number }): void { + this._serverId = body.id; + } + + protected saveRotation(rotation: number, frame: number): void { + const undoRotation = this.rotation; + const redoRotation = rotation; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + this.history.do( + HistoryActions.CHANGED_ROTATION, + () => { + this.source = undoSource; + this.rotation = undoRotation; + this.updated = Date.now(); + }, + () => { + this.source = redoSource; + this.rotation = redoRotation; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.source = redoSource; + this.rotation = redoRotation; + } + + protected savePoints(points: number[], frame: number): void { + const undoPoints = this.points; + const redoPoints = points; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + this.history.do( + HistoryActions.CHANGED_POINTS, + () => { + this.points = undoPoints; + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + this.points = redoPoints; + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.source = redoSource; + this.points = redoPoints; + } + + protected saveOccluded(occluded: boolean, frame: number): void { + const undoOccluded = this.occluded; + const redoOccluded = occluded; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + this.history.do( + HistoryActions.CHANGED_OCCLUDED, + () => { + this.occluded = undoOccluded; + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + this.occluded = redoOccluded; + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.source = redoSource; + this.occluded = redoOccluded; + } + + protected saveOutside(outside: boolean, frame: number): void { + const undoOutside = this.outside; + const redoOutside = outside; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + this.history.do( + HistoryActions.CHANGED_OCCLUDED, + () => { + this.outside = undoOutside; + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + this.occluded = redoOutside; + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.source = redoSource; + this.outside = redoOutside; + } + + protected saveZOrder(zOrder: number, frame: number): void { + const undoZOrder = this.zOrder; + const redoZOrder = zOrder; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + this.history.do( + HistoryActions.CHANGED_ZORDER, + () => { + this.zOrder = undoZOrder; + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + this.zOrder = redoZOrder; + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + + this.source = redoSource; + this.zOrder = redoZOrder; + } + + public save(frame: number, data: ObjectState): ObjectState { + if (frame !== this.frame) { + throw new ScriptingError('Received frame is not equal to the frame of the shape'); + } + + if (this.lock && data.lock) { + return new ObjectState(this.get(frame)); + } + + const updated = data.updateFlags; + for (const readOnlyField of this.readOnlyFields) { + delete updated[readOnlyField]; + } + + const fittedPoints = this.validateStateBeforeSave(data, updated, frame); + const { rotation } = data; + + // Now when all fields are validated, we can apply them + if (updated.label) { + this.saveLabel(data.label, frame); + } + + if (updated.attributes) { + this.saveAttributes(data.attributes, frame); + } + + if (updated.descriptions) { + this.saveDescriptions(data.descriptions); + } + + if (updated.rotation) { + this.saveRotation(rotation, frame); + } + + if (updated.points && fittedPoints.length) { + this.savePoints(fittedPoints, frame); + } + + if (updated.occluded) { + this.saveOccluded(data.occluded, frame); + } + + if (updated.outside) { + this.saveOutside(data.outside, frame); + } + + if (updated.zOrder) { + this.saveZOrder(data.zOrder, frame); + } + + if (updated.lock) { + this.saveLock(data.lock, frame); + } + + if (updated.pinned) { + this.savePinned(data.pinned, frame); + } + + if (updated.color) { + this.saveColor(data.color, frame); + } + + if (updated.hidden) { + this.saveHidden(data.hidden, frame); + } + + this.updateTimestamp(updated); + updated.reset(); + + return new ObjectState(this.get(frame)); + } +} diff --git a/cvat-core/src/annotations-objects/skeleton-shape.ts b/cvat-core/src/annotations-objects/skeleton-shape.ts new file mode 100644 index 0000000..a327b2f --- /dev/null +++ b/cvat-core/src/annotations-objects/skeleton-shape.ts @@ -0,0 +1,321 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import ObjectState, { type SerializedData } from '../object-state'; +import { ScriptingError } from '../exceptions'; +import type { Label } from '../labels'; +import { + ShapeType, ObjectType, HistoryActions, +} from '../enums'; +import type { SerializedShape } from '../server-response-types'; +import { computeWrappingBox, rotatePoint } from '../object-utils'; +import { Shape } from './shape'; +import type { AnnotationInjection } from './types'; +import { computeNewSource, serializeAttributes } from './utils'; +// eslint-disable-next-line import/no-cycle +import { shapeFactory } from './index'; + +export class SkeletonShape extends Shape { + public elements: Shape[]; + + constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.SKELETON; + this.pinned = false; + this.rotation = 0; + this.occluded = false; + this.points = []; + this.readOnlyFields = ['points', 'label', 'occluded']; + + const [cx, cy] = data.elements.reduce((acc, element, idx) => { + const result = [acc[0] + element.points[0], acc[1] + element.points[1]]; + if (idx === data.elements.length - 1) { + // length can not be 0 because we are inside reduce + result[0] /= data.elements.length; + result[1] /= data.elements.length; + } + return result; + }, [0, 0]); + + this.elements = this.label.structure.sublabels.map((sublabel: Label) => { + const element = data.elements.find((_element) => _element.label_id === sublabel.id); + const elementData = element || { + label_id: sublabel.id, + attributes: [], + occluded: false, + outside: true, + points: [cx, cy], + type: sublabel.type as unknown as ShapeType, + }; + + return shapeFactory({ + ...elementData, + group: this.group, + z_order: this.zOrder, + source: this.source, + rotation: 0, + frame: data.frame, + }, injection.nextClientID(), { + ...injection, + parentId: this.clientID, + }); + }); + } + + static distance(points: number[], x: number, y: number): number | null { + const distances = []; + let xtl = Number.MAX_SAFE_INTEGER; + let ytl = Number.MAX_SAFE_INTEGER; + let xbr = 0; + let ybr = 0; + + const MARGIN = 20; + for (let i = 0; i < points.length; i += 2) { + const x1 = points[i]; + const y1 = points[i + 1]; + xtl = Math.min(x1, xtl); + ytl = Math.min(y1, ytl); + xbr = Math.max(x1, xbr); + ybr = Math.max(y1, ybr); + + distances.push(Math.sqrt((x1 - x) ** 2 + (y1 - y) ** 2)); + } + + xtl -= MARGIN; + xbr += MARGIN; + ytl -= MARGIN; + ybr += MARGIN; + + if (!(x >= xtl && x <= xbr && y >= ytl && y <= ybr)) { + // Cursor is outside of a box + return null; + } + + return Math.min.apply(null, distances); + } + + // Method is used to export data to the server + public toJSON(): SerializedShape { + const elements = this.elements.map((element) => ({ + ...element.toJSON(), + outside: element.outside, + points: [...element.points], + source: this.source, + group: this.group, + z_order: this.zOrder, + rotation: 0, + })); + + const result: SerializedShape = { + type: this.shapeType, + clientID: this.clientID, + occluded: elements.every((el) => el.occluded), + outside: elements.every((el) => el.outside), + z_order: this.zOrder, + points: [], + rotation: 0, + attributes: serializeAttributes(this.attributes), + elements, + frame: this.frame, + label_id: this.label.id, + group: this.group, + source: this.source, + score: this.score, + }; + + if (typeof this._serverId === 'number') { + result.id = this._serverId; + } + + return result; + } + + public get(frame): Omit, 'keyframe' | 'keyframes'> { + if (frame !== this.frame) { + throw new ScriptingError('Received frame is not equal to the frame of the shape'); + } + + const elements = this.elements.map((element) => ({ + ...element.get(frame), + source: this.source, + group: this.groupObject, + zOrder: this.zOrder, + rotation: 0, + })); + + return { + objectType: ObjectType.SHAPE, + shapeType: this.shapeType, + clientID: this.clientID, + serverID: this._serverId ?? null, + parentID: this._parentId ?? null, + points: this.points, + zOrder: this.zOrder, + rotation: 0, + attributes: Object.fromEntries(this.attributes), + descriptions: [...this.descriptions], + elements, + label: this.label, + group: this.groupObject, + color: this.color, + updated: Math.max(this.updated, ...this.elements.map((element) => element.updated)), + pinned: this.pinned, + outside: elements.every((el) => el.outside), + occluded: elements.every((el) => el.occluded), + lock: elements.every((el) => el.lock), + hidden: elements.every((el) => el.hidden), + frame, + source: this.source, + score: this.score, + votes: this.votes, + __internal: this.withContext(), + }; + } + + public updateFromServerResponse( + body: Parameters[0] & { + elements?: { id: number, label_id: number }[]; + }, + ): void { + super.updateFromServerResponse(body); + if ('elements' in body) { + for (const element of body.elements) { + const context = this.elements.find((_element: Shape) => _element.label.id === element.label_id); + context.updateFromServerResponse(element); + } + } + } + + public clearServerId(): void { + super.clearServerId(); + for (const element of this.elements) { + element.clearServerId(); + } + } + + protected saveRotation(rotation, frame): void { + const undoSkeletonPoints = this.elements.map((element) => element.points); + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + const bbox = computeWrappingBox(undoSkeletonPoints.flat()); + const [cx, cy] = [bbox.x + bbox.width / 2, bbox.y + bbox.height / 2]; + for (const element of this.elements) { + const { points } = element; + const rotatedPoints = []; + for (let i = 0; i < points.length; i += 2) { + const [x, y] = [points[i], points[i + 1]]; + rotatedPoints.push(...rotatePoint(x, y, rotation, cx, cy)); + } + + element.points = rotatedPoints; + } + this.source = redoSource; + + const redoSkeletonPoints = this.elements.map((element) => element.points); + this.history.do( + HistoryActions.CHANGED_ROTATION, + () => { + for (let i = 0; i < this.elements.length; i++) { + this.elements[i].points = undoSkeletonPoints[i]; + this.elements[i].updated = Date.now(); + } + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + for (let i = 0; i < this.elements.length; i++) { + this.elements[i].points = redoSkeletonPoints[i]; + this.elements[i].updated = Date.now(); + } + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID, ...this.elements.map((element) => element.clientID)], + frame, + ); + } + + public save(frame: number, data: ObjectState): ObjectState { + if (this.lock && data.lock) { + return new ObjectState(this.get(frame)); + } + + const updateElements = ( + affectedElements: ObjectState[], + action: HistoryActions, + property: K, + ): void => { + const undoSkeletonProperties = this.elements.map((element): Shape[K] => element[property]); + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + try { + this.history.freeze(true); + affectedElements.forEach((element, idx) => { + const annotationContext = this.elements[idx]; + annotationContext.save(frame, element); + }); + } finally { + this.history.freeze(false); + } + + const redoSkeletonProperties = this.elements.map((element): Shape[K] => element[property]); + + this.history.do( + action, + () => { + for (let i = 0; i < this.elements.length; i++) { + this.elements[i][property] = undoSkeletonProperties[i]; + this.elements[i].updated = Date.now(); + } + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + for (let i = 0; i < this.elements.length; i++) { + this.elements[i][property] = redoSkeletonProperties[i]; + this.elements[i].updated = Date.now(); + } + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID, ...affectedElements.map((element) => element.clientID)], + frame, + ); + }; + + const updatedPoints = data.elements.filter((el) => el.updateFlags.points); + const updatedOccluded = data.elements.filter((el) => el.updateFlags.occluded); + const updatedHidden = data.elements.filter((el) => el.updateFlags.hidden); + const updatedLock = data.elements.filter((el) => el.updateFlags.lock); + + updatedOccluded.forEach((el) => { el.updateFlags.occluded = false; }); + updatedHidden.forEach((el) => { el.updateFlags.hidden = false; }); + updatedLock.forEach((el) => { el.updateFlags.lock = false; }); + + if (updatedPoints.length) { + updateElements(updatedPoints, HistoryActions.CHANGED_POINTS, 'points'); + } + + if (updatedOccluded.length) { + updatedOccluded.forEach((el) => { el.updateFlags.occluded = true; }); + updateElements(updatedOccluded, HistoryActions.CHANGED_OCCLUDED, 'occluded'); + } + + if (updatedHidden.length) { + updatedHidden.forEach((el) => { el.updateFlags.hidden = true; }); + updateElements(updatedHidden, HistoryActions.CHANGED_OUTSIDE, 'hidden'); + } + + if (updatedLock.length) { + updatedLock.forEach((el) => { el.updateFlags.lock = true; }); + updateElements(updatedLock, HistoryActions.CHANGED_LOCK, 'lock'); + } + + const result = Shape.prototype.save.call(this, frame, data); + return result; + } +} diff --git a/cvat-core/src/annotations-objects/skeleton-track.ts b/cvat-core/src/annotations-objects/skeleton-track.ts new file mode 100644 index 0000000..00d09cd --- /dev/null +++ b/cvat-core/src/annotations-objects/skeleton-track.ts @@ -0,0 +1,416 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import ObjectState, { type SerializedData } from '../object-state'; +import { + ShapeType, ObjectType, HistoryActions, +} from '../enums'; +import type { SerializedTrack } from '../server-response-types'; +import { computeWrappingBox, rotatePoint } from '../object-utils'; +import { type ImageObject, InterpolationNotPossibleError } from './image-object'; +import { Track } from './track'; +import type { AnnotationInjection, InterpolatedPosition } from './types'; +import { computeNewSource, copyShape } from './utils'; +// eslint-disable-next-line import/no-cycle +import { SkeletonShape } from './skeleton-shape'; +import { trackFactory } from './index'; + +export class SkeletonTrack extends Track { + public elements: Track[]; + + constructor(data: SerializedTrack, clientID: number, color: string, injection: AnnotationInjection) { + super(data, clientID, color, injection); + this.shapeType = ShapeType.SKELETON; + this.readOnlyFields = ['points', 'label', 'occluded', 'outside']; + this.pinned = false; + + const [cx, cy] = data.elements.reduce((acc, element, idx) => { + const shape = element.shapes[0]; + if (!shape || shape.frame !== this.frame) { + return acc; + } + + const result = [acc[0] + shape.points[0], acc[1] + shape.points[1], acc[2] + 1]; + if (idx === data.elements.length - 1) { + // avoid division by 0, additionally + return [result[0] / (result[2] || 1), result[1] / (result[2] || 1)]; + } + + return result; + }, [0, 0, 0]); + + this.elements = this.label.structure.sublabels.map((sublabel) => { + const element = data.elements.find((_element) => _element.label_id === sublabel.id); + const elementData = element || { + label_id: sublabel.id, + frame: this.frame, + attributes: [], + shapes: [{ + attributes: [], + points: [cx, cy], + frame: this.frame, + occluded: false, + outside: true, + rotation: 0, + type: sublabel.type as unknown as ShapeType, + }], + }; + + return trackFactory({ + ...elementData, + group: this.group, + source: this.source, + shapes: elementData.shapes.map((shape) => ({ + ...shape, + z_order: this.shapes[shape.frame]?.zOrder || 0, + })), + }, injection.nextClientID(), { + ...injection, + parentId: this.clientID, + }); + }).filter(Boolean).sort((a: ImageObject, b: ImageObject) => a.label.id - b.label.id); + } + + public updateFromServerResponse(body: Parameters[0] & { + elements?: { + id: number; + frame: number; + label_id: number; + shapes: SerializedTrack['shapes']; + }[]; + }): void { + super.updateFromServerResponse(body); + if ('elements' in body) { + for (const element of body.elements) { + const context = this.elements.find((_element: Track) => _element.label.id === element.label_id); + context.updateFromServerResponse(element); + } + } + } + + public clearServerId(): void { + super.clearServerId(); + for (const element of this.elements) { + element.clearServerId(); + } + } + + protected saveRotation(rotation: number, frame: number): void { + const undoSkeletonShapes = this.elements.map((element) => element.shapes[frame]); + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + const elementsData = this.elements.map((element) => element.get(frame)); + const skeletonPoints = elementsData.map((data) => data.points); + const bbox = computeWrappingBox(skeletonPoints.flat()); + const [cx, cy] = [bbox.x + bbox.width / 2, bbox.y + bbox.height / 2]; + + for (let i = 0; i < this.elements.length; i++) { + const element = this.elements[i]; + const { points } = elementsData[i]; + + const rotatedPoints = []; + for (let j = 0; j < points.length; j += 2) { + const [x, y] = [points[j], points[j + 1]]; + rotatedPoints.push(...rotatePoint(x, y, rotation, cx, cy)); + } + + if (undoSkeletonShapes[i]) { + element.shapes[frame] = { + ...undoSkeletonShapes[i], + points: rotatedPoints, + }; + } else { + element.shapes[frame] = { + ...copyShape(elementsData[i]), + points: rotatedPoints, + }; + } + } + this.source = redoSource; + + const redoSkeletonShapes = this.elements.map((element) => element.shapes[frame]); + this.history.do( + HistoryActions.CHANGED_ROTATION, + () => { + for (let i = 0; i < this.elements.length; i++) { + const element = this.elements[i]; + if (undoSkeletonShapes[i]) { + element.shapes[frame] = undoSkeletonShapes[i]; + } else { + delete element.shapes[frame]; + } + this.updated = Date.now(); + } + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + for (let i = 0; i < this.elements.length; i++) { + const element = this.elements[i]; + element.shapes[frame] = redoSkeletonShapes[i]; + this.updated = Date.now(); + } + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID, ...this.elements.map((element) => element.clientID)], + frame, + ); + } + + // Method is used to export data to the server + public toJSON(): SerializedTrack { + const result: SerializedTrack = Track.prototype.toJSON.call(this); + + result.shapes = result.shapes.map((shape) => ({ + ...shape, + points: [], + })); + + result.elements = this.elements.map((el) => ({ + ...el.toJSON(), + source: this.source, + group: this.group, + })); + + result.elements.forEach((element) => { + element.shapes.forEach((shape) => { + shape.rotation = 0; + const { frame } = shape; + const skeletonShape = result.shapes + .find((_skeletonShape) => _skeletonShape.frame === frame); + if (skeletonShape) { + shape.z_order = skeletonShape.z_order; + } + }); + }); + + return result; + } + + public get(frame: number): Omit, 'score' | 'votes'> { + const { prev, next } = this.boundedKeyframes(frame); + const position = this.getPosition(frame, prev, next); + const elements = this.elements.map((element) => ({ + ...element.get(frame), + source: this.source, + group: this.groupObject, + zOrder: position.zOrder, + rotation: 0, + })); + + return { + ...position, + parentID: null, + keyframe: position.keyframe || elements.some((el) => el.keyframe), + attributes: this.getAttributes(frame), + descriptions: [...this.descriptions], + group: this.groupObject, + objectType: ObjectType.TRACK, + shapeType: this.shapeType, + clientID: this.clientID, + serverID: this._serverId ?? null, + color: this.color, + updated: Math.max(this.updated, ...this.elements.map((element) => element.updated)), + label: this.label, + pinned: this.pinned, + keyframes: this.deepBoundedKeyframes(frame), + elements, + frame, + source: this.source, + outside: elements.every((el) => el.outside), + occluded: elements.every((el) => el.occluded), + lock: elements.every((el) => el.lock), + hidden: elements.every((el) => el.hidden), + __internal: this.withContext(), + }; + } + + // finds keyframes considering keyframes of nested elements + private deepBoundedKeyframes(targetFrame: number): ObjectState['keyframes'] { + const boundedKeyframes = Track.prototype.boundedKeyframes.call(this, targetFrame); + + for (const element of this.elements) { + const keyframes = element.boundedKeyframes(targetFrame); + if (keyframes.prev !== null) { + boundedKeyframes.prev = boundedKeyframes.prev === null || keyframes.prev > boundedKeyframes.prev ? + keyframes.prev : boundedKeyframes.prev; + } + + if (keyframes.next !== null) { + boundedKeyframes.next = boundedKeyframes.next === null || keyframes.next < boundedKeyframes.next ? + keyframes.next : boundedKeyframes.next; + } + + if (keyframes.first !== null) { + boundedKeyframes.first = + boundedKeyframes.first === null || keyframes.first < boundedKeyframes.first ? + keyframes.first : boundedKeyframes.first; + } + + if (keyframes.last !== null) { + boundedKeyframes.last = boundedKeyframes.last === null || keyframes.last > boundedKeyframes.last ? + keyframes.last : boundedKeyframes.last; + } + } + + return boundedKeyframes; + } + + public save(frame: number, data: ObjectState): ObjectState { + if (this.lock && data.lock) { + return new ObjectState(this.get(frame)); + } + + const updateElements = (affectedElements, action, property: 'hidden' | 'lock' | null = null): void => { + const undoSkeletonProperties = this.elements.map((element) => element[property] || null); + const undoSkeletonShapes = this.elements.map((element) => element.shapes[frame]); + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + + const errors = []; + try { + this.history.freeze(true); + affectedElements.forEach((element, idx) => { + try { + const annotationContext = this.elements[idx]; + annotationContext.save(frame, element); + } catch (error: any) { + errors.push(error); + } + }); + } finally { + this.history.freeze(false); + } + + const redoSkeletonProperties = this.elements.map((element) => element[property] || null); + const redoSkeletonShapes = this.elements.map((element) => element.shapes[frame]); + + this.history.do( + action, + () => { + for (let i = 0; i < this.elements.length; i++) { + if (property) { + this.elements[i][property] = undoSkeletonProperties[i]; + } if (undoSkeletonShapes[i]) { + this.elements[i].shapes[frame] = undoSkeletonShapes[i]; + } else if (redoSkeletonShapes[i]) { + delete this.elements[i].shapes[frame]; + } + this.elements[i].updated = Date.now(); + } + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + for (let i = 0; i < this.elements.length; i++) { + if (property) { + this.elements[i][property] = redoSkeletonProperties[i]; + } else if (redoSkeletonShapes[i]) { + this.elements[i].shapes[frame] = redoSkeletonShapes[i]; + } else if (undoSkeletonShapes[i]) { + delete this.elements[i].shapes[frame]; + } + this.elements[i].updated = Date.now(); + } + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID, ...affectedElements.map((element) => element.clientID)], + frame, + ); + + if (errors.length) { + throw new Error(`Several errors occurred during saving skeleton:\n ${errors.join(';\n')}`); + } + }; + + const updatedPoints = data.elements.filter((el) => el.updateFlags.points); + const updatedOccluded = data.elements.filter((el) => el.updateFlags.occluded); + const updatedOutside = data.elements.filter((el) => el.updateFlags.outside); + const updatedKeyframe = data.elements.filter((el) => el.updateFlags.keyframe); + const updatedHidden = data.elements.filter((el) => el.updateFlags.hidden); + const updatedLock = data.elements.filter((el) => el.updateFlags.lock); + + updatedOccluded.forEach((el) => { el.updateFlags.occluded = false; }); + updatedOutside.forEach((el) => { el.updateFlags.outside = false; }); + updatedKeyframe.forEach((el) => { el.updateFlags.keyframe = false; }); + updatedHidden.forEach((el) => { el.updateFlags.hidden = false; }); + updatedLock.forEach((el) => { el.updateFlags.lock = false; }); + + if (updatedPoints.length) { + updateElements(updatedPoints, HistoryActions.CHANGED_POINTS); + } + + if (updatedOccluded.length) { + updatedOccluded.forEach((el) => { el.updateFlags.occluded = true; }); + updateElements(updatedOccluded, HistoryActions.CHANGED_OCCLUDED); + } + + if (updatedOutside.length) { + updatedOutside.forEach((el) => { el.updateFlags.outside = true; }); + updateElements(updatedOutside, HistoryActions.CHANGED_OUTSIDE); + } + + if (updatedKeyframe.length) { + updatedKeyframe.forEach((el) => { el.updateFlags.keyframe = true; }); + // todo: fix extra undo/redo change + this.validateStateBeforeSave(data, data.updateFlags, frame); + this.saveKeyframe(frame, data.keyframe); + // eslint-disable-next-line no-param-reassign + data.updateFlags.keyframe = false; + updateElements(updatedKeyframe, HistoryActions.CHANGED_KEYFRAME); + } + + if (updatedHidden.length) { + updatedHidden.forEach((el) => { el.updateFlags.hidden = true; }); + updateElements(updatedHidden, HistoryActions.CHANGED_HIDDEN, 'hidden'); + } + + if (updatedLock.length) { + updatedLock.forEach((el) => { el.updateFlags.lock = true; }); + updateElements(updatedLock, HistoryActions.CHANGED_LOCK, 'lock'); + } + + const result = Track.prototype.save.call(this, frame, data); + return result; + } + + protected getPosition( + targetFrame: number, leftKeyframe: number | null, rightKeyframe: number | null, + ): InterpolatedPosition & { keyframe: boolean } { + const leftFrame = targetFrame in this.shapes ? targetFrame : leftKeyframe; + const rightPosition = Number.isInteger(rightKeyframe) ? this.shapes[rightKeyframe] : null; + const leftPosition = Number.isInteger(leftFrame) ? this.shapes[leftFrame] : null; + + if (leftPosition && rightPosition) { + return { + rotation: 0, + occluded: leftPosition.occluded, + outside: leftPosition.outside, + zOrder: leftPosition.zOrder, + keyframe: targetFrame in this.shapes, + points: [], + }; + } + + const singlePosition = leftPosition || rightPosition; + if (singlePosition) { + return { + rotation: 0, + occluded: singlePosition.occluded, + zOrder: singlePosition.zOrder, + keyframe: targetFrame in this.shapes, + outside: singlePosition === rightPosition ? true : singlePosition.outside, + points: [], + }; + } + + throw new InterpolationNotPossibleError(); + } +} + +Object.defineProperty(SkeletonTrack, 'distance', { value: SkeletonShape.distance }); diff --git a/cvat-core/src/annotations-objects/tag.ts b/cvat-core/src/annotations-objects/tag.ts new file mode 100644 index 0000000..68cc57a --- /dev/null +++ b/cvat-core/src/annotations-objects/tag.ts @@ -0,0 +1,110 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import ObjectState, { type SerializedData } from '../object-state'; +import { ScriptingError } from '../exceptions'; +import { ObjectType } from '../enums'; +import type { SerializedTag } from '../server-response-types'; +import { ImageObject } from './image-object'; +import { serializeAttributes } from './utils'; + +export class Tag extends ImageObject { + protected withContext(): ReturnType & { + save: (data: ObjectState) => ObjectState; + export: () => SerializedTag; + } { + return { + ...super.withContext(), + save: this.save.bind(this), + export: this.toJSON.bind(this), + }; + } + + // Method is used to export data to the server + public toJSON(): SerializedTag { + const result: SerializedTag = { + clientID: this.clientID, + frame: this.frame, + label_id: this.label.id, + source: this.source, + group: 0, // TODO: why server requires group for tags? + attributes: serializeAttributes(this.attributes), + }; + + if (typeof this._serverId === 'number') { + result.id = this._serverId; + } + + return result; + } + + public get(frame: number): Omit, + 'elements' | 'occluded' | 'outside' | 'rotation' | 'zOrder' | + 'points' | 'hidden' | 'pinned' | 'keyframe' | 'shapeType' | + 'parentID' | 'descriptions' | 'keyframes' | 'score' | 'votes' + > { + if (frame !== this.frame) { + throw new ScriptingError('Received frame is not equal to the frame of the shape'); + } + + return { + objectType: ObjectType.TAG, + clientID: this.clientID, + serverID: this._serverId ?? null, + lock: this.lock, + attributes: Object.fromEntries(this.attributes), + label: this.label, + group: this.groupObject, + color: this.color, + updated: this.updated, + frame, + source: this.source, + __internal: this.withContext(), + }; + } + + public updateFromServerResponse(body: { id: number }): void { + this._serverId = body.id; + } + + public save(frame: number, data: ObjectState): ObjectState { + if (frame !== this.frame) { + throw new ScriptingError('Received frame is not equal to the frame of the tag'); + } + + if (this.lock && data.lock) { + return new ObjectState(this.get(frame)); + } + + const updated = data.updateFlags; + for (const readOnlyField of this.readOnlyFields) { + delete updated[readOnlyField]; + } + + this.validateStateBeforeSave(data, updated); + + // Now when all fields are validated, we can apply them + if (updated.label) { + this.saveLabel(data.label, frame); + } + + if (updated.attributes) { + this.saveAttributes(data.attributes, frame); + } + + if (updated.lock) { + this.saveLock(data.lock, frame); + } + + if (updated.color) { + this.saveColor(data.color, frame); + } + + this.updateTimestamp(updated); + updated.reset(); + + return new ObjectState(this.get(frame)); + } +} diff --git a/cvat-core/src/annotations-objects/track.ts b/cvat-core/src/annotations-objects/track.ts new file mode 100644 index 0000000..efaf60a --- /dev/null +++ b/cvat-core/src/annotations-objects/track.ts @@ -0,0 +1,583 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { omit } from 'lodash'; +import ObjectState, { type SerializedData } from '../object-state'; +import { ObjectType, HistoryActions } from '../enums'; +import type { Label } from '../labels'; +import type { SerializedTrack } from '../server-response-types'; +import { attrsAsAnObject } from '../object-utils'; +import { Drawn } from './drawn'; +import { InterpolationNotPossibleError } from './image-object'; +import type { AnnotationInjection, InterpolatedPosition, TrackedShape } from './types'; +import { + computeNewSource, deserializeTrackedShapes, copyShape, isChildObject, serializeAttributes, +} from './utils'; + +export class Track extends Drawn { + public shapes: Record; + constructor( + data: SerializedTrack | SerializedTrack['elements'][0], + clientID: number, + color: string, + injection: AnnotationInjection, + ) { + super(data, clientID, color, injection); + this.shapes = deserializeTrackedShapes(data.shapes); + } + + protected withContext(): ReturnType & { + save: (data: ObjectState) => ObjectState; + export: () => SerializedTrack; + } { + return { + ...super.withContext(), + save: this.save.bind(this), + export: this.toJSON.bind(this), + }; + } + + // Method is used to export data to the server + public toJSON(): SerializedTrack | SerializedTrack['elements'][0] { + const labelAttributes = attrsAsAnObject(this.label.attributes); + + const result: SerializedTrack = { + clientID: this.clientID, + label_id: this.label.id, + frame: this.frame, + group: this.group, + source: this.source, + attributes: serializeAttributes(this.attributes) + .filter((attr) => !labelAttributes[attr.spec_id].mutable), + elements: [], + shapes: Object.keys(this.shapes).reduce((shapesAccumulator, frame) => { + shapesAccumulator.push({ + type: this.shapeType, + occluded: this.shapes[frame].occluded, + z_order: this.shapes[frame].zOrder, + rotation: this.shapes[frame].rotation, + outside: this.shapes[frame].outside, + attributes: serializeAttributes(this.shapes[frame].attributes) + .filter((attr) => labelAttributes[attr.spec_id].mutable), + id: this.shapes[frame].serverId, + frame: +frame, + }); + + if (this.shapes[frame].points) { + // eslint-disable-next-line no-param-reassign + shapesAccumulator[shapesAccumulator.length - 1].points = [...this.shapes[frame].points]; + } + + return shapesAccumulator; + }, []), + }; + + if (typeof this._serverId === 'number') { + result.id = this._serverId; + } + + if (isChildObject(this._parentId)) { + return omit(result, 'elements'); + } + + return result; + } + + public get(frame: number): Omit, 'elements' | 'score' | 'votes'> { + const { + prev, next, first, last, + } = this.boundedKeyframes(frame); + + return { + ...this.getPosition(frame, prev, next), + attributes: this.getAttributes(frame), + descriptions: [...this.descriptions], + group: this.groupObject, + objectType: ObjectType.TRACK, + shapeType: this.shapeType, + clientID: this.clientID, + serverID: this._serverId ?? null, + parentID: this._parentId ?? null, + lock: this.lock, + color: this.color, + hidden: this.hidden, + updated: this.updated, + label: this.label, + pinned: this.pinned, + keyframes: { + prev, + next, + first, + last, + }, + frame, + source: this.source, + __internal: this.withContext(), + }; + } + + public boundedKeyframes(targetFrame: number): ObjectState['keyframes'] { + const frames = Object.keys(this.shapes).map((frame) => +frame); + let lDiff = Number.MAX_SAFE_INTEGER; + let rDiff = Number.MAX_SAFE_INTEGER; + let first = Number.MAX_SAFE_INTEGER; + let last = Number.MIN_SAFE_INTEGER; + + for (const frame of frames) { + if (this.framesInfo.isFrameDeleted(frame)) { + continue; + } + + if (frame < first) { + first = frame; + } + if (frame > last) { + last = frame; + } + + const diff = Math.abs(targetFrame - frame); + + if (frame < targetFrame && diff < lDiff) { + lDiff = diff; + } else if (frame > targetFrame && diff < rDiff) { + rDiff = diff; + } + } + + const prev = lDiff === Number.MAX_SAFE_INTEGER ? null : targetFrame - lDiff; + const next = rDiff === Number.MAX_SAFE_INTEGER ? null : targetFrame + rDiff; + + return { + prev, + next, + first, + last, + }; + } + + protected getAttributes(targetFrame: number): Record { + const result = {}; + + // First of all copy all unmutable attributes + for (const [id, value] of this.attributes.entries()) { + result[id] = value; + } + + // Secondly get latest mutable attributes up to target frame + const frames = Object.keys(this.shapes).sort((a, b) => +a - +b); + for (const frame of frames) { + if (+frame <= targetFrame) { + const { attributes } = this.shapes[frame]; + for (const [id, value] of attributes.entries()) { + result[id] = value; + } + } + } + + return result; + } + + public updateFromServerResponse(body: { + id: number; + frame: number; + shapes: SerializedTrack['shapes']; + }): void { + this._serverId = body.id; + this.frame = body.frame; + this.shapes = deserializeTrackedShapes(body.shapes); + } + + public clearServerId(): void { + super.clearServerId(); + for (const shape of Object.values(this.shapes)) { + shape.serverId = undefined; + } + } + + protected saveLabel(label: Label, frame: number): void { + const undoLabel = this.label; + const redoLabel = label; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + const undoAttributes = { + unmutable: new Map(this.attributes), + mutable: Object.keys(this.shapes).map((key) => ({ + frame: +key, + attributes: new Map(this.shapes[+key].attributes), + })), + }; + + this.label = label; + this.source = redoSource; + this.attributes = new Map(); + for (const shape of Object.values(this.shapes)) { + shape.attributes = new Map(); + } + this.appendDefaultAttributes(label); + + const redoAttributes = { + unmutable: new Map(this.attributes), + mutable: Object.keys(this.shapes).map((key) => ({ + frame: +key, + attributes: new Map(this.shapes[+key].attributes), + })), + }; + + this.history.do( + HistoryActions.CHANGED_LABEL, + () => { + this.label = undoLabel; + this.attributes = undoAttributes.unmutable; + this.source = undoSource; + for (const mutable of undoAttributes.mutable) { + this.shapes[mutable.frame].attributes = mutable.attributes; + } + this.updated = Date.now(); + }, + () => { + this.label = redoLabel; + this.attributes = redoAttributes.unmutable; + this.source = redoSource; + for (const mutable of redoAttributes.mutable) { + this.shapes[mutable.frame].attributes = mutable.attributes; + } + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + } + + protected saveAttributes(attributes: Record, frame: number): void { + const current = this.get(frame); + const labelAttributes = attrsAsAnObject(this.label.attributes); + + const wasKeyframe = frame in this.shapes; + const undoAttributes = new Map(this.attributes); + const undoShape = wasKeyframe ? this.shapes[frame] : undefined; + + let mutableAttributesUpdated = false; + const redoAttributes = new Map(this.attributes); + for (const [attrID, attrValue] of Object.entries(attributes)) { + if (!labelAttributes[attrID].mutable) { + redoAttributes.set(+attrID, attrValue); + } else if (attrValue !== current.attributes[attrID]) { + mutableAttributesUpdated = mutableAttributesUpdated || + // not keyframe yet + !(frame in this.shapes) || + // keyframe, but without this attrID + !this.shapes[frame].attributes.has(+attrID) || + // keyframe with attrID, but with another value + this.shapes[frame].attributes.get(+attrID) !== attrValue; + } + } + let redoShape: TrackedShape | undefined; + if (mutableAttributesUpdated) { + if (wasKeyframe) { + redoShape = { + ...this.shapes[frame], + attributes: new Map(this.shapes[frame].attributes), + }; + } else { + redoShape = copyShape(current); + } + } + + for (const [attrID, attrValue] of Object.entries(attributes)) { + if (redoShape && labelAttributes[attrID].mutable && attrValue !== current.attributes[attrID]) { + redoShape.attributes.set(+attrID, attrValue); + } + } + + this.attributes = redoAttributes; + if (redoShape) { + this.shapes[frame] = redoShape; + } + + this.history.do( + HistoryActions.CHANGED_ATTRIBUTES, + () => { + this.attributes = undoAttributes; + if (undoShape) { + this.shapes[frame] = undoShape; + } else if (redoShape) { + delete this.shapes[frame]; + } + this.updated = Date.now(); + }, + () => { + this.attributes = redoAttributes; + if (redoShape) { + this.shapes[frame] = redoShape; + } + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + } + + protected appendShapeActionToHistory(actionType, frame, undoShape, redoShape, undoSource, redoSource): void { + this.history.do( + actionType, + () => { + if (!undoShape) { + delete this.shapes[frame]; + } else { + this.shapes[frame] = undoShape; + } + this.source = undoSource; + this.updated = Date.now(); + }, + () => { + if (!redoShape) { + delete this.shapes[frame]; + } else { + this.shapes[frame] = redoShape; + } + this.source = redoSource; + this.updated = Date.now(); + }, + [this.clientID], + frame, + ); + } + + protected saveRotation(rotation: number, frame: number): void { + const wasKeyframe = frame in this.shapes; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + const undoShape = wasKeyframe ? this.shapes[frame] : undefined; + const redoShape = wasKeyframe ? + { ...this.shapes[frame], rotation } : copyShape(this.get(frame), { rotation }); + + this.shapes[frame] = redoShape; + this.source = redoSource; + this.appendShapeActionToHistory( + HistoryActions.CHANGED_ROTATION, + frame, + undoShape, + redoShape, + undoSource, + redoSource, + ); + } + + protected savePoints(points: number[], frame: number): void { + const wasKeyframe = frame in this.shapes; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + const undoShape = wasKeyframe ? this.shapes[frame] : undefined; + const redoShape = wasKeyframe ? + { ...this.shapes[frame], points } : copyShape(this.get(frame), { points }); + + this.shapes[frame] = redoShape; + this.source = redoSource; + this.appendShapeActionToHistory( + HistoryActions.CHANGED_POINTS, + frame, + undoShape, + redoShape, + undoSource, + redoSource, + ); + } + + protected saveOutside(frame: number, outside: boolean): void { + const wasKeyframe = frame in this.shapes; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + const undoShape = wasKeyframe ? this.shapes[frame] : undefined; + const redoShape = wasKeyframe ? + { ...this.shapes[frame], outside } : + copyShape(this.get(frame), { outside }); + + this.shapes[frame] = redoShape; + this.source = redoSource; + this.appendShapeActionToHistory( + HistoryActions.CHANGED_OUTSIDE, + frame, + undoShape, + redoShape, + undoSource, + redoSource, + ); + } + + protected saveOccluded(occluded: boolean, frame: number): void { + const wasKeyframe = frame in this.shapes; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + const undoShape = wasKeyframe ? this.shapes[frame] : undefined; + const redoShape = wasKeyframe ? + { ...this.shapes[frame], occluded } : + copyShape(this.get(frame), { occluded }); + + this.shapes[frame] = redoShape; + this.source = redoSource; + this.appendShapeActionToHistory( + HistoryActions.CHANGED_OCCLUDED, + frame, + undoShape, + redoShape, + undoSource, + redoSource, + ); + } + + protected saveZOrder(zOrder: number, frame: number): void { + const wasKeyframe = frame in this.shapes; + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + const undoShape = wasKeyframe ? this.shapes[frame] : undefined; + const redoShape = wasKeyframe ? + { ...this.shapes[frame], zOrder } : + copyShape(this.get(frame), { zOrder }); + + this.shapes[frame] = redoShape; + this.source = redoSource; + this.appendShapeActionToHistory( + HistoryActions.CHANGED_ZORDER, + frame, + undoShape, + redoShape, + undoSource, + redoSource, + ); + } + + protected saveKeyframe(frame: number, keyframe: boolean): void { + const wasKeyframe = frame in this.shapes; + + if ((keyframe && wasKeyframe) || (!keyframe && !wasKeyframe)) { + return; + } + + const undoSource = this.source; + const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source); + const undoShape = wasKeyframe ? this.shapes[frame] : undefined; + const redoShape = keyframe ? copyShape(this.get(frame)) : undefined; + + this.source = redoSource; + if (redoShape) { + this.shapes[frame] = redoShape; + } else { + delete this.shapes[frame]; + } + + this.appendShapeActionToHistory( + HistoryActions.CHANGED_KEYFRAME, + frame, + undoShape, + redoShape, + undoSource, + redoSource, + ); + } + + public save(frame: number, data: ObjectState): ObjectState { + if (this.lock && data.lock) { + return new ObjectState(this.get(frame)); + } + + const updated = data.updateFlags; + for (const readOnlyField of this.readOnlyFields) { + delete updated[readOnlyField]; + } + + const fittedPoints = this.validateStateBeforeSave(data, updated, frame); + const { rotation } = data; + + if (updated.label) { + this.saveLabel(data.label, frame); + } + + if (updated.lock) { + this.saveLock(data.lock, frame); + } + + if (updated.pinned) { + this.savePinned(data.pinned, frame); + } + + if (updated.color) { + this.saveColor(data.color, frame); + } + + if (updated.hidden) { + this.saveHidden(data.hidden, frame); + } + + if (updated.points && fittedPoints.length) { + this.savePoints(fittedPoints, frame); + } + + if (updated.rotation) { + this.saveRotation(rotation, frame); + } + + if (updated.outside) { + this.saveOutside(frame, data.outside); + } + + if (updated.occluded) { + this.saveOccluded(data.occluded, frame); + } + + if (updated.zOrder) { + this.saveZOrder(data.zOrder, frame); + } + + if (updated.attributes) { + this.saveAttributes(data.attributes, frame); + } + + if (updated.descriptions) { + this.saveDescriptions(data.descriptions); + } + + if (updated.keyframe) { + this.saveKeyframe(frame, data.keyframe); + } + + this.updateTimestamp(updated); + updated.reset(); + + return new ObjectState(this.get(frame)); + } + + protected getPosition( + targetFrame: number, leftKeyframe: number | null, rightFrame: number | null, + ): InterpolatedPosition & { keyframe: boolean } { + const leftFrame = targetFrame in this.shapes ? targetFrame : leftKeyframe; + const rightPosition = Number.isInteger(rightFrame) ? this.shapes[rightFrame] : null; + const leftPosition = Number.isInteger(leftFrame) ? this.shapes[leftFrame] : null; + + if (leftPosition && rightPosition) { + return { + ...(this as any).interpolatePosition( + leftPosition, + rightPosition, + (targetFrame - leftFrame) / (rightFrame - leftFrame), + ), + keyframe: targetFrame in this.shapes, + }; + } + + const singlePosition = leftPosition || rightPosition; + if (singlePosition) { + return { + points: [...singlePosition.points], + rotation: singlePosition.rotation, + occluded: singlePosition.occluded, + zOrder: singlePosition.zOrder, + keyframe: targetFrame in this.shapes, + outside: singlePosition === rightPosition ? true : singlePosition.outside, + }; + } + + throw new InterpolationNotPossibleError(); + } +} diff --git a/cvat-core/src/annotations-objects/types.ts b/cvat-core/src/annotations-objects/types.ts new file mode 100644 index 0000000..abdd687 --- /dev/null +++ b/cvat-core/src/annotations-objects/types.ts @@ -0,0 +1,87 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import type AnnotationHistory from '../annotations-history'; +import type { Label } from '../labels'; +import type { DimensionType, JobType } from '../enums'; +import type { MaskShape } from './mask-shape'; + +export type FrameInfo = { + width: number; + height: number; +}; + +export type GroupsInfo = { + max: number; + colors: Record; +}; + +export interface BasicInjection { + labels: Record; + groupsInfo: GroupsInfo; + framesInfo: Readonly<{ + [index: number]: Readonly; + isFrameDeleted: (frame: number) => boolean; + }>; + history: AnnotationHistory; + parentId?: number; + dimension: DimensionType; + jobType: JobType; // comes from job data or is set to JobType.ANNOTATION for task-level collections + replicasCount?: number; // comes from job data or is undefined for task-level collections + nextClientID: () => number; + getMasksOnFrame: (frame: number) => MaskShape[]; +} + +export type AnnotationInjection = BasicInjection & { + parentId?: number; +}; + +export interface TrackedShape { + serverId?: number; + occluded: boolean; + outside: boolean; + rotation: number; + zOrder: number; + points?: number[]; + attributes: Map; +} + +export interface InterpolatedPosition { + points: number[]; + rotation: number; + occluded: boolean; + outside: boolean; + zOrder: number; +} + +export interface Point2D { + x: number; + y: number; +} + +export interface CommonUpdateFlags { + label?: boolean; + attributes?: boolean; + lock?: boolean; + color?: boolean; + hidden?: boolean; + reset: () => void; +} + +export interface UpdateFlags extends CommonUpdateFlags { + description?: boolean; + points?: boolean; + rotation?: boolean; + outside?: boolean; + occluded?: boolean; + keyframe?: boolean; + zOrder?: boolean; + pinned?: boolean; + descriptions?: boolean; +} + +export interface AudioIntervalUpdateFlags extends CommonUpdateFlags { + position?: boolean; +} diff --git a/cvat-core/src/annotations-objects/utils.ts b/cvat-core/src/annotations-objects/utils.ts new file mode 100644 index 0000000..727f7a0 --- /dev/null +++ b/cvat-core/src/annotations-objects/utils.ts @@ -0,0 +1,70 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { Source } from '../enums'; +import type { SerializedAttributes, SerializedTrack } from '../server-response-types'; +import type { TrackedShape } from './types'; + +export const defaultGroupColor = '#E0E0E0'; + +type CopyShapeState = Omit; + +export function copyShape(state: CopyShapeState, data: Partial = {}): TrackedShape { + return { + rotation: state.rotation, + zOrder: state.zOrder, + points: state.points, + occluded: state.occluded, + outside: state.outside, + attributes: new Map(), + ...data, + }; +} + +export function deserializeAttributes(attributes: SerializedAttributes): Map { + const map = new Map(); + for (const attr of attributes) { + map.set(attr.spec_id, attr.value); + } + return map; +} + +export function serializeAttributes(attributes: Map): SerializedAttributes { + return Array.from(attributes.entries()).reduce((acc, [id, value]) => { + acc.push({ spec_id: id, value }); + return acc; + }, []); +} + +export function deserializeTrackedShapes(shapes: SerializedTrack['shapes']): Record { + return shapes.reduce((acc, shape) => { + acc[shape.frame] = { + serverId: shape.id, + occluded: shape.occluded, + zOrder: shape.z_order, + points: shape.points, + outside: shape.outside, + rotation: shape.rotation || 0, + attributes: deserializeAttributes(shape.attributes), + }; + return acc; + }, {} as Record); +} + +export function computeNewSource(currentSource: Source): Source { + if ([Source.AUTO, Source.SEMI_AUTO].includes(currentSource)) { + return Source.SEMI_AUTO; + } + + if (currentSource === Source.CONSENSUS) { + return Source.CONSENSUS; + } + + return Source.MANUAL; +} + +export function isChildObject(parentId?: number): boolean { + return typeof parentId === 'number'; +} diff --git a/cvat-core/src/annotations-saver.ts b/cvat-core/src/annotations-saver.ts new file mode 100644 index 0000000..033981d --- /dev/null +++ b/cvat-core/src/annotations-saver.ts @@ -0,0 +1,573 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import _ from 'lodash'; +import serverProxy, { sleep } from './server-proxy'; +import { Job, Task } from './session'; +import { DataError, ServerError } from './exceptions'; +import { SerializedCollection, SerializedTrack } from './server-response-types'; + +interface ExtractedIDs { + shapes: number[]; + tracks: number[]; + tags: number[]; + intervals: number[]; +} + +interface SplittedCollection { + created: SerializedCollection; + updated: SerializedCollection; + deleted: SerializedCollection; +} + +type CollectionObject = SerializedCollection[keyof SerializedCollection][number]; + +const COLLECTION_KEYS = ['shapes', 'tracks', 'tags', 'intervals'] as const; +const JSON_SERIALIZER_KEYS = [ + 'id', + 'label_id', + 'group', + 'frame', + 'start', + 'stop', + 'occluded', + 'z_order', + 'points', + 'rotation', + 'type', + 'shapes', + 'elements', + 'attributes', + 'value', + 'spec_id', + 'source', + 'outside', +]; + +const sortAttributes = (attributes: { spec_id: number }[]): { spec_id: number }[] => ( + attributes.sort(({ spec_id: specID1 }, { spec_id: specID2 }) => specID1 - specID2) +); + +const sortTrackedShapes = (shapes: SerializedTrack['shapes']): SerializedTrack['shapes'] => ( + shapes.sort(({ frame: frame1 }, { frame: frame2 }) => frame1 - frame2) +); + +const isTheSameAttributes = ( + a: { spec_id: number }[], + b: { spec_id: number }[], +): boolean => ( + JSON.stringify(sortAttributes(a)) === JSON.stringify(sortAttributes(b)) +); + +const isTheSamePoints = ( + a: number[] | undefined, + b: number[] | undefined, +): boolean => ( + (a === undefined && b === undefined) || + (a.length === b.length && a.every((coord, index) => coord.toFixed(1) === b[index].toFixed(1))) +); + +const isTheSameTrackedShapes = ( + a: SerializedTrack['shapes'], + b: SerializedTrack['shapes'], +): boolean => { + const sortedA = sortTrackedShapes(a); + const sortedB = sortTrackedShapes(b); + + return sortedA.length === sortedB.length && + sortedA.every((shape, index) => { + // The server can extend tracked shape attributes with defaults or previous mutable values. + // During 504 recovery, only attributes sent by the client must be preserved for matching. + const receivedAttributes = sortedB[index].attributes.filter((attr) => ( + shape.attributes.some((sentAttr) => sentAttr.spec_id === attr.spec_id) + )); + + return shape.frame === sortedB[index].frame && + shape.type === sortedB[index].type && + shape.occluded === sortedB[index].occluded && + shape.outside === sortedB[index].outside && + shape.z_order === sortedB[index].z_order && + shape.rotation === sortedB[index].rotation && + isTheSamePoints(shape.points, sortedB[index].points) && + isTheSameAttributes(shape.attributes, receivedAttributes); + }); +}; + +const isTheSameTag = ( + a: SerializedCollection['tags'][number], + b: SerializedCollection['tags'][number], +): boolean => ( + a.label_id === b.label_id && + a.frame === b.frame && + a.group === b.group && + a.source === b.source && + isTheSameAttributes(a.attributes, b.attributes) +); + +const isTheSameInterval = ( + a: SerializedCollection['intervals'][number], + b: SerializedCollection['intervals'][number], +): boolean => ( + a.label_id === b.label_id && + a.start === b.start && + a.stop === b.stop && + a.group === b.group && + a.source === b.source && + isTheSameAttributes(a.attributes, b.attributes) +); + +const isTheSameShape = ( + a: Omit, + b: Omit, +): boolean => { + const isSame = a.label_id === b.label_id && + a.frame === b.frame && + a.group === b.group && + a.source === b.source && + a.occluded === b.occluded && + a.z_order === b.z_order && + a.rotation === b.rotation && + a.type === b.type && + isTheSameAttributes(a.attributes, b.attributes) && + isTheSamePoints(a.points, b.points); + + if ('elements' in a && Array.isArray(a.elements) && 'elements' in b && Array.isArray(b.elements)) { + return isSame && a.elements.length === b.elements.length && + a.elements.every((element, index) => isTheSameShape(element, b.elements[index])); + } + + return isSame; +}; + +type SerializedTrackLike = Omit & { + elements?: SerializedTrackLike[]; +}; +const isTheSameTrack = ( + a: SerializedTrackLike, + b: SerializedTrackLike, +): boolean => { + const isSame = a.label_id === b.label_id && + a.group === b.group && + a.source === b.source && + isTheSameAttributes(a.attributes, b.attributes) && + isTheSameTrackedShapes(a.shapes, b.shapes); + + if ('elements' in a && Array.isArray(a.elements) && 'elements' in b && Array.isArray(b.elements)) { + return isSame && a.elements.length === b.elements.length && + a.elements.every((element, index) => isTheSameTrack(element, b.elements[index])); + } + + return isSame; +}; + +const isTheSameObject = ( + type: typeof COLLECTION_KEYS[number], + left: CollectionObject, + right: CollectionObject, +): boolean => { + switch (type) { + case 'shapes': + return isTheSameShape( + left as SerializedCollection['shapes'][number], + right as SerializedCollection['shapes'][number], + ); + case 'tracks': + return isTheSameTrack( + left as SerializedCollection['tracks'][number], + right as SerializedCollection['tracks'][number], + ); + case 'tags': + return isTheSameTag( + left as SerializedCollection['tags'][number], + right as SerializedCollection['tags'][number], + ); + case 'intervals': + return isTheSameInterval( + left as SerializedCollection['intervals'][number], + right as SerializedCollection['intervals'][number], + ); + default: + throw new Error(`Unknown collection type: ${type}`); + } +}; + +function removeIDFromObject( + object: T, + property: 'id' | 'clientID', +): T { + delete object[property]; + if ('shapes' in object && Array.isArray(object.shapes)) { + for (const shape of object.shapes) { + delete shape[property]; + } + } + + if ('elements' in object && Array.isArray(object.elements)) { + object.elements = object.elements.map((element) => removeIDFromObject(element, property)); + } + + return object; +} + +export default class AnnotationsSaver { + private sessionType: 'task' | 'job'; + private id: number; + private collection: any; + private hash: string; + private initialObjects: { + shapes: Map, + tracks: Map, + tags: Map, + intervals: Map, + }; + + constructor(collection, session: Task | Job) { + this.sessionType = session instanceof Task ? 'task' : 'job'; + this.id = session.id; + this.collection = collection; + this.hash = this._getHash(); + this.initialObjects = { + shapes: new Map(), + tracks: new Map(), + tags: new Map(), + intervals: new Map(), + }; + + // We need use data from export instead of initialData + // Otherwise we have differ keys order and JSON comparison code works incorrectly + const exported = this.collection.export(); + for (const key of COLLECTION_KEYS) { + for (const object of exported[key]) { + this.initialObjects[key].set(object.id, object); + } + } + } + + _getHash(): string { + const exported = this.collection.export(); + return JSON.stringify(exported); + } + + async _request(data: SerializedCollection, action: 'put' | 'create' | 'update' | 'delete'): Promise { + const collection = await serverProxy.annotations.updateAnnotations(this.sessionType, this.id, data, action); + return collection; + } + + async _put(data: SerializedCollection): Promise { + const result = await this._request(data, 'put'); + return result; + } + + async _create(created: SerializedCollection): Promise { + const result = await this._request(created, 'create'); + return result; + } + + async _update(updated: SerializedCollection): Promise { + const result = await this._request(updated, 'update'); + return result; + } + + async _delete(deleted: SerializedCollection): Promise { + const result = await this._request(deleted, 'delete'); + return result; + } + + _split(exported: SerializedCollection): SplittedCollection { + const splitted: SplittedCollection = { + created: { + shapes: [], + tracks: [], + tags: [], + intervals: [], + }, + updated: { + shapes: [], + tracks: [], + tags: [], + intervals: [], + }, + deleted: { + shapes: [], + tracks: [], + tags: [], + intervals: [], + }, + }; + + // Find created and updated objects + for (const objectType of COLLECTION_KEYS) { + for (const object of exported[objectType]) { + if (Number.isInteger(object.id) && this.initialObjects[objectType].has(object.id)) { + const exportedHash = JSON.stringify(object, JSON_SERIALIZER_KEYS); + const initialHash = JSON.stringify( + this.initialObjects[objectType].get(object.id), JSON_SERIALIZER_KEYS, + ); + + if (exportedHash !== initialHash) { + splitted.updated[objectType].push(object as any); + } + } else { + removeIDFromObject(object, 'id'); + splitted.created[objectType].push(object as any); + } + } + } + + // Now find deleted objects + const exportedServerIds = { + shapes: new Set(exported.shapes.map((object) => +object.id)), + tracks: new Set(exported.tracks.map((object) => +object.id)), + tags: new Set(exported.tags.map((object) => +object.id)), + intervals: new Set(exported.intervals.map((object) => +object.id)), + }; + + for (const type of COLLECTION_KEYS) { + for (const id of this.initialObjects[type].keys()) { + if (!exportedServerIds[type].has(id)) { + const object = this.initialObjects[type].get(id); + splitted.deleted[type].push(object as any); + } + } + } + + return splitted; + } + + _updateSavedObjects(saved: SerializedCollection, indexes: ExtractedIDs): void { + const savedLength = COLLECTION_KEYS.reduce((acc, type) => acc + saved[type].length, 0); + const indexesLength = COLLECTION_KEYS.reduce((acc, type) => acc + indexes[type].length, 0); + if (indexesLength !== savedLength) { + throw new DataError( + 'Server returned different number of objects than client sent ' + + `(${savedLength} vs ${indexesLength}).`, + ); + } + + // Updated IDs of created objects + for (const type of Object.keys(indexes)) { + for (let i = 0; i < indexes[type].length; i++) { + const clientID = indexes[type][i]; + this.collection.objects[clientID].updateFromServerResponse(saved[type][i]); + } + } + } + + _extractClientIDs(exported: SerializedCollection): ExtractedIDs { + // Receive client IDs before saving + const indexes = { + tracks: exported.tracks.map((track) => track.clientID), + shapes: exported.shapes.map((shape) => shape.clientID), + tags: exported.tags.map((tag) => tag.clientID), + intervals: exported.intervals.map((interval) => interval.clientID), + }; + + // Remove them from the request body + for (const type of COLLECTION_KEYS) { + for (const object of exported[type]) { + removeIDFromObject(object, 'clientID'); + } + } + + return indexes; + } + + _updateInitialObjects(responseBody: SerializedCollection): void { + for (const type of COLLECTION_KEYS) { + for (const object of responseBody[type]) { + this.initialObjects[type].set(object.id, object as any); + } + } + } + + async save(onUpdateArg?: (message: string) => void): Promise { + const onUpdate = typeof onUpdateArg === 'function' ? onUpdateArg : (message) => { + console.log(message); + }; + + const exported = this.collection.export(); + const { flush } = this.collection; + if (flush) { + onUpdate('Collection is being saved on the server'); + // remove server IDs if there are any, annotations will be rewritten + const indexes = this._extractClientIDs(exported); + for (const type of COLLECTION_KEYS) { + for (const object of exported[type]) { + removeIDFromObject(object, 'id'); + } + } + + const savedData = await this._put(exported); + this.collection.flush = false; + + this._updateSavedObjects(savedData, indexes); + this.initialObjects = { + shapes: new Map(), + tracks: new Map(), + tags: new Map(), + intervals: new Map(), + }; + + for (const type of COLLECTION_KEYS) { + for (const object of savedData[type]) { + this.initialObjects[type].set(object.id, object as any); + } + } + } else { + // with using ASGI server it is possible to get 504 (RequestTimeout) + // from nginx proxy, when the request is still being processed by the server + // that is not good that client knows about the server details + // but we implemented a workaround here + + const findPair = ( + key: typeof COLLECTION_KEYS[number], + objectToSave: CollectionObject, + serverCollection: SerializedCollection, + ): CollectionObject | null => { + const serverObjects = serverCollection[key]; + const existingIDs = Array.from(this.initialObjects[key].keys()); + const { label_id: labelID } = objectToSave; + + // optimization to avoid stringifying each object in collection + const potentialObjects = serverObjects.filter( + (object) => { + const isPotential = object.label_id === labelID && !existingIDs.includes(object.id); + if (key === 'intervals') { + return isPotential && ['start', 'stop'].every((property) => object[property] === objectToSave[property]); + } + + if (key === 'shapes' || key === 'tags') { + return isPotential && ['frame'].every((property) => object[property] === objectToSave[property]); + } + + return isPotential; + }, + ); + + return potentialObjects.find((object) => isTheSameObject(key, objectToSave, object)) ?? null; + }; + + const retryIf504Status = async ( + error: unknown, + requestBody: SerializedCollection, + action: 'update' | 'delete' | 'create', + ): Promise => { + if (error instanceof ServerError && error.code === 504) { + setTimeout(() => { + // just for logging + throw new Error( + `Code 504 received from the server when ${action} objects, running workaround`, + ); + }); + + const RETRY_PERIOD = 30000; + let retryCount = 3; + while (retryCount) { + try { + await sleep(RETRY_PERIOD); + switch (action) { + case 'update': { + return await this._update(requestBody); + } + case 'delete': { + return await this._delete(requestBody); + } + case 'create': { + const serverCollection = await serverProxy.annotations + .getAnnotations(this.sessionType, this.id); + const foundPairs: SerializedCollection = { + shapes: [], + tracks: [], + tags: [], + intervals: [], + }; + + for (const type of COLLECTION_KEYS) { + for (const obj of requestBody[type]) { + const pair = findPair(type, obj, serverCollection); + if (pair === null) { + throw new Error('Pair not found this iteration'); + } + foundPairs[type].push(pair as any); + } + } + + return foundPairs; + } + default: + throw new Error('Unknown action'); + } + } catch { + retryCount--; + } + } + } + + throw error; + }; + + const { created, updated, deleted } = this._split(exported); + + if (COLLECTION_KEYS.some((type) => updated[type].length)) { + onUpdate('Updated objects are being saved on the server'); + const updatedIndexes = this._extractClientIDs(updated); + let updatedData = null; + try { + updatedData = await this._update(updated); + } catch (error: unknown) { + updatedData = await retryIf504Status(error, updated, 'update'); + } + + this._updateSavedObjects(updatedData, updatedIndexes); + for (const type of Object.keys(this.initialObjects)) { + for (const object of updatedData[type]) { + this.initialObjects[type].set(object.id, object as any); + } + } + } + + if (COLLECTION_KEYS.some((type) => deleted[type].length)) { + onUpdate('Deleted objects are being deleted from the server'); + this._extractClientIDs(deleted); + let deletedData = null; + try { + deletedData = await this._delete(deleted); + } catch (error: unknown) { + deletedData = await retryIf504Status(error, deleted, 'delete'); + } + + for (const type of Object.keys(this.initialObjects)) { + for (const object of deletedData[type]) { + this.initialObjects[type].delete(object.id); + } + } + } + + if (COLLECTION_KEYS.some((type) => created[type].length)) { + onUpdate('Created objects are being saved on the server'); + const createdIndexes = this._extractClientIDs(created); + let createdData = null; + try { + createdData = await this._create(created); + } catch (error: unknown) { + createdData = await retryIf504Status(error, created, 'create'); + } + + this._updateSavedObjects(createdData, createdIndexes); + for (const type of Object.keys(this.initialObjects)) { + for (const object of createdData[type]) { + this.initialObjects[type].set(object.id, object as any); + } + } + } + } + + this.hash = this._getHash(); + } + + hasUnsavedChanges(): boolean { + return this._getHash() !== this.hash; + } +} diff --git a/cvat-core/src/annotations.ts b/cvat-core/src/annotations.ts new file mode 100644 index 0000000..ee84013 --- /dev/null +++ b/cvat-core/src/annotations.ts @@ -0,0 +1,299 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { Storage } from './storage'; +import serverProxy from './server-proxy'; +import AnnotationsCollection from './annotations-collection'; +import AnnotationsSaver from './annotations-saver'; +import AnnotationsHistory from './annotations-history'; +import { checkObjectType } from './common'; +import Project from './project'; +import { Task, Job } from './session'; +import { ArgumentError } from './exceptions'; +import { getFramesMeta, getJobFramesMetaSync } from './frames'; +import { JobType } from './enums'; + +const jobCollectionCache = new WeakMap(); +const taskCollectionCache = new WeakMap(); + +// save history separately as not all history actions are related to annotations (e.g. delete, restore frame are not) +const jobHistoryCache = new WeakMap(); +const taskHistoryCache = new WeakMap(); + +function getCache(sessionType: 'task' | 'job'): { + collection: typeof jobCollectionCache; + history: typeof jobHistoryCache; +} { + if (sessionType === 'task') { + return { + collection: taskCollectionCache, + history: taskHistoryCache, + }; + } + + return { + collection: jobCollectionCache, + history: jobHistoryCache, + }; +} + +class InstanceNotInitializedError extends Error {} + +export function getCollection(session): AnnotationsCollection { + const sessionType = session instanceof Task ? 'task' : 'job'; + const { collection } = getCache(sessionType); + + if (collection.has(session)) { + return collection.get(session).collection; + } + + throw new InstanceNotInitializedError( + 'Session has not been initialized yet. Call annotations.get() or annotations.clear({ reload: true }) before', + ); +} + +export function getSaver(session): AnnotationsSaver { + const sessionType = session instanceof Task ? 'task' : 'job'; + const { collection } = getCache(sessionType); + + if (collection.has(session)) { + return collection.get(session).saver; + } + + throw new InstanceNotInitializedError( + 'Session has not been initialized yet. Call annotations.get() or annotations.clear({ reload: true }) before', + ); +} + +export function getHistory(session): AnnotationsHistory { + const sessionType = session instanceof Task ? 'task' : 'job'; + const { history } = getCache(sessionType); + + if (history.has(session)) { + return history.get(session); + } + + const initiatedHistory = new AnnotationsHistory(); + history.set(session, initiatedHistory); + return initiatedHistory; +} + +async function getAnnotationsFromServer(session: Job | Task): Promise { + const sessionType = session instanceof Task ? 'task' : 'job'; + const cache = getCache(sessionType); + + if (!cache.collection.has(session)) { + const serializedAnnotations = await serverProxy.annotations.getAnnotations(sessionType, session.id); + + // Get meta information about frames + const frameMeta = await getFramesMeta(sessionType, session.id); + const frameNumbers = frameMeta.getSegmentFrameNumbers(session instanceof Job ? session.startFrame : 0); + + const history = cache.history.has(session) ? cache.history.get(session) : new AnnotationsHistory(); + const collection = new AnnotationsCollection({ + // Job collections use real job metadata. Task collections use constant defaults because + // task-level annotations are not tied to a particular job type or consensus replica set. + jobType: session instanceof Job ? session.type : JobType.ANNOTATION, + stopFrame: session instanceof Job ? session.stopFrame : session.size - 1, + labels: session.labels, + dimension: session.dimension, + replicasCount: session instanceof Job ? session.replicasCount : undefined, + framesInfo: { + isFrameDeleted: session instanceof Job ? + (frame: number) => !!getJobFramesMetaSync(session.id).deletedFrames[frame] : + (frame: number) => !!frameMeta.deletedFrames[frame], + ...frameMeta.frames.reduce((acc, frameInfo, idx) => { + // keep only static information + acc[frameNumbers[idx]] = { + width: frameInfo.width, + height: frameInfo.height, + }; + return acc; + }, {}), + }, + history, + }); + + collection.import(serializedAnnotations); + const saver = new AnnotationsSaver(collection, session); + cache.collection.set(session, { collection, saver }); + cache.history.set(session, history); + } +} + +export function clearCache(session): void { + const sessionType = session instanceof Task ? 'task' : 'job'; + const cache = getCache(sessionType); + + if (cache.collection.has(session)) { + cache.collection.delete(session); + } + + if (cache.history.has(session)) { + cache.history.delete(session); + } +} + +export async function getAnnotations( + session: Job | Task, + frame: number, + allTracks: boolean, + filters: object[], +): Promise> { + try { + return getCollection(session).get(frame, allTracks, filters); + } catch (error) { + if (error instanceof InstanceNotInitializedError) { + await getAnnotationsFromServer(session); + return getCollection(session).get(frame, allTracks, filters); + } + throw error; + } +} + +export async function getAllIntervals( + session: Job | Task, + filters: object[], +): Promise> { + try { + return getCollection(session).getAllIntervals(filters); + } catch (error) { + if (error instanceof InstanceNotInitializedError) { + await getAnnotationsFromServer(session); + return getCollection(session).getAllIntervals(filters); + } + throw error; + } +} + +export async function clearAnnotations( + session: Task | Job, + options: Parameters[0], +): Promise { + const sessionType = session instanceof Task ? 'task' : 'job'; + const cache = getCache(sessionType); + + if (Object.hasOwn(options ?? {}, 'reload')) { + const { reload } = options; + checkObjectType('reload', reload, 'boolean'); + + if (reload) { + cache.collection.delete(session); + // delete history as it may relate to objects from collection we deleted above + cache.history.delete(session); + return getAnnotationsFromServer(session); + } + } + + return getCollection(session).clear(options); +} + +export async function exportDataset( + instance, + format: string, + saveImages: boolean, + useDefaultSettings: boolean, + targetStorage: Storage, + name?: string, +): Promise { + if (!(instance instanceof Task || instance instanceof Project || instance instanceof Job)) { + throw new ArgumentError('A dataset can only be created from a job, task or project'); + } + + let result = null; + if (instance instanceof Task) { + result = await serverProxy.tasks + .exportDataset(instance.id, format, saveImages, useDefaultSettings, targetStorage, name); + } else if (instance instanceof Job) { + result = await serverProxy.jobs + .exportDataset(instance.id, format, saveImages, useDefaultSettings, targetStorage, name); + } else { + result = await serverProxy.projects + .exportDataset(instance.id, format, saveImages, useDefaultSettings, targetStorage, name); + } + + return result; +} + +export function importDataset( + instance: any, + format: string, + useDefaultSettings: boolean, + sourceStorage: Storage, + file: File | string, + options: { + convMaskToPoly?: boolean, + importMode?: 'replace' | 'append', + updateStatusCallback?: (message: string, progress: number) => void, + } = {}, +): Promise { + const updateStatusCallback = options.updateStatusCallback || (() => {}); + const convMaskToPoly = options.convMaskToPoly ?? true; + const importMode = options.importMode ?? 'replace'; + + if (!(instance instanceof Project || instance instanceof Task || instance instanceof Job)) { + throw new ArgumentError('Instance must be a Project || Task || Job instance'); + } + if (!(typeof updateStatusCallback === 'function')) { + throw new ArgumentError('Callback must be a function'); + } + if (!(typeof convMaskToPoly === 'boolean')) { + throw new ArgumentError('Option "convMaskToPoly" must be a boolean'); + } + if (importMode !== 'replace' && importMode !== 'append') { + throw new ArgumentError('Option "importMode" must be "replace" or "append"'); + } + const allowedFileExtensions = [ + '.zip', '.xml', '.json', '.tsv', + ]; + const allowedFileExtensionsList = allowedFileExtensions.join(', '); + if (typeof file === 'string' && !(allowedFileExtensions.some((ext) => file.toLowerCase().endsWith(ext)))) { + throw new ArgumentError( + `File must be file instance with one of the following extensions: ${allowedFileExtensionsList}. ` + + `Found ${file}`, + ); + } + const allowedMimeTypes = [ + 'application/zip', 'application/x-zip-compressed', + 'application/xml', 'text/xml', + 'application/json', 'text/tab-separated-values', + ]; + if (file instanceof File && !(allowedMimeTypes.includes(file.type))) { + throw new ArgumentError( + `File must be file instance with one of the following extensions: ${allowedFileExtensionsList}. ` + + `Found ${file.type}`, + ); + } + + if (instance instanceof Project) { + return serverProxy.projects + .importDataset( + instance.id, + format, + useDefaultSettings, + sourceStorage, + file, + { + updateStatusCallback, + convMaskToPoly, + }, + ); + } + + const instanceType = instance instanceof Task ? 'task' : 'job'; + return serverProxy.annotations + .uploadAnnotations( + instanceType, + instance.id, + format, + useDefaultSettings, + sourceStorage, + file, + { + convMaskToPoly, + importMode, + }, + ); +} diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts new file mode 100644 index 0000000..0c77446 --- /dev/null +++ b/cvat-core/src/api-implementation.ts @@ -0,0 +1,585 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { omit } from 'lodash'; +import config from './config'; + +import PluginRegistry from './plugins'; +import serverProxy from './server-proxy'; +import lambdaManager from './lambda-manager'; +import requestsManager from './requests-manager'; +import { + isBoolean, + isInteger, + isString, + isPageSize, + checkFilter, + checkExclusiveFields, + checkObjectType, + filterFieldsToSnakeCase, + fieldsToSnakeCase, +} from './common'; + +import User from './user'; +import AnnotationFormats from './annotation-formats'; +import ApiToken from './api-token'; +import { Task, Job } from './session'; +import Project from './project'; +import CloudStorage from './cloud-storage'; +import Organization, { Invitation } from './organization'; +import Webhook from './webhook'; +import { ArgumentError } from './exceptions'; +import { + AnalyticsEventsFilter, QualityConflictsFilter, + SerializedAsset, ConsensusSettingsFilter, SerializedOrganization, +} from './server-response-types'; +import QualityReport from './quality-report'; +import AboutData from './about'; +import QualityConflict, { ConflictSeverity } from './quality-conflict'; +import QualitySettings from './quality-settings'; +import { getFramesMeta } from './frames'; +import ConsensusSettings from './consensus-settings'; +import { + callAction, listActions, registerAction, unregisterAction, runAction, +} from './annotations-actions/annotations-actions'; +import { convertDescriptions, getServerAPISchema } from './server-schema'; +import { JobType } from './enums'; +import { PaginatedResource } from './core-types'; +import CVATCore from '.'; + +// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type +function implementationMixin(func: Function, implementation: Function): void { + Object.assign(func, { implementation }); +} + +export default function implementAPI(cvat: CVATCore): CVATCore { + implementationMixin(cvat.plugins.list, PluginRegistry.list); + implementationMixin(cvat.plugins.register, PluginRegistry.register.bind(cvat)); + implementationMixin(cvat.actions.list, listActions); + implementationMixin(cvat.actions.register, registerAction); + implementationMixin(cvat.actions.unregister, unregisterAction); + implementationMixin(cvat.actions.run, runAction); + implementationMixin(cvat.actions.call, callAction); + + implementationMixin(cvat.lambda.list, lambdaManager.list.bind(lambdaManager)); + implementationMixin(cvat.lambda.run, lambdaManager.run.bind(lambdaManager)); + implementationMixin(cvat.lambda.call, lambdaManager.call.bind(lambdaManager)); + implementationMixin(cvat.lambda.cancel, lambdaManager.cancel.bind(lambdaManager)); + implementationMixin(cvat.lambda.listen, lambdaManager.listen.bind(lambdaManager)); + implementationMixin(cvat.lambda.requests, lambdaManager.requests.bind(lambdaManager)); + + implementationMixin(cvat.requests.list, requestsManager.list.bind(requestsManager)); + implementationMixin(cvat.requests.listen, requestsManager.listen.bind(requestsManager)); + implementationMixin(cvat.requests.cancel, requestsManager.cancel.bind(requestsManager)); + + implementationMixin(cvat.server.about, async () => { + const result = await serverProxy.server.about(); + return new AboutData(result); + }); + implementationMixin(cvat.server.share, async ( + ...args: Parameters + ) => { + const result = await serverProxy.server.share(...args); + return result.map((item) => ({ ...omit(item, 'mime_type'), mimeType: item.mime_type })); + }); + implementationMixin(cvat.server.formats, async () => { + const result = await serverProxy.server.formats(); + return new AnnotationFormats(result); + }); + implementationMixin(cvat.server.userAgreements, async () => { + const result = await serverProxy.server.userAgreements(); + return result; + }); + implementationMixin(cvat.server.register, async ( + ...args: Parameters + ) => { + const result = await serverProxy.server.register(...args); + return result; + }); + implementationMixin(cvat.server.login, async ( + ...args: Parameters + ) => { + await serverProxy.server.login(...args); + }); + implementationMixin(cvat.server.logout, async () => { + await serverProxy.server.logout(); + }); + implementationMixin(cvat.server.changePassword, async ( + ...args: Parameters + ) => { + await serverProxy.server.changePassword(...args); + }); + implementationMixin(cvat.server.requestPasswordReset, async ( + ...args: Parameters + ) => { + await serverProxy.server.requestPasswordReset(...args); + }); + implementationMixin(cvat.server.resetPassword, async ( + ...args: Parameters + ) => { + await serverProxy.server.resetPassword(...args); + }); + implementationMixin(cvat.server.authenticated, async () => { + const result = await serverProxy.server.authenticated(); + return result; + }); + implementationMixin(cvat.server.healthCheck, async ( + ...args: Parameters + ) => { + const result = await serverProxy.server.healthCheck(...args); + return result; + }); + implementationMixin(cvat.server.request, async ( + ...args: Parameters + ) => { + const result = await serverProxy.server.request(...args); + return result; + }); + implementationMixin(cvat.server.installedApps, async () => { + const result = await serverProxy.server.installedApps(); + return result; + }); + + implementationMixin(cvat.server.apiSchema, async () => { + const result = await getServerAPISchema(); + return result; + }); + + implementationMixin(cvat.assets.create, async ( + ...args: Parameters + ): Promise => { + const [file, guideId] = args; + if (!(file instanceof File)) { + throw new ArgumentError('Assets expect a file'); + } + + const result = await serverProxy.assets.create(file, guideId); + return result; + }); + + implementationMixin(cvat.users.get, async (filter) => { + checkFilter(filter, { + id: isInteger, + is_active: isBoolean, + self: isBoolean, + search: isString, + limit: isInteger, + }); + + let users = null; + if ('self' in filter && filter.self) { + users = await serverProxy.users.self(); + users = [users]; + } else { + const searchParams = {}; + for (const key in filter) { + if (filter[key] && key !== 'self') { + searchParams[key] = filter[key]; + } + } + users = await serverProxy.users.get(searchParams); + } + + users = users.map((user) => new User(user)); + return users; + }); + + implementationMixin(cvat.apiTokens.get, async (filter) => { + checkFilter(filter, { + id: isInteger, + page: isInteger, + page_size: isInteger, + filter: isString, + sort: isString, + search: isString, + name: isString, + owner: isInteger, + read_only: isBoolean, + created_date: isString, + updated_date: isString, + expiry_date: isString, + last_used_date: isString, + }); + + const result = await serverProxy.apiTokens.get(filter); + const tokens = result.map((tokenData) => new ApiToken(tokenData)); + Object.assign(tokens, { count: result.count }); + return tokens as PaginatedResource; + }); + + implementationMixin(cvat.jobs.get, async ( + query: Parameters[0], + aggregate: Parameters[1], + ): ReturnType => { + checkFilter(query, { + page: isInteger, + pageSize: isInteger, + filter: isString, + sort: isString, + search: isString, + jobID: isInteger, + taskID: isInteger, + projectID: isInteger, + type: isString, + }); + + checkExclusiveFields(query, ['jobID', 'filter', 'search'], ['page', 'pageSize', 'sort']); + if ('jobID' in query) { + const results = await serverProxy.jobs.get({ id: query.jobID }); + const [job] = results; + if (job) { + // When request job by ID we also need to add labels to work with them + const labels = await serverProxy.labels.get({ job_id: job.id }); + return Object.assign([new Job({ ...job, labels: labels.results })], { count: 1 }); + } + + return Object.assign([], { count: 0 }); + } + + const searchParams = fieldsToSnakeCase({ ...query }); + + const jobsData = await serverProxy.jobs.get(searchParams, aggregate); + if (query.type === JobType.GROUND_TRUTH && jobsData.count === 1) { + const labels = await serverProxy.labels.get({ job_id: jobsData[0].id }); + return Object.assign([ + new Job({ + ...omit(jobsData[0], 'labels'), + labels: labels.results, + }), + ], { count: 1 }); + } + + const jobs = jobsData.map((jobData) => new Job(omit(jobData, 'labels'))); + return Object.assign(jobs, { count: jobsData.count }); + }); + + implementationMixin(cvat.tasks.get, async ( + filter: Parameters[0], + aggregate: Parameters[1], + ): ReturnType => { + checkFilter(filter, { + page: isInteger, + pageSize: isInteger, + projectId: isInteger, + id: isInteger, + sort: isString, + search: isString, + filter: isString, + ordering: isString, + }); + + checkExclusiveFields(filter, ['id'], ['page', 'pageSize']); + const searchParams = filterFieldsToSnakeCase(filter, ['projectId']); + + const tasksData = await serverProxy.tasks.get(searchParams, aggregate); + const tasks = await Promise.all(tasksData.map(async (taskItem) => { + if ('id' in filter) { + // When request task by ID we also need to add labels and jobs to work with them + const labels = await serverProxy.labels.get({ task_id: taskItem.id }); + const jobs = await serverProxy.jobs.get({ task_id: taskItem.id }, true); + return new Task({ + ...omit(taskItem, ['jobs', 'labels']), + progress: taskItem.jobs, + jobs, + labels: labels.results, + }); + } + + return new Task({ + ...omit(taskItem, ['jobs', 'labels']), + progress: taskItem.jobs, + }); + })); + + Object.assign(tasks, { count: tasksData.count }); + return tasks as PaginatedResource; + }); + + implementationMixin(cvat.projects.get, async ( + filter: Parameters[0], + ): ReturnType => { + checkFilter(filter, { + id: isInteger, + page: isInteger, + pageSize: isInteger, + search: isString, + sort: isString, + filter: isString, + }); + + checkExclusiveFields(filter, ['id'], ['page', 'pageSize']); + const searchParams = fieldsToSnakeCase(filter); + + const projectsData = await serverProxy.projects.get(searchParams); + const projects = await Promise.all(projectsData.map(async (projectItem) => { + if ('id' in filter) { + // When request a project by ID we also need to add labels to work with them + const labels = await serverProxy.labels.get({ project_id: projectItem.id }); + return new Project({ ...projectItem, labels: labels.results }); + } + + return new Project({ ...projectItem }); + })); + + return Object.assign(projects, { count: projectsData.count }); + }); + + implementationMixin(cvat.projects.searchNames, + async (search, limit) => serverProxy.projects.searchNames(search, limit)); + + implementationMixin(cvat.cloudStorages.get, async (filter) => { + checkFilter(filter, { + page: isInteger, + pageSize: isInteger, + filter: isString, + sort: isString, + id: isInteger, + search: isString, + }); + + checkExclusiveFields(filter, ['id', 'search'], ['page', 'pageSize']); + const searchParams = fieldsToSnakeCase(filter); + + const cloudStoragesData = await serverProxy.cloudStorages.get(searchParams); + const cloudStorages = cloudStoragesData.map((cloudStorage) => new CloudStorage(cloudStorage)); + Object.assign(cloudStorages, { count: cloudStoragesData.count }); + return cloudStorages; + }); + + implementationMixin(cvat.organizations.get, async (filter) => { + checkFilter(filter, { + search: isString, + filter: isString, + page: isInteger, + page_size: isInteger, + sort: isString, + }); + + const organizationsPage = await serverProxy.organizations.get(filter); + const results = organizationsPage.results.map((org_) => new Organization(org_)); + Object.assign(results, { + count: organizationsPage.count, + next: organizationsPage.next, + }); + + return results as PaginatedResource; + }); + implementationMixin(cvat.organizations.activate, (organization) => { + checkObjectType('organization', organization, null, { cls: Organization, name: 'Organization' }); + config.organization = { + organizationID: organization.id, + organizationSlug: organization.slug, + }; + }); + implementationMixin(cvat.organizations.deactivate, async () => { + config.organization = { + organizationID: null, + organizationSlug: null, + }; + }); + implementationMixin( + cvat.organizations.acceptInvitation, + serverProxy.organizations.acceptInvitation, + ); + implementationMixin( + cvat.organizations.declineInvitation, + serverProxy.organizations.declineInvitation, + ); + implementationMixin(cvat.organizations.invitations, (async (filter) => { + checkFilter(filter, { + page: isInteger, + pageSize: isInteger, + filter: isString, + }); + checkExclusiveFields(filter, ['filter'], ['page', 'pageSize']); + + const invitationsData = await serverProxy.organizations.invitations(filter); + const invitations = invitationsData.results.map((invitationData) => new Invitation({ ...invitationData })); + return Object.assign(invitations, { count: invitationsData.count }); + }) as typeof cvat.organizations.invitations); + + implementationMixin(cvat.webhooks.get, async (filter) => { + checkFilter(filter, { + page: isInteger, + pageSize: isInteger, + id: isInteger, + projectId: isInteger, + filter: isString, + search: isString, + sort: isString, + }); + + checkExclusiveFields(filter, ['id', 'projectId'], ['page', 'pageSize']); + + const searchParams = filterFieldsToSnakeCase(filter, ['projectId']); + + const webhooksData = await serverProxy.webhooks.get(searchParams); + const webhooks = webhooksData.map((webhookData) => new Webhook(webhookData)); + Object.assign(webhooks, { count: webhooksData.count }); + return webhooks; + }); + + implementationMixin(cvat.consensus.settings.get, async (filter: ConsensusSettingsFilter) => { + checkFilter(filter, { + taskID: isInteger, + }); + + const params = fieldsToSnakeCase(filter); + + const settings = await serverProxy.consensus.settings.get(params); + const schema = await getServerAPISchema(); + const descriptions = convertDescriptions(schema.components.schemas.ConsensusSettings.properties); + + return new ConsensusSettings({ ...settings, descriptions }); + }); + + implementationMixin(cvat.analytics.quality.reports, async ( + filter: Parameters[0], + aggregate?: Parameters[1], + ) => { + checkFilter(filter, { + page: isInteger, + pageSize: isPageSize, + parentID: isInteger, + projectID: isInteger, + taskID: isInteger, + jobID: isInteger, + target: isString, + filter: isString, + search: isString, + sort: isString, + }); + + const params = fieldsToSnakeCase({ ...filter, sort: '-created_date' }); + + const reportsData = await serverProxy.analytics.quality.reports(params, aggregate); + const reports = Object.assign( + reportsData.map((report) => new QualityReport({ ...report })), + { count: reportsData.count }, + ); + return reports; + }); + implementationMixin(cvat.analytics.quality.conflicts, async (filter: QualityConflictsFilter) => { + checkFilter(filter, { + reportID: isInteger, + }); + + const params = fieldsToSnakeCase(filter); + + const conflictsData = await serverProxy.analytics.quality.conflicts(params); + const conflicts = conflictsData.map((conflict) => new QualityConflict({ ...conflict })); + const frames = Array.from(new Set(conflicts.map((conflict) => conflict.frame))) + .sort((a, b) => a - b); + + // each QualityConflict may have several AnnotationConflicts bound + // at the same time, many quality conflicts may refer + // to the same labeled object (e.g. mismatch label, low overlap) + // the code below unites quality conflicts bound to the same object into one QualityConflict object + const mergedConflicts: QualityConflict[] = []; + + for (const frame of frames) { + const frameConflicts = conflicts.filter((conflict) => conflict.frame === frame); + const conflictsByObject: Record = {}; + + frameConflicts.forEach((qualityConflict: QualityConflict) => { + const { type, serverID } = qualityConflict.annotationConflicts[0]; + const firstObjID = `${type}_${serverID}`; + conflictsByObject[firstObjID] = conflictsByObject[firstObjID] || []; + conflictsByObject[firstObjID].push(qualityConflict); + }); + + for (const objectConflicts of Object.values(conflictsByObject)) { + if (objectConflicts.length === 1) { + // only one quality conflict refers to the object on current frame + mergedConflicts.push(objectConflicts[0]); + } else { + const mainObjectConflict = objectConflicts + .find((conflict) => conflict.severity === ConflictSeverity.ERROR) || objectConflicts[0]; + const descriptionList: string[] = [mainObjectConflict.description]; + + for (const objectConflict of objectConflicts) { + if (objectConflict !== mainObjectConflict) { + descriptionList.push(objectConflict.description); + + for (const annotationConflict of objectConflict.annotationConflicts) { + if (!mainObjectConflict.annotationConflicts.find((_annotationConflict) => ( + _annotationConflict.serverID === annotationConflict.serverID && + _annotationConflict.type === annotationConflict.type)) + ) { + mainObjectConflict.annotationConflicts.push(annotationConflict); + } + } + } + } + + // decorate the original conflict to avoid changing it + const description = descriptionList.join(', '); + const visibleConflict = new Proxy(mainObjectConflict, { + get(target, prop) { + if (prop === 'description') { + return description; + } + + // By default, it looks like Reflect.get(target, prop, receiver) + // which has a different value of `this`. It doesn't allow to + // work with methods / properties that use private members. + const val = Reflect.get(target, prop); + return typeof val === 'function' ? (...args: any[]) => val.apply(target, args) : val; + }, + }); + + mergedConflicts.push(visibleConflict); + } + } + } + + return mergedConflicts; + }); + implementationMixin( + cvat.analytics.quality.settings.get, async ( + filter: Parameters[0], + aggregate?: Parameters[1], + ) => { + checkFilter(filter, { + taskID: isInteger, + projectID: isInteger, + parentType: isString, + }); + + const params = fieldsToSnakeCase(filter); + + const settingsList = await serverProxy.analytics.quality.settings.get(params, aggregate); + const schema = await getServerAPISchema(); + const descriptions = convertDescriptions(schema.components.schemas.QualitySettings.properties); + + const settings = settingsList.map((setting) => new QualitySettings({ ...setting, descriptions })); + return settings; + }); + implementationMixin(cvat.analytics.events.export, async ( + filter: AnalyticsEventsFilter, + ): ReturnType => { + checkFilter(filter, { + orgId: isInteger, + userId: isInteger, + jobId: isInteger, + taskId: isInteger, + projectId: isInteger, + from: isString, + to: isString, + filename: isString, + }); + + checkExclusiveFields(filter, ['jobId', 'taskId', 'projectId'], ['from', 'to']); + + const params = fieldsToSnakeCase(filter); + return serverProxy.events.export(params); + }); + implementationMixin(cvat.frames.getMeta, async (type: 'job' | 'task', id: number) => { + const result = await getFramesMeta(type, id); + return result; + }); + + return cvat; +} diff --git a/cvat-core/src/api-token.ts b/cvat-core/src/api-token.ts new file mode 100644 index 0000000..838c781 --- /dev/null +++ b/cvat-core/src/api-token.ts @@ -0,0 +1,143 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import PluginRegistry from './plugins'; +import serverProxy from './server-proxy'; +import User from './user'; +import { SerializedApiToken } from './server-response-types'; +import { APIApiTokenModifiableFields, ApiTokenModifiableFields } from './server-request-types'; +import { fieldsToSnakeCase } from './common'; + +export default class ApiToken { + #id?: number; + #name: string; + #createdDate?: string; + #updatedDate?: string; + #expiryDate: string | null; + #lastUsedDate?: string | null; + #readOnly: boolean; + #owner?: User; + #value?: string; + + constructor(initialData: Partial) { + this.#id = initialData.id; + this.#name = initialData.name; + this.#createdDate = initialData.created_date; + this.#updatedDate = initialData.updated_date; + this.#expiryDate = initialData.expiry_date; + this.#lastUsedDate = initialData.last_used_date; + this.#readOnly = initialData.read_only; + this.#owner = initialData.owner ? new User(initialData.owner) : undefined; + this.#value = initialData.value; + } + + get id(): number { + return this.#id; + } + + get name(): string { + return this.#name; + } + + get createdDate(): string { + return this.#createdDate; + } + + get updatedDate(): string { + return this.#updatedDate; + } + + get expiryDate(): string | null { + return this.#expiryDate; + } + + get lastUsedDate(): string | null { + return this.#lastUsedDate; + } + + get readOnly(): boolean { + return this.#readOnly; + } + + get owner(): User { + return this.#owner; + } + + get value(): string | undefined { + return this.#value; + } + + public async save(fields: ApiTokenModifiableFields = {}): Promise { + const result = await PluginRegistry.apiWrapper.call(this, ApiToken.prototype.save, fields); + return result; + } + + public async revoke(): Promise { + const result = await PluginRegistry.apiWrapper.call(this, ApiToken.prototype.revoke); + return result; + } + + public toJSON(): SerializedApiToken { + const result: SerializedApiToken = { + name: this.#name, + read_only: this.#readOnly, + expiry_date: this.#expiryDate, + }; + + if (Number.isInteger(this.#id)) { + result.id = this.#id; + } + + if (this.#createdDate) { + result.created_date = this.#createdDate; + } + + if (this.#updatedDate) { + result.updated_date = this.#updatedDate; + } + + if (this.#expiryDate) { + result.expiry_date = this.#expiryDate; + } + + if (this.#lastUsedDate) { + result.last_used_date = this.#lastUsedDate; + } + + if (this.#value !== undefined) { + result.value = this.#value; + } + + return result; + } +} + +Object.defineProperties(ApiToken.prototype.save, { + implementation: { + writable: false, + enumerable: false, + value: async function implementation(fields: Parameters[0]) { + const data: APIApiTokenModifiableFields = fieldsToSnakeCase(fields); + + if (Number.isInteger(this.id)) { + const result = await serverProxy.apiTokens.update(this.id, data); + return new ApiToken(result); + } + + const result = await serverProxy.apiTokens.create(this.toJSON()); + return new ApiToken(result); + }, + }, +}); + +Object.defineProperties(ApiToken.prototype.revoke, { + implementation: { + writable: false, + enumerable: false, + value: async function implementation() { + await serverProxy.apiTokens.revoke(this.id); + return this; + }, + }, +}); diff --git a/cvat-core/src/api.ts b/cvat-core/src/api.ts new file mode 100644 index 0000000..73f61cd --- /dev/null +++ b/cvat-core/src/api.ts @@ -0,0 +1,523 @@ +// Copyright (C) 2019-2022 Intel Corporation +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import PluginRegistry from './plugins'; +import logger from './logger'; +import { Event } from './event'; +import ObjectState from './object-state'; +import Statistics from './statistics'; +import Comment from './comment'; +import Issue from './issue'; +import { Job, Task } from './session'; +import { implementJob, implementTask } from './session-implementation'; +import Project from './project'; +import implementProject from './project-implementation'; +import { Attribute, Label } from './labels'; +import MLModel from './ml-model'; +import { FrameData, FramesMetaData } from './frames'; +import CloudStorage from './cloud-storage'; +import Organization from './organization'; +import Webhook from './webhook'; +import AnnotationGuide from './guide'; +import { BaseAction } from './annotations-actions/base-action'; +import { BaseCollectionAction } from './annotations-actions/base-collection-action'; +import { BaseShapesAction } from './annotations-actions/base-shapes-action'; +import QualityReport from './quality-report'; +import QualityConflict from './quality-conflict'; +import QualitySettings from './quality-settings'; +import ApiToken from './api-token'; +import { JobValidationLayout, TaskValidationLayout } from './validation-layout'; +import { Request } from './request'; +import { createOpenCVInterface } from './opencv/opencv-interface'; + +import * as enums from './enums'; + +import { + Exception, ArgumentError, DataError, ScriptingError, ServerError, +} from './exceptions'; + +import { getVisibleSkeletonElements, propagateShapes, validateAttributeValue } from './object-utils'; +import { mask2Rle, rle2Mask } from './rle-utils'; +import User from './user'; +import config from './config'; + +import implementAPI from './api-implementation'; +import CVATCore from '.'; + +function build(): CVATCore { + const cvat: CVATCore = { + server: { + async about() { + const result = await PluginRegistry.apiWrapper(cvat.server.about); + return result; + }, + async share(directory = '/', searchPrefix?: string) { + const result = await PluginRegistry.apiWrapper(cvat.server.share, directory, searchPrefix); + return result; + }, + async formats() { + const result = await PluginRegistry.apiWrapper(cvat.server.formats); + return result; + }, + async userAgreements() { + const result = await PluginRegistry.apiWrapper(cvat.server.userAgreements); + return result; + }, + async register(username, firstName, lastName, email, password, userConfirmations) { + const result = await PluginRegistry.apiWrapper( + cvat.server.register, + username, + firstName, + lastName, + email, + password, + userConfirmations, + ); + return result; + }, + async login(username, password) { + const result = await PluginRegistry.apiWrapper(cvat.server.login, username, password); + return result; + }, + async logout() { + const result = await PluginRegistry.apiWrapper(cvat.server.logout); + return result; + }, + async changePassword(oldPassword, newPassword1, newPassword2) { + const result = await PluginRegistry.apiWrapper( + cvat.server.changePassword, + oldPassword, + newPassword1, + newPassword2, + ); + return result; + }, + async requestPasswordReset(email) { + const result = await PluginRegistry.apiWrapper(cvat.server.requestPasswordReset, email); + return result; + }, + async resetPassword(newPassword1, newPassword2, uid, token) { + const result = await PluginRegistry.apiWrapper( + cvat.server.resetPassword, + newPassword1, + newPassword2, + uid, + token, + ); + return result; + }, + async authenticated() { + const result = await PluginRegistry.apiWrapper(cvat.server.authenticated); + return result; + }, + async healthCheck(maxRetries = 1, checkPeriod = 3000, requestTimeout = 5000, progressCallback = undefined) { + const result = await PluginRegistry.apiWrapper( + cvat.server.healthCheck, + maxRetries, + checkPeriod, + requestTimeout, + progressCallback, + ); + return result; + }, + async request(url, data, requestConfig) { + const result = await PluginRegistry.apiWrapper(cvat.server.request, url, data, requestConfig); + return result; + }, + async installedApps() { + const result = await PluginRegistry.apiWrapper(cvat.server.installedApps); + return result; + }, + async apiSchema() { + const result = await PluginRegistry.apiWrapper(cvat.server.apiSchema); + return result; + }, + }, + projects: { + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.projects.get, filter); + return result; + }, + async searchNames(search = '', limit = 10) { + const result = await PluginRegistry.apiWrapper(cvat.projects.searchNames, search, limit); + return result; + }, + }, + tasks: { + async get(filter = {}, aggregate = false) { + const result = await PluginRegistry.apiWrapper(cvat.tasks.get, filter, aggregate); + return result; + }, + }, + assets: { + async create(file: File, guideId: number) { + const result = await PluginRegistry.apiWrapper(cvat.assets.create, file, guideId); + return result; + }, + }, + jobs: { + async get(filter = {}, aggregate = false) { + const result = await PluginRegistry.apiWrapper(cvat.jobs.get, filter, aggregate); + return result; + }, + }, + frames: { + async getMeta(type, id) { + const result = await PluginRegistry.apiWrapper(cvat.frames.getMeta, type, id); + return result; + }, + }, + users: { + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.users.get, filter); + return result; + }, + }, + apiTokens: { + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.apiTokens.get, filter); + return result; + }, + }, + plugins: { + async list() { + const result = await PluginRegistry.apiWrapper(cvat.plugins.list); + return result; + }, + async register(plugin) { + const result = await PluginRegistry.apiWrapper(cvat.plugins.register, plugin); + return result; + }, + }, + actions: { + async list() { + const result = await PluginRegistry.apiWrapper(cvat.actions.list); + return result; + }, + async register(action: BaseAction) { + const result = await PluginRegistry.apiWrapper(cvat.actions.register, action); + return result; + }, + async unregister(action: BaseAction) { + const result = await PluginRegistry.apiWrapper(cvat.actions.unregister, action); + return result; + }, + async run( + instance: Job | Task, + actions: BaseAction, + actionsParameters: Record, + frameFrom: number, + frameTo: number, + filters: object[], + onProgress: ( + message: string, + progress: number, + ) => void, + cancelled: () => boolean, + ) { + const result = await PluginRegistry.apiWrapper( + cvat.actions.run, + instance, + actions, + actionsParameters, + frameFrom, + frameTo, + filters, + onProgress, + cancelled, + ); + return result; + }, + async call( + instance: Job | Task, + actions: BaseAction, + actionsParameters: Record, + frame: number, + states: ObjectState[], + onProgress: ( + message: string, + progress: number, + ) => void, + cancelled: () => boolean, + ) { + const result = await PluginRegistry.apiWrapper( + cvat.actions.call, + instance, + actions, + actionsParameters, + frame, + states, + onProgress, + cancelled, + ); + return result; + }, + }, + lambda: { + async list() { + const result = await PluginRegistry.apiWrapper(cvat.lambda.list); + return result; + }, + async run(task, model, args) { + const result = await PluginRegistry.apiWrapper(cvat.lambda.run, task, model, args); + return result; + }, + async call(task, model, args) { + const result = await PluginRegistry.apiWrapper(cvat.lambda.call, task, model, args); + return result; + }, + async cancel(requestID) { + const result = await PluginRegistry.apiWrapper(cvat.lambda.cancel, requestID); + return result; + }, + async listen(requestID, onChange) { + const result = await PluginRegistry.apiWrapper(cvat.lambda.listen, requestID, onChange); + return result; + }, + async requests() { + const result = await PluginRegistry.apiWrapper(cvat.lambda.requests); + return result; + }, + }, + logger, + config: { + get backendAPI() { + return config.backendAPI; + }, + set backendAPI(value) { + config.backendAPI = value; + }, + get origin() { + return config.origin; + }, + set origin(value) { + config.origin = value; + }, + get uploadChunkSize() { + return config.uploadChunkSize; + }, + set uploadChunkSize(value) { + config.uploadChunkSize = value; + }, + get opencvPath() { + return config.opencvPath; + }, + set opencvPath(value) { + config.opencvPath = value; + }, + removeUnderlyingMaskPixels: { + get enabled() { + return config.removeUnderlyingMaskPixels.enabled; + }, + set enabled(value: boolean) { + config.removeUnderlyingMaskPixels.enabled = value; + }, + set onEmptyMaskOccurrence(value: () => void) { + config.removeUnderlyingMaskPixels.onEmptyMaskOccurrence = value; + }, + }, + get onOrganizationChange(): (orgId: number) => void { + return config.onOrganizationChange; + }, + set onOrganizationChange(value: (orgId: number) => void) { + config.onOrganizationChange = value; + }, + set globalObjectsCounter(value: number) { + config.globalObjectsCounter = value; + }, + get requestsStatusDelay() { + return config.requestsStatusDelay; + }, + set requestsStatusDelay(value) { + config.requestsStatusDelay = value; + }, + get jobMetaDataReloadPeriod() { + return config.jobMetaDataReloadPeriod; + }, + set jobMetaDataReloadPeriod(value) { + config.jobMetaDataReloadPeriod = value; + }, + get previewPlaceholders() { + return config.previewPlaceholders; + }, + set previewPlaceholders(value: Record) { + config.previewPlaceholders = value; + }, + }, + enums, + exceptions: { + Exception, + ArgumentError, + DataError, + ScriptingError, + ServerError, + }, + cloudStorages: { + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.cloudStorages.get, filter); + return result; + }, + }, + organizations: { + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.organizations.get, filter); + return result; + }, + async activate(organization) { + const result = await PluginRegistry.apiWrapper(cvat.organizations.activate, organization); + return result; + }, + async deactivate() { + const result = await PluginRegistry.apiWrapper(cvat.organizations.deactivate); + return result; + }, + async acceptInvitation(key) { + const result = await PluginRegistry.apiWrapper( + cvat.organizations.acceptInvitation, + key, + ); + return result; + }, + async declineInvitation(key) { + const result = await PluginRegistry.apiWrapper( + cvat.organizations.declineInvitation, + key, + ); + return result; + }, + async invitations(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.organizations.invitations, filter); + return result; + }, + }, + webhooks: { + async get(filter: any) { + const result = await PluginRegistry.apiWrapper(cvat.webhooks.get, filter); + return result; + }, + }, + consensus: { + settings: { + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.consensus.settings.get, filter); + return result; + }, + }, + }, + analytics: { + events: { + async export(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.analytics.events.export, filter); + return result; + }, + }, + quality: { + async reports(filter = {}, aggregate = false) { + const result = await PluginRegistry.apiWrapper(cvat.analytics.quality.reports, filter, aggregate); + return result; + }, + async conflicts(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.analytics.quality.conflicts, filter); + return result; + }, + settings: { + async get(filter = {}, aggregate = false) { + const result = await PluginRegistry.apiWrapper( + cvat.analytics.quality.settings.get, + filter, + aggregate, + ); + return result; + }, + }, + }, + }, + requests: { + async list() { + const result = await PluginRegistry.apiWrapper(cvat.requests.list); + return result; + }, + async cancel(rqID: string) { + const result = await PluginRegistry.apiWrapper(cvat.requests.cancel, rqID); + return result; + }, + async listen( + rqID: string, + options: { + callback: (request: Request) => void, + initialRequest?: Request, + }, + ) { + const result = await PluginRegistry.apiWrapper(cvat.requests.listen, rqID, options); + return result; + }, + }, + classes: { + User, + Project: implementProject(Project), + Task: implementTask(Task), + Job: implementJob(Job), + Event, + Attribute, + Label, + Statistics, + ObjectState, + MLModel, + Comment, + Issue, + FrameData, + CloudStorage, + Organization, + Webhook, + AnnotationGuide, + BaseShapesAction, + BaseCollectionAction, + QualitySettings, + QualityConflict, + QualityReport, + ApiToken, + Request, + FramesMetaData, + JobValidationLayout, + TaskValidationLayout, + }, + utils: { + mask2Rle, + rle2Mask, + propagateShapes, + validateAttributeValue, + getVisibleSkeletonElements, + }, + opencv: { + createOpenCVInterface, + }, + }; + + cvat.server = Object.freeze(cvat.server); + cvat.projects = Object.freeze(cvat.projects); + cvat.tasks = Object.freeze(cvat.tasks); + cvat.assets = Object.freeze(cvat.assets); + cvat.jobs = Object.freeze(cvat.jobs); + cvat.frames = Object.freeze(cvat.frames); + cvat.users = Object.freeze(cvat.users); + cvat.plugins = Object.freeze(cvat.plugins); + cvat.lambda = Object.freeze(cvat.lambda); + // logger: todo: logger storage implemented other way + cvat.config = Object.freeze(cvat.config); + cvat.enums = Object.freeze(cvat.enums); + cvat.exceptions = Object.freeze(cvat.exceptions); + cvat.cloudStorages = Object.freeze(cvat.cloudStorages); + cvat.organizations = Object.freeze(cvat.organizations); + cvat.webhooks = Object.freeze(cvat.webhooks); + cvat.consensus = Object.freeze(cvat.consensus); + cvat.analytics = Object.freeze(cvat.analytics); + cvat.classes = Object.freeze(cvat.classes); + cvat.utils = Object.freeze(cvat.utils); + + const implemented = Object.freeze(implementAPI(cvat)); + return implemented; +} + +export default build(); diff --git a/cvat-core/src/audio/audio-data.ts b/cvat-core/src/audio/audio-data.ts new file mode 100644 index 0000000..70d6d79 --- /dev/null +++ b/cvat-core/src/audio/audio-data.ts @@ -0,0 +1,117 @@ +// Copyright (C) CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { ChunkQuality } from 'cvat-data'; +import serverProxy from '../server-proxy'; + +function encodeWav(buffer: AudioBuffer): Blob { + const { numberOfChannels, sampleRate, length } = buffer; + const bitsPerSample = 16; + const bytesPerSample = bitsPerSample / 8; + const blockAlign = numberOfChannels * bytesPerSample; + const dataLength = length * blockAlign; + const headerLength = 44; + + const out = new ArrayBuffer(headerLength + dataLength); + const view = new DataView(out); + + const writeStr = (off: number, s: string): void => { + for (let i = 0; i < s.length; i++) view.setUint8(off + i, s.charCodeAt(i)); + }; + + writeStr(0, 'RIFF'); + view.setUint32(4, headerLength + dataLength - 8, true); + writeStr(8, 'WAVE'); + writeStr(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numberOfChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * blockAlign, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, bitsPerSample, true); + writeStr(36, 'data'); + view.setUint32(40, dataLength, true); + + const byChannelDate = new Array(numberOfChannels); + for (let ch = 0; ch < numberOfChannels; ch++) { + byChannelDate[ch] = buffer.getChannelData(ch); + } + + let offset = headerLength; + for (let i = 0; i < length; i++) { + for (let ch = 0; ch < numberOfChannels; ch++) { + const sample = Math.max(-1, Math.min(1, byChannelDate[ch][i])); + view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7FFF, true); + offset += bytesPerSample; + } + } + + return new Blob([out], { type: 'audio/wav' }); +} + +export async function fetchAndAssembleAudio( + jobId: number, + totalFrames: number, + chunkSize: number, + quality: ChunkQuality = ChunkQuality.COMPRESSED, +): Promise { + const chunkCount = Math.ceil(totalFrames / chunkSize); + + // NB: every chunk is decoded to PCM and re-encoded as WAV, including the + // single-chunk case. Serving the raw compressed blob (e.g. a VBR MP3) + // directly to the player desynchronizes the waveform from playback: the + // waveform is drawn from the Web-Audio-decoded buffer (exact duration), + // while seeking/cursor use the