chore: import upstream snapshot with attribution

.github/workflows/auto-bump-and-release.yaml

@@ -0,0 +1,62 @@
+name: Auto Bump and Release
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  auto-bump-and-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone the repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Update Application Version
+        id: update-version
+        uses: anothrNick/github-tag-action@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WITH_V: true
+          DEFAULT_BUMP: patch
+          MAJOR_STRING_TOKEN: "bump:major"
+          MINOR_STRING_TOKEN: "bump:minor"
+          PATCH_STRING_TOKEN: "bump:patch"
+      - name: Create release for ${{ steps.update-version.outputs.new_tag }}
+        # need to repeat this if statement because Github Action doesn't support early
+        # stopping for steps
+        if: ${{ steps.update-version.outputs.new_tag != steps.update-version.outputs.old_tag }}
+        run: |
+          echo Create release folder
+          mkdir kotaemon-app
+          echo ${{ steps.update-version.outputs.new_tag }} > kotaemon-app/VERSION
+          cp LICENSE.txt kotaemon-app/
+          cp flowsettings.py kotaemon-app/
+          cp app.py kotaemon-app/
+          cp .env.example kotaemon-app/.env
+          cp -r scripts kotaemon-app/
+          mkdir -p kotaemon-app/libs/ktem/ktem/
+          cp -r libs/ktem/ktem/assets kotaemon-app/libs/ktem/ktem/
+
+          tree kotaemon-app
+          zip -r kotaemon-app.zip kotaemon-app
+      - name: Release ${{ steps.update-version.outputs.new_tag }}
+        if: ${{ steps.update-version.outputs.new_tag != steps.update-version.outputs.old_tag }}
+        uses: softprops/action-gh-release@v2
+        with:
+          files: kotaemon-app.zip
+          fail_on_unmatched_files: true
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generate_release_notes: true
+          tag_name: ${{ steps.update-version.outputs.new_tag }}
+          make_latest: true
+      - name: Setup latest branch locally without switching current branch
+        if: ${{ steps.update-version.outputs.new_tag != steps.update-version.outputs.old_tag }}
+        run: git fetch origin latest:latest
+      - name: Update latest branch
+        if: ${{ steps.update-version.outputs.new_tag != steps.update-version.outputs.old_tag }}
+        run: |
+          git branch -f latest tags/${{ steps.update-version.outputs.new_tag }}
+          git checkout latest
+          git push -f -u origin latest

.github/workflows/build-push-docker.yaml

@@ -0,0 +1,106 @@
+name: Build and Push Docker Image
+
+on:
+  release:
+    types:
+      - created
+
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+"
+
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build:
+    name: Build and push container
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      matrix:
+        target:
+          - lite
+          - full
+          - ollama
+      # The maximum number of jobs that can run simultaneously
+      max-parallel: 1
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: true
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
+      - name: Set repository and image name
+        run: |
+          echo "FULL_IMAGE_NAME=${{ env.REGISTRY }}/${IMAGE_NAME,,}" >>${GITHUB_ENV}
+        env:
+          IMAGE_NAME: "${{ github.repository }}"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: arm64,arm
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.FULL_IMAGE_NAME }}
+          tags: |
+            # branch
+            type=ref,event=branch,suffix=-${{ matrix.target }}
+            # semver with suffix for lite/full targets
+            type=semver,pattern={{version}},suffix=-${{ matrix.target }}
+            # latest tag with suffix for lite/full targets
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'pre') }},suffix=-${{ matrix.target }}
+          flavor: |
+            # This is disabled here so we can use the raw form above
+            latest=false
+            # Suffix is not used here since there's no way to disable it above
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build docker image
+        uses: docker/build-push-action@v6
+        with:
+          file: Dockerfile
+          context: .
+          push: true
+          platforms: linux/amd64, linux/arm64
+          tags: |
+            ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          target: ${{ matrix.target }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/pr-lint.yaml

@@ -0,0 +1,77 @@
+name: "Lint PR"
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+permissions:
+  pull-requests: write
+
+jobs:
+  pr-title:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        id: lint_pr_title
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: marocchino/sticky-pull-request-comment@v2
+        # When the previous steps fails, the workflow would stop. By adding this
+        # condition you can continue the execution with the populated error message.
+        if: always() && (steps.lint_pr_title.outputs.error_message != null)
+        with:
+          header: pr-title-lint-error
+          message: |
+            Hey there and thank you for opening this pull request! 👋🏼
+
+            We require pull request titles to follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) and it looks like your proposed title needs to be adjusted.
+            Details:
+            ```
+            ${{ steps.lint_pr_title.outputs.error_message }}
+            ```
+
+      # Delete a previous comment when the issue has been resolved
+      - if: ${{ steps.lint_pr_title.outputs.error_message == null }}
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: pr-title-lint-error
+          delete: true
+
+  commitlint:
+    if: false # Disable this job for now
+    name: Validate commit messages
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - uses: actions/checkout@v4
+      - uses: wagoid/commitlint-github-action@v6
+        id: commitlint
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          configFile: ./.commitlintrc
+      - uses: buildingcash/json-to-markdown-table-action@v1
+        if: always() && (steps.commitlint.outcome != 'success')
+        id: table
+        with:
+          json: ${{ steps.commitlint.outputs.results }}
+      - uses: marocchino/sticky-pull-request-comment@v2
+        if: always() && (steps.commitlint.outcome != 'success')
+        with:
+          header: commitlint-error
+          message: |
+            **All commits** in this PR need to follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) and [.commitlintrc](${{ github.server_url }}/${{ github.repository }}/blob/${{ github.head_ref || github.ref_name }}/.commitlintrc).
+            Details:
+            ${{ steps.table.outputs.table }}
+
+      - if: ${{ steps.commitlint.outcome == 'success' }}
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: commitlint-error
+          delete: true

.github/workflows/style-check.yaml

@@ -0,0 +1,20 @@
+name: style-check
+
+on:
+  pull_request:
+    branches: [main, develop]
+  push:
+    branches: [main, develop]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone the repo
+        uses: actions/checkout@v4
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: run pre-commit
+        uses: pre-commit/action@v3.0.0

.github/workflows/unit-test.yaml

@@ -0,0 +1,114 @@
+name: unit-test
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+env:
+  THEFLOW_TEMP_PATH: ./tmp
+
+jobs:
+  unit-test:
+    # if: false # temporary disable this job due to legacy interface
+    #TODO: enable this job after the new interface is ready
+    if: ${{ !cancelled() }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
+        include:
+          - os: ubuntu-latest
+            shell: bash
+            ACTIVATE_ENV: ". env/bin/activate"
+            GITHUB_OUTPUT: "$GITHUB_OUTPUT"
+          # - os: windows-latest
+          #   shell: pwsh
+          #   ACTIVATE_ENV: env/Scripts/activate.ps1
+          #   GITHUB_OUTPUT: "$env:GITHUB_OUTPUT"
+
+    name: unit testing with python ${{ matrix.python-version }}
+    steps:
+      - name: Clone the repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Get Head Commit Message
+        id: get-head-commit-message
+        run: echo "message=$(git show -s --format=%s)" | tee -a ${{ matrix.GITHUB_OUTPUT }}
+
+      - name: Check ignore caching
+        id: check-ignore-cache
+        run: |
+          ignore_cache=${{ contains(steps.get-head-commit-message.outputs.message, '[ignore cache]') }}
+          echo "check=$ignore_cache" | tee -a ${{ matrix.GITHUB_OUTPUT }}
+
+      - name: Set up Python ${{ matrix.python-version }} on ${{ runner.os }}
+        uses: actions/setup-python@v4
+        id: setup_python
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Get cache key
+        id: get-cache-key
+        run: |
+          pip install "setuptools-git-versioning>=2.0,<3"
+          package_version=$(setuptools-git-versioning)
+          cache_key="${{ runner.os }}-py${{ matrix.python-version }}-v${package_version}"
+          echo "key=$cache_key" | tee -a ${{ matrix.GITHUB_OUTPUT }}
+
+      - name: Try to restore dependencies from ${{ steps.get-cache-key.outputs.key }}
+        id: restore-dependencies
+        if: steps.check-ignore-cache.outputs.check != 'true'
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ steps.get-cache-key.outputs.key }}
+          # could using cache of previous ver to reuse unchanged packages
+          restore-keys: ${{ runner.os }}-py${{ matrix.python-version }}
+
+      - name: Check cache hit
+        id: check-cache-hit
+        run: |
+          echo "cache-hit=${{ steps.restore-dependencies.outputs.cache-hit }}"
+          echo "cache-matched-key=${{ steps.restore-dependencies.outputs.cache-matched-key }}"
+          cache_hit=${{ steps.restore-dependencies.outputs.cache-primary-key == steps.restore-dependencies.outputs.cache-matched-key }}
+          echo "check=$cache_hit" | tee -a ${{ matrix.GITHUB_OUTPUT }}
+
+      - name: Install additional dependencies (if any)
+        run: |
+          uv sync --frozen --no-cache
+
+      - name: New dependencies cache for key ${{ steps.restore-dependencies.outputs.cache-primary-key }}
+        if: |
+          steps.check-ignore-cache.outputs.check != 'true' &&
+          steps.check-cache-hit.outputs.check != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ steps.restore-dependencies.outputs.cache-primary-key }}
+
+      - name: Install OS-based packages
+        run: |
+          sudo apt update -qqy
+          sudo apt install -y poppler-utils libpoppler-dev tesseract-ocr
+
+      - name: Test kotaemon with pytest
+        run: |
+          source .venv/bin/activate
+          uv pip show pytest
+          cd libs/kotaemon
+          pytest