botpress--botpress/plugins/file-synchronizer/src/sync-queue/directory-traversal-with-batching.test.ts

import type * as sdk from '@botpress/sdk'
import type * as models from '../../definitions/models'
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
import { enumerateAllFilesRecursive, type EnumerationState } from './directory-traversal-with-batching'

const _getMocks = () => ({
  logger: {
    debug: vi.fn(),
  } as unknown as sdk.BotLogger,
  integration: {
    listItemsInFolder: vi.fn().mockResolvedValue({ items: [], meta: {} }),
  },
  globMatcher: {
    matchItem: vi.fn().mockReturnValue({ shouldBeIgnored: false, shouldApplyOptions: {} }),
  },
  configuration: {
    includeFiles: [{ pathGlobPattern: '**' }],
    excludeFiles: [],
  },
  pushFilesToQueue: vi.fn(),
})

describe.concurrent('enumerateAllFilesRecursive', () => {
  describe('Basic enumeration', () => {
    it('should enumerate an empty root folder', async () => {
      // Arrange
      const mocks = _getMocks()

      // Act
      const enumerationState = await enumerateAllFilesRecursive(mocks)

      // Assert
      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
      expect(mocks.integration.listItemsInFolder).toHaveBeenNthCalledWith(1, {
        folderId: undefined,
        nextToken: undefined,
      })
      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [])
      expect(enumerationState).toBeUndefined()
    })

    it('should enumerate a single file in root folder', async () => {
      // Arrange
      const mocks = _getMocks()
      const testFile: models.FolderItem = {
        id: 'file1',
        type: 'file',
        name: 'test.txt',
      }

      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
        items: [testFile],
        meta: { nextToken: undefined },
      })

      // Act
      const enumerationState = await enumerateAllFilesRecursive(mocks)

      // Assert
      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
        expect.objectContaining({
          ...testFile,
          absolutePath: '/test.txt',
        }),
      ])
      expect(enumerationState).toBeUndefined()
    })
  })

  describe('Recursive folder traversal', () => {
    it('should recursively enumerate files in nested folders', async () => {
      // Arrange
      const mocks = _getMocks()

      // Root folder items:
      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
        items: [
          { id: 'folder1', type: 'folder', name: 'folder1' },
          { id: 'file1', type: 'file', name: 'root-file.txt' },
        ],
        meta: { nextToken: undefined },
      })

      // folder1 items:
      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
        items: [{ id: 'file2', type: 'file', name: 'nested-file.txt' }],
        meta: { nextToken: undefined },
      })

      // Act
      const enumerationState = await enumerateAllFilesRecursive(mocks)

      // Assert
      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(2)
      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
        expect.objectContaining({
          absolutePath: '/root-file.txt',
        }),
        expect.objectContaining({
          absolutePath: '/folder1/nested-file.txt',
        }),
      ])
      expect(enumerationState).toBeUndefined()
    })

    it('should handle deeply nested folder structures', async () => {
      // Arrange
      const mocks = _getMocks()
      const folderDepth = 100

      // Create a chain of nested folders
      for (let i = 1; i <= folderDepth; i++) {
        mocks.integration.listItemsInFolder.mockResolvedValueOnce(
          i === folderDepth
            ? {
                items: [{ id: 'deepFile', type: 'file', name: 'deepFile.txt' }],
                meta: { nextToken: undefined },
              }
            : {
                items: [{ id: `folder${i}`, type: 'folder', name: `folder${i}` }],
                meta: { nextToken: undefined },
              }
        )
      }

      // Act
      const enumerationState = await enumerateAllFilesRecursive(mocks)

      // Assert
      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(folderDepth)
      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
        expect.objectContaining({
          absolutePath: `/${Array.from({ length: folderDepth - 1 }, (_, i) => `folder${i + 1}`).join('/')}/deepFile.txt`,
        }),
      ])
      expect(enumerationState).toBeUndefined()
    })
  })

  describe('Pagination handling', () => {
    it('should support pagination within a folder', async () => {
      // Arrange
      const mocks = _getMocks()

      // First page of root folder
      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
        items: [{ id: 'file1', type: 'file', name: 'file1.txt' }],
        meta: { nextToken: 'page2' },
      })

      // Second page of root folder
      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
        items: [{ id: 'file2', type: 'file', name: 'file2.txt' }],
        meta: { nextToken: undefined },
      })

      // Act
      const enumerationState = await enumerateAllFilesRecursive(mocks)

      // Assert
      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(2)
      expect(mocks.pushFilesToQueue).toHaveBeenCalledWith([
        expect.objectContaining({
          id: 'file1',
        }),
        expect.objectContaining({
          id: 'file2',
        }),
      ])
      expect(enumerationState).toBeUndefined()
    })

    it('should resume using an enumeration state', async () => {
      // Arrange
      const mocks = _getMocks()

      const previousEnumerationState: EnumerationState = {
        pendingFolders: [{ absolutePath: '/folder1/', folderId: 'folder1' }],
      }

      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
        items: [{ id: 'file3', type: 'file', name: 'file3.txt' }],
        meta: { nextToken: undefined },
      })

      // Act
      const newEnumerationState = await enumerateAllFilesRecursive({
        ...mocks,
        currentEnumerationState: previousEnumerationState,
      })

      // Assert
      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
      expect(mocks.integration.listItemsInFolder).toHaveBeenCalledWith({
        folderId: 'folder1',
        nextToken: undefined,
      })
      expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
      expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
        expect.objectContaining({
          absolutePath: '/folder1/file3.txt',
        }),
      ])
      expect(newEnumerationState).toBeUndefined()
    })
  })

  describe('Glob file filtering', () => {
    it('should properly apply file filtering', async () => {
      // Arrange
      const mocks = _getMocks()

      mocks.integration.listItemsInFolder.mockResolvedValueOnce({
        items: [
          { id: 'file1', type: 'file', name: 'included.txt' },
          { id: 'file2', type: 'file', name: 'excluded.txt' },
        ],
        meta: { nextToken: undefined },
      })

      // First file is included, second is excluded
      mocks.globMatcher.matchItem
        .mockReturnValueOnce({
          shouldBeIgnored: false,
          shouldApplyOptions: { addToKbId: 'kb1' },
        })
        .mockReturnValueOnce({
          shouldBeIgnored: true,
          reason: 'matches-exclude-pattern',
        })

      // Act
      void (await enumerateAllFilesRecursive(mocks))

      // Assert
      expect(mocks.pushFilesToQueue).toHaveBeenCalledWith([
        expect.objectContaining({
          id: 'file1',
        }),
      ])
      expect(mocks.logger.debug).toHaveBeenCalledWith('Ignoring item', expect.any(Object), expect.any(String))
    })
  })
})

describe.sequential('enumerateAllFilesRecursive duration handling', () => {
  beforeEach(() => {
    vi.useFakeTimers()
  })

  afterEach(() => {
    vi.restoreAllMocks()
    vi.useRealTimers()
  })

  it('should not proceed to the next batch in case of a timeout', async () => {
    // Arrange
    const mocks = _getMocks()
    const maximumExecutionTimeMs = 5_000 // 5 seconds

    mocks.integration.listItemsInFolder.mockResolvedValueOnce({
      items: [
        { id: 'file1', type: 'file', name: 'file1.txt' },
        { id: 'file2', type: 'file', name: 'file2.txt' },
        { id: 'file3', type: 'file', name: 'file3.txt' },
      ],
      meta: { nextToken: 'abcd' },
    })

    const resultPromise = enumerateAllFilesRecursive({
      ...mocks,
      maximumExecutionTimeMs,
    })

    // Advance time after the first batch is processed:
    vi.advanceTimersByTime(maximumExecutionTimeMs + 100)

    void (await resultPromise)

    // Assert
    expect(mocks.integration.listItemsInFolder).toHaveBeenCalledTimes(1)
  })

  it('should process the entire current batch in case of a timeout', async () => {
    // Arrange
    const mocks = _getMocks()
    const maximumExecutionTimeMs = 5_000 // 5 seconds

    mocks.integration.listItemsInFolder.mockResolvedValueOnce({
      items: [
        { id: 'file1', type: 'file', name: 'file1.txt' },
        { id: 'file2', type: 'file', name: 'file2.txt' },
        { id: 'file3', type: 'file', name: 'file3.txt' },
      ],
      meta: { nextToken: 'abcd' },
    })

    const resultPromise = enumerateAllFilesRecursive({
      ...mocks,
      maximumExecutionTimeMs,
    })

    // Advance time after the first batch is processed:
    vi.advanceTimersByTime(maximumExecutionTimeMs + 100)

    void (await resultPromise)

    // Assert
    expect(mocks.pushFilesToQueue).toHaveBeenCalledTimes(1)
    expect(mocks.pushFilesToQueue).toHaveBeenNthCalledWith(1, [
      expect.objectContaining({
        id: 'file1',
      }),
      expect.objectContaining({
        id: 'file2',
      }),
      expect.objectContaining({
        id: 'file3',
      }),
    ])
  })
})