chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,13 @@
|
||||
import rootConfig from '../../eslint.config.mjs'
|
||||
|
||||
export default [
|
||||
...rootConfig,
|
||||
{
|
||||
languageOptions: {
|
||||
parserOptions: {
|
||||
project: ['./tsconfig.json'],
|
||||
tsconfigRootDir: import.meta.dirname,
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1,33 @@
|
||||
# Description
|
||||
|
||||
Enable your bot with the ability to list and manage your files in Google Drive and to download/upload data between Google Drive and the Botpress files API.
|
||||
|
||||
# Configuration
|
||||
|
||||
Due to the potentially sensitive nature of the files in your Google Drive, the Google Drive integration requires a secure connection between Botpress and Google Drive. To establish this secure connection, you **must** configure the Google Drive integration using OAuth.
|
||||
|
||||
## Automatic configuration with OAuth
|
||||
|
||||
To set up the Google Drive integration using OAuth, click the authorization button and follow the on-screen instructions to connect your Botpress bot to Google Drive.
|
||||
|
||||
When using this configuration mode, a Botpress-managed Google Drive application will be used to connect to your Google Drive account. However, actions taken by the bot will be attributed to the user who authorized the connection, rather than the application. For this reason, **we do not recommend using personal Google Drive accounts** for this integration. You should set up a service account and use this account to authorize the connection. You can then share specific folders and files with this service account to give your bot access to these files.
|
||||
|
||||
## Configuring the integration in Botpress
|
||||
|
||||
1. Authorize the Google Drive integration by clicking the authorization button.
|
||||
2. Follow the on-screen instructions to connect your Botpress chatbot to Google Drive.
|
||||
3. Once the connection is established, you can save the configuration and enable the integration.
|
||||
|
||||
# Using the integration
|
||||
|
||||
Use the available actions to manage your files and download/upload content from and to Google Drive.
|
||||
|
||||
Use the available triggers to know when a file or folder was created or deleted.
|
||||
|
||||
Use the 'syncChannels' action to create and update subscription channels on all available files and folders. These channels are what allow your bot to be notified on resource creation and deletion. The channels are valid up to one day. Make sure this action is called once a day to prevent event loss. Calling this action too often may result in errors and events being lost due to the Google Drive subscription creation rate limit.
|
||||
|
||||
# Limitations
|
||||
|
||||
Standard Google Drive API limitations apply to the Google Drive integration in Botpress. These limitations include rate limits, file size restrictions, and other constraints imposed by the Google Drive platform. Ensure that your bot adheres to these limitations to maintain optimal performance and reliability.
|
||||
|
||||
More details are available in the [Google Drive API documentation](https://developers.google.com/drive/api/guides/about-sdk).
|
||||
@@ -0,0 +1,8 @@
|
||||
<svg viewBox="0 0 87.3 78" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="m6.6 66.85 3.85 6.65c.8 1.4 1.95 2.5 3.3 3.3l13.75-23.8h-27.5c0 1.55.4 3.1 1.2 4.5z" fill="#0066da"/>
|
||||
<path d="m43.65 25-13.75-23.8c-1.35.8-2.5 1.9-3.3 3.3l-25.4 44a9.06 9.06 0 0 0 -1.2 4.5h27.5z" fill="#00ac47"/>
|
||||
<path d="m73.55 76.8c1.35-.8 2.5-1.9 3.3-3.3l1.6-2.75 7.65-13.25c.8-1.4 1.2-2.95 1.2-4.5h-27.502l5.852 11.5z" fill="#ea4335"/>
|
||||
<path d="m43.65 25 13.75-23.8c-1.35-.8-2.9-1.2-4.5-1.2h-18.5c-1.6 0-3.15.45-4.5 1.2z" fill="#00832d"/>
|
||||
<path d="m59.8 53h-32.3l-13.75 23.8c1.35.8 2.9 1.2 4.5 1.2h50.8c1.6 0 3.15-.45 4.5-1.2z" fill="#2684fc"/>
|
||||
<path d="m73.4 26.5-12.7-22c-.8-1.4-1.95-2.5-3.3-3.3l-13.75 23.8 16.15 28h27.45c0-1.55-.4-3.1-1.2-4.5z" fill="#ffba00"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 755 B |
@@ -0,0 +1,228 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import filesReadonly from './bp_modules/files-readonly'
|
||||
import {
|
||||
fileSchema,
|
||||
createFileArgSchema,
|
||||
updateFileArgSchema,
|
||||
uploadFileDataArgSchema,
|
||||
downloadFileDataArgSchema,
|
||||
listFoldersOutputSchema,
|
||||
listFilesOutputSchema,
|
||||
readFileArgSchema,
|
||||
listItemsInputSchema,
|
||||
deleteFileArgSchema,
|
||||
downloadFileDataOutputSchema,
|
||||
fileDeletedEventSchema,
|
||||
folderSchema,
|
||||
folderDeletedEventSchema,
|
||||
baseDiscriminatedFileSchema,
|
||||
fileChannelSchema,
|
||||
} from './src/schemas'
|
||||
|
||||
// TODO: use default options
|
||||
const toJSONSchemaOptions: Partial<sdk.z.transforms.JSONSchemaGenerationOptions> = {
|
||||
discriminatedUnionStrategy: 'anyOf',
|
||||
discriminator: false,
|
||||
}
|
||||
|
||||
export default new sdk.IntegrationDefinition({
|
||||
name: 'googledrive',
|
||||
title: 'Google Drive',
|
||||
description: 'Access and manage your Google Drive files from your bot.',
|
||||
version: '0.4.3',
|
||||
readme: 'hub.md',
|
||||
icon: 'icon.svg',
|
||||
attributes: {
|
||||
category: 'File Management',
|
||||
repo: 'botpress',
|
||||
},
|
||||
configuration: {
|
||||
identifier: {
|
||||
linkTemplateScript: 'linkTemplate.vrl',
|
||||
},
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
actions: {
|
||||
listFiles: {
|
||||
// TODO: Implement listable
|
||||
title: 'List Files',
|
||||
description: 'List files in Google Drive',
|
||||
input: {
|
||||
schema: listItemsInputSchema,
|
||||
},
|
||||
output: {
|
||||
schema: listFilesOutputSchema,
|
||||
},
|
||||
},
|
||||
listFolders: {
|
||||
// TODO: Implement listable
|
||||
title: 'List folders',
|
||||
description: 'List folders in Google Drive',
|
||||
input: {
|
||||
schema: listItemsInputSchema,
|
||||
},
|
||||
output: {
|
||||
schema: listFoldersOutputSchema,
|
||||
},
|
||||
},
|
||||
createFile: {
|
||||
// TODO: Implement creatable
|
||||
title: 'Create File',
|
||||
description: 'Create an empty file in Google Drive',
|
||||
input: {
|
||||
schema: createFileArgSchema,
|
||||
},
|
||||
output: {
|
||||
schema: fileSchema.describe('The file created in Google Drive'),
|
||||
},
|
||||
},
|
||||
readFile: {
|
||||
// TODO: Implement readable
|
||||
title: 'Read File',
|
||||
description: "Read a file's metadata in a Google Drive",
|
||||
input: {
|
||||
schema: readFileArgSchema,
|
||||
},
|
||||
output: {
|
||||
schema: fileSchema.describe('The file read from Google Drive'),
|
||||
},
|
||||
},
|
||||
updateFile: {
|
||||
// TODO: Implement updatable
|
||||
title: 'Update File',
|
||||
description: "Update a file's metadata in Google Drive",
|
||||
input: {
|
||||
schema: updateFileArgSchema,
|
||||
},
|
||||
output: {
|
||||
schema: fileSchema.describe('The file updated in Google Drive'),
|
||||
},
|
||||
},
|
||||
deleteFile: {
|
||||
// TODO: Implement deletable
|
||||
title: 'Delete File',
|
||||
description: 'Deletes a file in Google Drive',
|
||||
input: {
|
||||
schema: deleteFileArgSchema,
|
||||
},
|
||||
output: {
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
},
|
||||
uploadFileData: {
|
||||
title: 'Upload file data',
|
||||
description: 'Upload data to a file in Google Drive',
|
||||
input: {
|
||||
schema: uploadFileDataArgSchema,
|
||||
},
|
||||
output: {
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
},
|
||||
downloadFileData: {
|
||||
title: 'Download file data',
|
||||
description: 'Download data from a file in Google Drive',
|
||||
input: {
|
||||
schema: downloadFileDataArgSchema,
|
||||
},
|
||||
output: {
|
||||
schema: downloadFileDataOutputSchema,
|
||||
},
|
||||
},
|
||||
syncChannels: {
|
||||
title: 'Sync Channels',
|
||||
description: 'Sync channels for file change subscriptions',
|
||||
input: {
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
output: {
|
||||
schema: sdk.z.object({}),
|
||||
},
|
||||
},
|
||||
},
|
||||
events: {
|
||||
fileCreated: {
|
||||
title: 'File Created',
|
||||
description: 'Triggered when a file is created in Google Drive',
|
||||
schema: fileSchema,
|
||||
},
|
||||
fileDeleted: {
|
||||
title: 'File Deleted',
|
||||
description: 'Triggered when a file is deleted in Google Drive',
|
||||
schema: fileDeletedEventSchema,
|
||||
},
|
||||
folderCreated: {
|
||||
title: 'Folder Created',
|
||||
description: 'Triggered when a folder is created in Google Drive',
|
||||
schema: folderSchema,
|
||||
},
|
||||
folderDeleted: {
|
||||
title: 'Folder Deleted',
|
||||
description: 'Triggered when a folder is deleted in Google Drive',
|
||||
schema: folderDeletedEventSchema,
|
||||
},
|
||||
},
|
||||
states: {
|
||||
configuration: {
|
||||
type: 'integration',
|
||||
schema: sdk.z.object({
|
||||
refreshToken: sdk.z
|
||||
.string()
|
||||
.title('Refresh token')
|
||||
.describe('The refresh token to use to authenticate with Google. It gets exchanged for a bearer token'),
|
||||
}),
|
||||
},
|
||||
filesCache: {
|
||||
type: 'integration',
|
||||
schema: sdk.z.object({
|
||||
filesCache: sdk.z
|
||||
.record(sdk.z.string(), baseDiscriminatedFileSchema)
|
||||
.title('Files cache')
|
||||
.describe('Map of known files'),
|
||||
}),
|
||||
},
|
||||
filesChannelsCache: {
|
||||
type: 'integration',
|
||||
schema: sdk.z.object({
|
||||
filesChannelsCache: sdk.z
|
||||
.record(sdk.z.string(), fileChannelSchema)
|
||||
.title('Files change subscription channels')
|
||||
.describe('Serialized set of channels for file change subscriptions'),
|
||||
}),
|
||||
},
|
||||
},
|
||||
secrets: {
|
||||
CLIENT_ID: {
|
||||
description: 'The client ID in your Google Cloud Credentials',
|
||||
},
|
||||
CLIENT_SECRET: {
|
||||
description: 'The client secret associated with your client ID',
|
||||
},
|
||||
WEBHOOK_SECRET: {
|
||||
description: 'The secret used to sign webhook tokens. Should be a high-entropy string that only Botpress knows',
|
||||
},
|
||||
FILE_PICKER_API_KEY: {
|
||||
description: 'The API key used to access the Google Picker API',
|
||||
},
|
||||
},
|
||||
__advanced: { toJSONSchemaOptions },
|
||||
}).extend(filesReadonly, ({}) => ({
|
||||
entities: {},
|
||||
actions: {
|
||||
listItemsInFolder: {
|
||||
name: 'filesReadonlyListItemsInFolder',
|
||||
attributes: { ...sdk.WELL_KNOWN_ATTRIBUTES.HIDDEN_IN_STUDIO },
|
||||
},
|
||||
transferFileToBotpress: {
|
||||
name: 'filesReadonlyTransferFileToBotpress',
|
||||
attributes: { ...sdk.WELL_KNOWN_ATTRIBUTES.HIDDEN_IN_STUDIO },
|
||||
},
|
||||
},
|
||||
events: {
|
||||
fileCreated: { name: 'filesReadonlyFileCreated' },
|
||||
fileUpdated: { name: 'filesReadonlyFileUpdated' },
|
||||
fileDeleted: { name: 'filesReadonlyFileDeleted' },
|
||||
folderDeletedRecursive: { name: 'filesReadonlyFolderDeletedRecursive' },
|
||||
aggregateFileChanges: { name: 'filesReadonlyAggregateFileChanges' },
|
||||
},
|
||||
}))
|
||||
@@ -0,0 +1,4 @@
|
||||
webhookId = to_string!(.webhookId)
|
||||
webhookUrl = to_string!(.webhookUrl)
|
||||
|
||||
"{{ webhookUrl }}/oauth/wizard/start?state={{ webhookId }}"
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "@botpresshub/googledrive",
|
||||
"scripts": {
|
||||
"check:type": "tsc --noEmit",
|
||||
"check:bplint": "bp lint",
|
||||
"build": "bp add -y && bp build",
|
||||
"test": "vitest --run"
|
||||
},
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@botpress/client": "workspace:*",
|
||||
"@botpress/common": "workspace:*",
|
||||
"@botpress/sdk": "workspace:*",
|
||||
"@botpress/sdk-addons": "workspace:*",
|
||||
"axios": "^1.7.7",
|
||||
"googleapis": "^144.0.0",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"uuid": "^9.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@botpress/cli": "workspace:*",
|
||||
"@botpress/sdk": "workspace:*",
|
||||
"@types/jsonwebtoken": "^9.0.3",
|
||||
"@types/uuid": "^9.0.1",
|
||||
"preact": "^10.26.6"
|
||||
},
|
||||
"bpDependencies": {
|
||||
"files-readonly": "../../interfaces/files-readonly"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import axios from 'axios'
|
||||
import { Stream } from 'stream'
|
||||
import { Client as DriveClient } from './client'
|
||||
import { wrapWithTryCatch } from './error-handling'
|
||||
import { FileChannelsCache } from './file-channels-cache'
|
||||
import { FileEventHandler } from './file-event-handler'
|
||||
import { downloadToBotpress } from './files-api-utils'
|
||||
import { FilesCache } from './files-cache'
|
||||
import { filesReadonlyActions } from './files-readonly/actions'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
type ActionPropsAndTools<T extends bp.AnyActionProps> = {
|
||||
driveClient: DriveClient
|
||||
filesCache: FilesCache
|
||||
fileChannelsCache: FileChannelsCache
|
||||
fileEventHandler: FileEventHandler
|
||||
} & T
|
||||
|
||||
/**
|
||||
* @returns Base actions props and tools used by actions
|
||||
*/
|
||||
const createActionPropsAndTools = async <T extends bp.AnyActionProps>(props: T): Promise<ActionPropsAndTools<T>> => {
|
||||
const { client, ctx, logger } = props
|
||||
const driveClient = await DriveClient.create({ client, ctx, logger })
|
||||
const filesCache = await FilesCache.load({ client, ctx })
|
||||
const fileChannelsCache = await FileChannelsCache.load({ client, ctx })
|
||||
driveClient.setCache(filesCache)
|
||||
return {
|
||||
driveClient,
|
||||
filesCache,
|
||||
fileChannelsCache,
|
||||
fileEventHandler: new FileEventHandler(client, driveClient, filesCache, fileChannelsCache),
|
||||
...props,
|
||||
}
|
||||
}
|
||||
|
||||
const saveAllCaches = async <T extends bp.AnyActionProps>(props: ActionPropsAndTools<T>) => {
|
||||
await props.filesCache.save()
|
||||
await props.fileChannelsCache.save()
|
||||
}
|
||||
|
||||
const makeSaveAllCachesAndReturnResult =
|
||||
<T extends bp.AnyActionProps>(props: ActionPropsAndTools<T>) =>
|
||||
async <R>(actionOutput: R) => {
|
||||
await saveAllCaches(props)
|
||||
return actionOutput
|
||||
}
|
||||
|
||||
const listFiles: bp.IntegrationProps['actions']['listFiles'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input } = props
|
||||
const saveAllCachesAndReturnResult = makeSaveAllCachesAndReturnResult(props)
|
||||
return await driveClient.listFiles(input).then(saveAllCachesAndReturnResult)
|
||||
}, 'Error listing files')
|
||||
|
||||
const listFolders: bp.IntegrationProps['actions']['listFolders'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input } = props
|
||||
const saveAllCachesAndReturnResult = makeSaveAllCachesAndReturnResult(props)
|
||||
return await driveClient.listFolders(input).then(saveAllCachesAndReturnResult)
|
||||
}, 'Error listing folders')
|
||||
|
||||
const createFile: bp.IntegrationProps['actions']['createFile'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input, fileEventHandler } = props
|
||||
const newFile = await driveClient.createFile(input)
|
||||
await fileEventHandler.handleFileCreated({ type: 'normal', ...newFile })
|
||||
await saveAllCaches(props)
|
||||
return newFile
|
||||
}, 'Error creating file')
|
||||
|
||||
const readFile: bp.IntegrationProps['actions']['readFile'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input } = props
|
||||
const saveAllCachesAndReturnResult = makeSaveAllCachesAndReturnResult(props)
|
||||
return await driveClient.readFile(input.id).then(saveAllCachesAndReturnResult)
|
||||
}, 'Error reading file')
|
||||
|
||||
const updateFile: bp.IntegrationProps['actions']['updateFile'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input } = props
|
||||
const saveAllCachesAndReturnResult = makeSaveAllCachesAndReturnResult(props)
|
||||
return await driveClient.updateFile(input).then(saveAllCachesAndReturnResult)
|
||||
}, 'Error updating file')
|
||||
|
||||
const deleteFile: bp.IntegrationProps['actions']['deleteFile'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input, fileEventHandler } = props
|
||||
await driveClient.deleteFile(input.id)
|
||||
const oldFile = props.filesCache.find(input.id)
|
||||
if (oldFile) {
|
||||
await fileEventHandler.handleFileDeleted(oldFile)
|
||||
}
|
||||
await saveAllCaches(props)
|
||||
return {}
|
||||
}, 'Error deleting file')
|
||||
|
||||
const uploadFileData: bp.IntegrationProps['actions']['uploadFileData'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input } = props
|
||||
const { id, url, mimeType } = input
|
||||
const { data } = await axios.get<Stream>(url, {
|
||||
responseType: 'stream',
|
||||
})
|
||||
await driveClient.uploadFileData({ id, mimeType, data })
|
||||
return {}
|
||||
}, 'Error uploading file')
|
||||
|
||||
const downloadFileData: bp.IntegrationProps['actions']['downloadFileData'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, input } = props
|
||||
const { id, index } = input
|
||||
|
||||
const { botpressFileId } = await downloadToBotpress({
|
||||
botpressFileKey: id,
|
||||
googleDriveFileId: id,
|
||||
client: props.client,
|
||||
driveClient,
|
||||
indexFile: index,
|
||||
})
|
||||
|
||||
await saveAllCaches(props)
|
||||
return { bpFileId: botpressFileId }
|
||||
}, 'Error downloading file')
|
||||
|
||||
const syncChannels: bp.IntegrationProps['actions']['syncChannels'] = wrapWithTryCatch(async (baseProps) => {
|
||||
const props = await createActionPropsAndTools(baseProps)
|
||||
const { driveClient, fileChannelsCache } = props
|
||||
const { fileChannels: newChannels } = await driveClient.tryWatchAll()
|
||||
const oldChannels = fileChannelsCache.setAll(newChannels)
|
||||
await fileChannelsCache.save()
|
||||
await driveClient.tryUnwatch(oldChannels)
|
||||
return {}
|
||||
}, 'Error syncing channels')
|
||||
|
||||
export default {
|
||||
listFiles,
|
||||
listFolders,
|
||||
createFile,
|
||||
readFile,
|
||||
updateFile,
|
||||
deleteFile,
|
||||
uploadFileData,
|
||||
downloadFileData,
|
||||
syncChannels,
|
||||
|
||||
...filesReadonlyActions,
|
||||
} as const satisfies bp.IntegrationProps['actions']
|
||||
@@ -0,0 +1,106 @@
|
||||
import * as oauthWizard from '@botpress/common/src/oauth-wizard'
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { google } from 'googleapis'
|
||||
import { GoogleOAuth2Client, GoogleDriveClient } from './types'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const getAuthenticatedGoogleClient = async ({
|
||||
client,
|
||||
ctx,
|
||||
}: {
|
||||
client: bp.Client
|
||||
ctx: bp.Context
|
||||
}): Promise<GoogleDriveClient> => {
|
||||
const oauth2Client = await _getAuthenticatedOAuthClient({ client, ctx })
|
||||
|
||||
return google.drive({ version: 'v3', auth: oauth2Client })
|
||||
}
|
||||
|
||||
export const getAccessToken = async (props: { client: bp.Client; ctx: bp.Context }) => {
|
||||
const oauth2Client = await _getAuthenticatedOAuthClient(props)
|
||||
const { token } = await oauth2Client.getAccessToken()
|
||||
|
||||
if (!token) {
|
||||
throw new sdk.RuntimeError('Unable to obtain access token. Please try the OAuth flow again.')
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The updated refresh token
|
||||
*/
|
||||
export const updateRefreshTokenFromAuthorizationCode = async ({
|
||||
authorizationCode,
|
||||
client,
|
||||
ctx,
|
||||
}: {
|
||||
authorizationCode: string
|
||||
client: bp.Client
|
||||
ctx: bp.Context
|
||||
}): Promise<string> => {
|
||||
const refreshToken = await exchangeAuthorizationCodeForRefreshToken(authorizationCode)
|
||||
await _saveRefreshTokenIntoStates({ client, ctx, refreshToken })
|
||||
return refreshToken
|
||||
}
|
||||
|
||||
const _getAuthenticatedOAuthClient = async ({
|
||||
client,
|
||||
ctx,
|
||||
}: {
|
||||
client: bp.Client
|
||||
ctx: bp.Context
|
||||
}): Promise<GoogleOAuth2Client> => {
|
||||
const token = await _getRefreshTokenFromStates({ client, ctx })
|
||||
|
||||
const oauth2Client = _getOAuthClient()
|
||||
oauth2Client.setCredentials({ refresh_token: token })
|
||||
|
||||
return oauth2Client
|
||||
}
|
||||
|
||||
const _getOAuthClient = (): GoogleOAuth2Client =>
|
||||
new google.auth.OAuth2(
|
||||
bp.secrets.CLIENT_ID,
|
||||
bp.secrets.CLIENT_SECRET,
|
||||
oauthWizard.getWizardStepUrl('oauth-callback').href
|
||||
)
|
||||
|
||||
const _getRefreshTokenFromStates = async ({ client, ctx }: { client: bp.Client; ctx: bp.Context }) => {
|
||||
const { state } = await client.getState({
|
||||
type: 'integration',
|
||||
name: 'configuration',
|
||||
id: ctx.integrationId,
|
||||
})
|
||||
|
||||
return state.payload.refreshToken
|
||||
}
|
||||
|
||||
const _saveRefreshTokenIntoStates = async ({
|
||||
client,
|
||||
ctx,
|
||||
refreshToken,
|
||||
}: {
|
||||
client: bp.Client
|
||||
ctx: bp.Context
|
||||
refreshToken: string
|
||||
}) => {
|
||||
await client.setState({
|
||||
type: 'integration',
|
||||
name: 'configuration',
|
||||
id: ctx.integrationId,
|
||||
payload: { refreshToken },
|
||||
})
|
||||
}
|
||||
|
||||
const exchangeAuthorizationCodeForRefreshToken = async (authorizationCode: string) => {
|
||||
const oauth2Client = _getOAuthClient()
|
||||
const { tokens } = await oauth2Client.getToken({
|
||||
code: authorizationCode,
|
||||
})
|
||||
|
||||
if (!tokens.refresh_token) {
|
||||
throw new sdk.RuntimeError('Unable to obtain refresh token. Please try the OAuth flow again.')
|
||||
}
|
||||
|
||||
return tokens.refresh_token
|
||||
}
|
||||
@@ -0,0 +1,555 @@
|
||||
import { RuntimeError } from '@botpress/sdk'
|
||||
import { Readable, Stream } from 'stream'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { getAuthenticatedGoogleClient } from './auth'
|
||||
import { handleNotFoundError, handleRateLimitError, isGaxiosError } from './error-handling'
|
||||
import { serializeToken } from './file-notification-token'
|
||||
import { FilesCache } from './files-cache'
|
||||
import { APP_GOOGLE_FOLDER_MIMETYPE, APP_GOOGLE_SHORTCUT_MIMETYPE, INDEXABLE_MIMETYPES } from './mime-types'
|
||||
import {
|
||||
BaseDiscriminatedFile,
|
||||
GoogleDriveClient,
|
||||
BaseNormalFile,
|
||||
File,
|
||||
BaseFolderFile,
|
||||
Folder,
|
||||
ListFilesOutput,
|
||||
ListFoldersOutput,
|
||||
CreateFileArgs,
|
||||
UpdateFileArgs,
|
||||
ListItemsInput,
|
||||
ListItemsOutput,
|
||||
BaseGenericFileUnion,
|
||||
FileChannel,
|
||||
GenericFile,
|
||||
} from './types'
|
||||
import { listItemsAndProcess, ListFunction, streamToBuffer, ListItemsInputWithArgs, listAllItems } from './utils'
|
||||
import {
|
||||
getFileTypeFromMimeType,
|
||||
parseChannel,
|
||||
parseBaseGeneric,
|
||||
parseBaseGenerics,
|
||||
parseBaseNormal,
|
||||
} from './validation'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
type DownloadFileDataClientOutput = {
|
||||
mimeType: string
|
||||
dataSize: number
|
||||
} & (
|
||||
| {
|
||||
dataType: 'buffer'
|
||||
data: Buffer
|
||||
}
|
||||
| {
|
||||
dataType: 'stream'
|
||||
data: Readable
|
||||
}
|
||||
)
|
||||
type TryWatchAllOutput = {
|
||||
fileChannels: FileChannel[]
|
||||
hasError: boolean
|
||||
}
|
||||
|
||||
const MAX_RESOURCE_WATCH_EXPIRATION_DELAY_MS = 86400 * 1000 // 24 hours
|
||||
const MAX_EXPORT_FILE_SIZE_BYTES = 10000000 // 10MB, as per the Google Drive API doc
|
||||
const MYDRIVE_ID_ALIAS = 'root'
|
||||
const PAGE_SIZE = 100
|
||||
const GOOGLE_API_EXPORTFORMATS_FIELDS = 'exportFormats'
|
||||
const GOOGLE_API_FILE_FIELDS =
|
||||
'id, name, mimeType, parents, size, sha256Checksum, md5Checksum, version, trashed, modifiedTime, driveId, sharedWithMeTime'
|
||||
const GOOGLE_API_FILELIST_FIELDS = `files(${GOOGLE_API_FILE_FIELDS}), nextPageToken`
|
||||
|
||||
const INCLUDE_FILES_FROM_ALL_DRIVES = {
|
||||
includeItemsFromAllDrives: true,
|
||||
supportsAllDrives: true,
|
||||
} as const
|
||||
|
||||
export class Client {
|
||||
private constructor(
|
||||
private _ctx: bp.Context,
|
||||
private _googleClient: GoogleDriveClient,
|
||||
private _filesCache: FilesCache,
|
||||
private _logger: bp.Logger
|
||||
) {}
|
||||
|
||||
public static async create({
|
||||
client,
|
||||
ctx,
|
||||
logger,
|
||||
}: {
|
||||
client: bp.Client
|
||||
ctx: bp.Context
|
||||
logger: bp.Logger
|
||||
}): Promise<Client> {
|
||||
const googleClient = await getAuthenticatedGoogleClient({
|
||||
client,
|
||||
ctx,
|
||||
})
|
||||
const filesCache = new FilesCache(client, ctx)
|
||||
return new Client(ctx, googleClient, filesCache, logger)
|
||||
}
|
||||
|
||||
public setCache(filesCache: FilesCache) {
|
||||
this._filesCache = filesCache
|
||||
}
|
||||
|
||||
public async getRootFolderId(): Promise<string> {
|
||||
try {
|
||||
const response = await this._googleClient.files.get({ fileId: MYDRIVE_ID_ALIAS })
|
||||
return response.data.id!
|
||||
} catch (thrown: unknown) {
|
||||
if (isGaxiosError(thrown) && thrown.toString().includes('File not found: ')) {
|
||||
return thrown.toString().split('File not found: ')[1]!.slice(0, -1)
|
||||
}
|
||||
throw thrown
|
||||
}
|
||||
}
|
||||
|
||||
public async listFiles({ nextToken }: ListItemsInput): Promise<ListFilesOutput> {
|
||||
const { items: baseFiles, meta } = await this._listBaseNormalFiles({ nextToken })
|
||||
const completeFilesPromises = baseFiles.map((f) => this._getCompleteFileFromBaseFile(f))
|
||||
const items = await Promise.all(completeFilesPromises)
|
||||
return {
|
||||
items,
|
||||
meta,
|
||||
}
|
||||
}
|
||||
|
||||
private async _listBaseNormalFiles({ nextToken }: ListItemsInput): Promise<ListItemsOutput<BaseNormalFile>> {
|
||||
const {
|
||||
items: newFiles,
|
||||
meta: { nextToken: newNextToken },
|
||||
} = await this._listBaseGenericFiles({
|
||||
nextToken,
|
||||
args: {
|
||||
searchQuery: `mimeType != '${APP_GOOGLE_FOLDER_MIMETYPE}' and mimeType != '${APP_GOOGLE_SHORTCUT_MIMETYPE}'`,
|
||||
},
|
||||
})
|
||||
|
||||
const items = newFiles.filter((f) => f.type === 'normal')
|
||||
return {
|
||||
items,
|
||||
meta: {
|
||||
nextToken: newNextToken,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
public async listFolders({ nextToken }: ListItemsInput): Promise<ListFoldersOutput> {
|
||||
const { items: baseFolders, meta } = await this._listBaseFolderFiles({ nextToken })
|
||||
|
||||
const completeFoldersPromises = baseFolders.map((f) => this._getCompleteFolderFromBaseFolder(f))
|
||||
const items = await Promise.all(completeFoldersPromises)
|
||||
return {
|
||||
items,
|
||||
meta,
|
||||
}
|
||||
}
|
||||
|
||||
private async _listBaseFolderFiles({ nextToken }: ListItemsInput): Promise<ListItemsOutput<BaseFolderFile>> {
|
||||
const {
|
||||
items: newFiles,
|
||||
meta: { nextToken: newNextToken },
|
||||
} = await this._listBaseGenericFiles({
|
||||
nextToken,
|
||||
args: {
|
||||
searchQuery: `mimeType = '${APP_GOOGLE_FOLDER_MIMETYPE}'`,
|
||||
},
|
||||
})
|
||||
if (nextToken === undefined) {
|
||||
// My Drive is not returned by list operation but needs to be part of list, so we add it to first page
|
||||
const myDriveFile = await this._fetchFile(MYDRIVE_ID_ALIAS)
|
||||
newFiles.push(myDriveFile)
|
||||
}
|
||||
const items = newFiles.filter((f) => f.type === 'folder')
|
||||
return {
|
||||
items,
|
||||
meta: {
|
||||
nextToken: newNextToken,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
public async getChildren(folderId: string): Promise<GenericFile[]> {
|
||||
const files = await listAllItems(this._listBaseGenericFiles.bind(this), {
|
||||
searchQuery: this._getParentsFilter(folderId),
|
||||
})
|
||||
return await Promise.all(files.map((f) => this._getCompleteFile(f)))
|
||||
}
|
||||
|
||||
public async getChildrenSubset({
|
||||
folderId,
|
||||
extraQuery,
|
||||
nextToken,
|
||||
}: {
|
||||
folderId: string
|
||||
extraQuery?: string
|
||||
nextToken?: string
|
||||
}) {
|
||||
const searchQuery = this._getParentsFilter(folderId) + (extraQuery ? ` and ${extraQuery}` : '')
|
||||
const listResponse = await this._googleClient.files.list({
|
||||
corpora: 'user',
|
||||
fields: GOOGLE_API_FILELIST_FIELDS,
|
||||
q: `${searchQuery} and trashed != true`,
|
||||
pageToken: nextToken,
|
||||
pageSize: PAGE_SIZE,
|
||||
spaces: 'drive',
|
||||
...INCLUDE_FILES_FROM_ALL_DRIVES,
|
||||
})
|
||||
return { files: listResponse.data.files, nextToken: listResponse.data.nextPageToken ?? undefined }
|
||||
}
|
||||
|
||||
private _getParentsFilter(parentId: string): string {
|
||||
return parentId === MYDRIVE_ID_ALIAS ? 'not trashed' : `'${parentId}' in parents`
|
||||
}
|
||||
|
||||
public async createFile({ name, parentId, mimeType }: CreateFileArgs): Promise<File> {
|
||||
const response = await this._googleClient.files.create({
|
||||
fields: GOOGLE_API_FILE_FIELDS,
|
||||
requestBody: {
|
||||
name,
|
||||
parents: parentId ? [parentId] : undefined,
|
||||
mimeType,
|
||||
},
|
||||
})
|
||||
const file = parseBaseNormal(response.data)
|
||||
this._filesCache.set({ type: 'normal', ...file })
|
||||
return await this._getCompleteFileFromBaseFile(file)
|
||||
}
|
||||
|
||||
public async readGenericFile(id: string): Promise<GenericFile> {
|
||||
const file = await this._fetchFile(id)
|
||||
return await this._getCompleteFile(file)
|
||||
}
|
||||
|
||||
public async readFile(id: string): Promise<File> {
|
||||
const file = await this._fetchFile(id)
|
||||
if (file.type !== 'normal') {
|
||||
throw new RuntimeError(`Attempted to read a file of type ${file.type}`)
|
||||
}
|
||||
return await this._getCompleteFileFromBaseFile(file)
|
||||
}
|
||||
|
||||
public async updateFile({ id: fileId, name, parentId }: UpdateFileArgs): Promise<File> {
|
||||
const addParents = parentId ? `${parentId}` : undefined
|
||||
const response = await this._googleClient.files.update({
|
||||
fields: GOOGLE_API_FILE_FIELDS,
|
||||
fileId,
|
||||
addParents, // Also removes old parents
|
||||
requestBody: {
|
||||
name,
|
||||
},
|
||||
})
|
||||
const file = parseBaseNormal(response.data)
|
||||
this._filesCache.set({ type: 'normal', ...file })
|
||||
return await this._getCompleteFileFromBaseFile(file)
|
||||
}
|
||||
|
||||
public async deleteFile(id: string) {
|
||||
await this._googleClient.files.delete({
|
||||
fileId: id,
|
||||
})
|
||||
}
|
||||
|
||||
public async uploadFileData({ id, mimeType, data }: { id: string; mimeType?: string; data: Stream }) {
|
||||
await this._googleClient.files.update({
|
||||
fileId: id,
|
||||
media: {
|
||||
body: data,
|
||||
mimeType,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
public async downloadFileData({ id }: { id: string }): Promise<DownloadFileDataClientOutput> {
|
||||
const file = await this._fetchFile(id)
|
||||
if (file.type !== 'normal') {
|
||||
throw new RuntimeError(`Attempted to download a file of type ${file.type}`)
|
||||
}
|
||||
|
||||
const exportType = await this._findExportType(file.mimeType)
|
||||
let output: DownloadFileDataClientOutput
|
||||
if (exportType) {
|
||||
// File size is unknown when exporting, download all data to buffer to know size
|
||||
const fileDownloadStream = await this._exportFileData(file, exportType)
|
||||
const buffer = await streamToBuffer(fileDownloadStream, MAX_EXPORT_FILE_SIZE_BYTES)
|
||||
output = {
|
||||
mimeType: exportType,
|
||||
dataSize: buffer.length,
|
||||
dataType: 'buffer',
|
||||
data: buffer,
|
||||
}
|
||||
} else {
|
||||
output = {
|
||||
mimeType: file.mimeType,
|
||||
dataSize: file.size,
|
||||
dataType: 'stream',
|
||||
data: await this._fetchFileData(file),
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
private _getRateLimitErrorHandler(): (error: unknown) => Promise<undefined> {
|
||||
return async (error: unknown) => {
|
||||
return handleRateLimitError(error, this._logger)
|
||||
}
|
||||
}
|
||||
|
||||
private _getNotFoundErrorHandler(): (error: unknown) => Promise<undefined> {
|
||||
return async (error: unknown) => {
|
||||
return handleNotFoundError(error, this._logger)
|
||||
}
|
||||
}
|
||||
|
||||
private async _tryWatchAllListableGenericFiles<T extends BaseGenericFileUnion>(
|
||||
listFn: ListFunction<T>
|
||||
): Promise<TryWatchAllOutput> {
|
||||
const fileChannels: FileChannel[] = []
|
||||
let hasError = false
|
||||
await listItemsAndProcess(listFn, async (item) => {
|
||||
const channel = await this._watch(item).catch(this._getRateLimitErrorHandler())
|
||||
if (channel) {
|
||||
fileChannels.push(channel)
|
||||
} else {
|
||||
hasError = true
|
||||
}
|
||||
})
|
||||
return {
|
||||
fileChannels,
|
||||
hasError,
|
||||
}
|
||||
}
|
||||
|
||||
public async watch(id: string): Promise<FileChannel> {
|
||||
const file = await this._fetchFile(id)
|
||||
return await this._watch(file)
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns Channel if successful, undefined if the subscription rate limit is exceeded
|
||||
*/
|
||||
public async tryWatch(id: string): Promise<FileChannel | undefined> {
|
||||
return await this.watch(id).catch(this._getRateLimitErrorHandler())
|
||||
}
|
||||
|
||||
private async _watch(file: BaseGenericFileUnion): Promise<FileChannel> {
|
||||
const absoluteExpirationTimeMs: number = Date.now() + MAX_RESOURCE_WATCH_EXPIRATION_DELAY_MS
|
||||
const { id: fileId, mimeType } = file
|
||||
const token = serializeToken(
|
||||
{
|
||||
fileId,
|
||||
fileType: getFileTypeFromMimeType(mimeType),
|
||||
},
|
||||
bp.secrets.WEBHOOK_SECRET
|
||||
)
|
||||
const response = await this._googleClient.files.watch({
|
||||
fileId,
|
||||
requestBody: {
|
||||
id: uuidv4(),
|
||||
type: 'web_hook',
|
||||
address: `${process.env.BP_WEBHOOK_URL}/${this._ctx.webhookId}`,
|
||||
token,
|
||||
expiration: absoluteExpirationTimeMs.toString(),
|
||||
},
|
||||
})
|
||||
const baseChannel = parseChannel(response.data)
|
||||
this._logger.forBot().debug(`Watching file '${file.name}' (${file.id}): channel ID = ${baseChannel.id}`)
|
||||
return {
|
||||
...baseChannel,
|
||||
fileId,
|
||||
}
|
||||
}
|
||||
|
||||
public async tryWatchAllFiles(): Promise<TryWatchAllOutput> {
|
||||
return await this._tryWatchAllListableGenericFiles(this._listBaseNormalFiles.bind(this))
|
||||
}
|
||||
|
||||
public async tryWatchAllFolders(): Promise<TryWatchAllOutput> {
|
||||
return await this._tryWatchAllListableGenericFiles(this._listBaseFolderFiles.bind(this))
|
||||
}
|
||||
|
||||
public async tryWatchAll(): Promise<TryWatchAllOutput> {
|
||||
const [filesResult, foldersResult] = await Promise.all([this.tryWatchAllFiles(), this.tryWatchAllFolders()])
|
||||
return {
|
||||
fileChannels: [...filesResult.fileChannels, ...foldersResult.fileChannels],
|
||||
hasError: filesResult.hasError || foldersResult.hasError,
|
||||
}
|
||||
}
|
||||
|
||||
public async unwatch(channels: FileChannel | FileChannel[]) {
|
||||
if (!Array.isArray(channels)) {
|
||||
channels = [channels]
|
||||
}
|
||||
const unwatchPromises = channels.map((channel) => {
|
||||
const fileName = this._filesCache.find(channel.fileId)?.name ?? '[unknown]'
|
||||
this._logger.forBot().debug(`Unwatching file ${fileName} (${channel.fileId}) with channel ID = ${channel.id}`)
|
||||
const { id, resourceId } = channel
|
||||
return this._googleClient.channels.stop({
|
||||
requestBody: {
|
||||
id,
|
||||
resourceId,
|
||||
},
|
||||
})
|
||||
})
|
||||
await Promise.all(unwatchPromises)
|
||||
}
|
||||
|
||||
public async tryUnwatch(channels: FileChannel | FileChannel[]) {
|
||||
await this.unwatch(channels).catch(this._getNotFoundErrorHandler())
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes internal fields and adds computed attributes
|
||||
*/
|
||||
private async _getCompleteFileFromBaseFile(file: BaseNormalFile): Promise<File> {
|
||||
return {
|
||||
...file,
|
||||
path: await this._getFilePath({ type: 'normal', ...file }),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes internal fields and adds computed attributes
|
||||
*/
|
||||
private async _getCompleteFolderFromBaseFolder(file: BaseFolderFile): Promise<Folder> {
|
||||
const { id, mimeType, name, parentId } = file
|
||||
return {
|
||||
id,
|
||||
mimeType,
|
||||
name,
|
||||
parentId,
|
||||
path: await this._getFilePath({ type: 'folder', ...file }),
|
||||
}
|
||||
}
|
||||
|
||||
private async _getCompleteFile(file: BaseDiscriminatedFile): Promise<GenericFile> {
|
||||
return {
|
||||
...file,
|
||||
path: await this._getFilePath(file),
|
||||
}
|
||||
}
|
||||
|
||||
private async _listBaseGenericFiles({
|
||||
nextToken,
|
||||
args,
|
||||
}: ListItemsInputWithArgs<{ searchQuery?: string }>): Promise<ListItemsOutput<BaseDiscriminatedFile>> {
|
||||
const searchQuery = args?.searchQuery
|
||||
const listResponse = await this._googleClient.files.list({
|
||||
corpora: 'user',
|
||||
fields: GOOGLE_API_FILELIST_FIELDS,
|
||||
q: (searchQuery ?? '') + (searchQuery?.length ? ' and ' : '') + 'trashed != true',
|
||||
pageToken: nextToken,
|
||||
pageSize: PAGE_SIZE,
|
||||
spaces: 'drive',
|
||||
...INCLUDE_FILES_FROM_ALL_DRIVES,
|
||||
})
|
||||
|
||||
const newNextToken = listResponse.data.nextPageToken ?? undefined
|
||||
const unvalidatedDriveFiles = listResponse.data.files
|
||||
if (!unvalidatedDriveFiles) {
|
||||
throw new RuntimeError('No files were returned by the API')
|
||||
}
|
||||
const newFiles = parseBaseGenerics(unvalidatedDriveFiles)
|
||||
for (const newFile of newFiles) {
|
||||
this._filesCache.set(newFile)
|
||||
}
|
||||
|
||||
return {
|
||||
items: newFiles,
|
||||
meta: {
|
||||
nextToken: newNextToken,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
private async _getOrFetchFile(id: string): Promise<BaseDiscriminatedFile> {
|
||||
let file = this._filesCache.find(id)
|
||||
if (!file) {
|
||||
file = await this._fetchFile(id)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
private async _fetchFile(id: string): Promise<BaseDiscriminatedFile> {
|
||||
const response = await this._googleClient.files.get({
|
||||
fileId: id,
|
||||
fields: GOOGLE_API_FILE_FIELDS,
|
||||
...INCLUDE_FILES_FROM_ALL_DRIVES,
|
||||
})
|
||||
const file = parseBaseGeneric(response.data)
|
||||
this._filesCache.set(file)
|
||||
return file
|
||||
}
|
||||
|
||||
private async _fetchFileData({ id: fileId }: BaseNormalFile): Promise<Readable> {
|
||||
const fileDownloadResponse = await this._googleClient.files.get(
|
||||
{
|
||||
fileId,
|
||||
alt: 'media',
|
||||
...INCLUDE_FILES_FROM_ALL_DRIVES,
|
||||
},
|
||||
{
|
||||
responseType: 'stream',
|
||||
}
|
||||
)
|
||||
return fileDownloadResponse.data
|
||||
}
|
||||
|
||||
private async _exportFileData({ id: fileId }: BaseNormalFile, mimeType: string): Promise<Readable> {
|
||||
const fileExportResponse = await this._googleClient.files.export(
|
||||
{
|
||||
fileId,
|
||||
mimeType,
|
||||
...INCLUDE_FILES_FROM_ALL_DRIVES,
|
||||
},
|
||||
{
|
||||
responseType: 'stream',
|
||||
}
|
||||
)
|
||||
return fileExportResponse.data
|
||||
}
|
||||
|
||||
private async _fetchExportFormatMap(): Promise<Record<string, string[]>> {
|
||||
const response = await this._googleClient.about.get({
|
||||
fields: GOOGLE_API_EXPORTFORMATS_FIELDS,
|
||||
})
|
||||
const { exportFormats } = response.data
|
||||
if (!exportFormats) {
|
||||
throw new RuntimeError('Export formats are missing in Schema$About from the API response')
|
||||
}
|
||||
return exportFormats
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns The export type to use, or undefined if the file cannot be exported
|
||||
*/
|
||||
private async _findExportType(originalContentType: string): Promise<string | undefined> {
|
||||
const exportFormatMap = await this._fetchExportFormatMap()
|
||||
const exportContentTypes = exportFormatMap[originalContentType]
|
||||
if (!exportContentTypes) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const indexableContentType = INDEXABLE_MIMETYPES.find((type) => exportContentTypes.includes(type))
|
||||
const defaultContentType = exportContentTypes[0]
|
||||
return indexableContentType ?? defaultContentType
|
||||
}
|
||||
|
||||
private _getFilePath = async (file: BaseDiscriminatedFile, pathAcc?: string[]): Promise<string[]> => {
|
||||
const path = [file.name, ...(pathAcc ?? [])]
|
||||
|
||||
if (!file.parentId) {
|
||||
return path
|
||||
}
|
||||
|
||||
try {
|
||||
const parent = await this._getOrFetchFile(file.parentId)
|
||||
|
||||
return await this._getFilePath(parent, path)
|
||||
} catch {
|
||||
return path
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { createAsyncFnWrapperWithErrorRedaction, defaultErrorRedactor } from '@botpress/common'
|
||||
import { z } from '@botpress/sdk'
|
||||
import { Common } from 'googleapis'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const wrapWithTryCatch = createAsyncFnWrapperWithErrorRedaction(defaultErrorRedactor)
|
||||
|
||||
const errorDetailSchema = z.object({
|
||||
domain: z.string(),
|
||||
reason: z.string(),
|
||||
message: z.string(),
|
||||
})
|
||||
type ErrorDetail = z.infer<typeof errorDetailSchema>
|
||||
// For some reason, the Google API typing for GaxiosError does not correspond
|
||||
// to the actual error object returned by the API. It is missing the `errors`
|
||||
// field which contains the actual error messages. This type is a workaround
|
||||
// to properly type the error object.
|
||||
export type AggregateGAxiosError = Common.GaxiosError & { errors: ErrorDetail[] }
|
||||
export const isGaxiosError = (error: unknown): error is AggregateGAxiosError => {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
'errors' in error &&
|
||||
Array.isArray(error['errors']) &&
|
||||
error['errors'].every((err) => errorDetailSchema.safeParse(err).success)
|
||||
)
|
||||
}
|
||||
|
||||
export type SubscriptionRateLimitError = AggregateGAxiosError // No discriminant
|
||||
const SUBSCRIPTION_RATE_LIMIT_ERR_REASON = 'subscriptionRateLimitExceeded'
|
||||
export const isSubscriptionRateLimitError = (error: unknown): error is SubscriptionRateLimitError => {
|
||||
if (!isGaxiosError(error)) {
|
||||
return false
|
||||
}
|
||||
return error.status === 403 && error.errors.some((err) => err.reason === SUBSCRIPTION_RATE_LIMIT_ERR_REASON)
|
||||
}
|
||||
|
||||
export type NotFoundError = AggregateGAxiosError // No discriminant
|
||||
export const isNotFoundError = (error: unknown): error is NotFoundError => {
|
||||
if (!isGaxiosError(error)) {
|
||||
return false
|
||||
}
|
||||
return error.status === 404
|
||||
}
|
||||
|
||||
export const handleRateLimitError = async (e: unknown, logger?: bp.Logger): Promise<undefined> => {
|
||||
if (!isSubscriptionRateLimitError(e)) {
|
||||
throw e
|
||||
}
|
||||
if (logger) {
|
||||
logger.forBot().warn('Subscription rate limit exceeded. Retry operation later.')
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export const handleNotFoundError = async (e: unknown, logger?: bp.Logger): Promise<undefined> => {
|
||||
if (!isNotFoundError(e)) {
|
||||
throw e
|
||||
}
|
||||
if (logger) {
|
||||
logger.forBot().error(e.errors.map((err) => err.message).join('\n'))
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { z } from '@botpress/sdk'
|
||||
import { fileChannelSchema } from './schemas'
|
||||
import { FileChannel } from './types'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const _fileChannelsSchema = z.record(z.string(), fileChannelSchema)
|
||||
type FileChannels = z.infer<typeof _fileChannelsSchema>
|
||||
type FileChannelsArray = FileChannel[]
|
||||
export class FileChannelsCache {
|
||||
private _channels: FileChannels
|
||||
private _dirty = false
|
||||
|
||||
public constructor(
|
||||
private _client: bp.Client,
|
||||
private _ctx: bp.Context
|
||||
) {
|
||||
this._channels = FileChannelsCache._getEmpty()
|
||||
}
|
||||
|
||||
public clear() {
|
||||
this._channels = FileChannelsCache._getEmpty()
|
||||
this._dirty = true
|
||||
}
|
||||
|
||||
public static async load({ client, ctx }: { client: bp.Client; ctx: bp.Context }) {
|
||||
const getStateResponse = await client.getOrSetState({
|
||||
id: ctx.integrationId,
|
||||
type: 'integration',
|
||||
name: 'filesChannelsCache',
|
||||
payload: {
|
||||
filesChannelsCache: this._getEmpty(),
|
||||
},
|
||||
})
|
||||
const fileChannels = new FileChannelsCache(client, ctx)
|
||||
fileChannels._channels = getStateResponse.state.payload.filesChannelsCache
|
||||
fileChannels._dirty = false
|
||||
return fileChannels
|
||||
}
|
||||
|
||||
public async save() {
|
||||
if (!this._dirty) {
|
||||
return
|
||||
}
|
||||
|
||||
this._dirty = false
|
||||
return await this._client.setState({
|
||||
id: this._ctx.integrationId,
|
||||
type: 'integration',
|
||||
name: 'filesChannelsCache',
|
||||
payload: {
|
||||
filesChannelsCache: this._channels,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
private static _getEmpty(): FileChannels {
|
||||
return {}
|
||||
}
|
||||
|
||||
public remove(fileId: string): FileChannel | undefined {
|
||||
const channel = this._channels[fileId]
|
||||
delete this._channels[fileId]
|
||||
this._dirty = true
|
||||
return channel
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns Channel that was replaced
|
||||
*/
|
||||
public set(channel: FileChannel): FileChannel | undefined {
|
||||
const oldChannel = this._channels[channel.fileId]
|
||||
this._channels[channel.fileId] = channel
|
||||
this._dirty = true
|
||||
return oldChannel
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns Channels that were replaced
|
||||
*/
|
||||
public setAll(channels: FileChannelsArray): FileChannelsArray {
|
||||
const newChannels = Object.fromEntries(channels.map((channel) => [channel.fileId, channel]))
|
||||
const oldChannels = { ...this._channels }
|
||||
this._channels = newChannels
|
||||
this._dirty = true
|
||||
return Object.values(oldChannels)
|
||||
}
|
||||
|
||||
public getAll(): FileChannelsArray {
|
||||
return Object.values(this._channels)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Client as DriveClient } from './client'
|
||||
import { FileChannelsCache } from './file-channels-cache'
|
||||
import { FilesCache } from './files-cache'
|
||||
import { BaseDiscriminatedFile, GenericFile } from './types'
|
||||
import { Client } from '.botpress'
|
||||
|
||||
export class FileEventHandler {
|
||||
public constructor(
|
||||
private _client: Client,
|
||||
private _driveClient: DriveClient,
|
||||
private _filesCache: FilesCache,
|
||||
private _fileChannelsCache: FileChannelsCache
|
||||
) {}
|
||||
|
||||
public async handleFileCreated(file: GenericFile) {
|
||||
this._filesCache.set(file) // GenericFile is compatible with BaseDiscriminatedFile
|
||||
const channel = await this._driveClient.tryWatch(file.id)
|
||||
if (channel) {
|
||||
this._fileChannelsCache.set(channel)
|
||||
}
|
||||
if (file.type === 'normal') {
|
||||
await this._client.createEvent({
|
||||
type: 'fileCreated',
|
||||
payload: file,
|
||||
})
|
||||
} else if (file.type === 'folder') {
|
||||
await this._client.createEvent({
|
||||
type: 'folderCreated',
|
||||
payload: file,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Work with BaseDiscriminatedFile, at this point the only file info available is in the cache
|
||||
public async handleFileDeleted(baseFile: BaseDiscriminatedFile) {
|
||||
this._fileChannelsCache.remove(baseFile.id) // No need to unwatch as resource is already deleted
|
||||
this._filesCache.remove(baseFile.id)
|
||||
|
||||
if (baseFile.type === 'normal') {
|
||||
await this._client.createEvent({
|
||||
type: 'fileDeleted',
|
||||
payload: { id: baseFile.id },
|
||||
})
|
||||
} else if (baseFile.type === 'folder') {
|
||||
await this._client.createEvent({
|
||||
type: 'folderDeleted',
|
||||
payload: { id: baseFile.id },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { z } from '@botpress/sdk'
|
||||
import * as jwt from 'jsonwebtoken'
|
||||
import { fileTypesUnionSchema } from './schemas'
|
||||
|
||||
const tokenSchema = z.object({
|
||||
fileId: z.string().min(1),
|
||||
fileType: fileTypesUnionSchema,
|
||||
})
|
||||
export type Token = z.infer<typeof tokenSchema>
|
||||
|
||||
export const serializeToken = (token: Token, secret: string): string => {
|
||||
return jwt.sign(token, secret, {
|
||||
noTimestamp: true,
|
||||
})
|
||||
}
|
||||
|
||||
export const deserializeToken = (serializedToken: string, secret: string): Token | undefined => {
|
||||
let object: any
|
||||
try {
|
||||
object = jwt.verify(serializedToken, secret)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
const tokenParseResult = tokenSchema.safeParse(object)
|
||||
if (!tokenParseResult.success) {
|
||||
return undefined
|
||||
}
|
||||
return tokenParseResult.data
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import axios, { AxiosError } from 'axios'
|
||||
import type { Client as DriveClient } from './client'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const downloadToBotpress = async ({
|
||||
client,
|
||||
driveClient,
|
||||
botpressFileKey,
|
||||
googleDriveFileId,
|
||||
indexFile,
|
||||
}: {
|
||||
client: bp.Client
|
||||
driveClient: DriveClient
|
||||
googleDriveFileId: string
|
||||
botpressFileKey: string
|
||||
indexFile?: boolean
|
||||
}) => {
|
||||
const content = await driveClient.downloadFileData({ id: googleDriveFileId })
|
||||
const { mimeType, dataSize, dataType, data } = content
|
||||
const uploadParams = {
|
||||
key: botpressFileKey,
|
||||
contentType: mimeType,
|
||||
index: indexFile ?? false,
|
||||
}
|
||||
let botpressFileId: string
|
||||
if (dataType === 'stream') {
|
||||
const upsertResponse = await client.upsertFile({
|
||||
...uploadParams,
|
||||
size: dataSize,
|
||||
})
|
||||
botpressFileId = upsertResponse.file.id
|
||||
await axios
|
||||
.put(upsertResponse.file.uploadUrl, data, {
|
||||
maxBodyLength: dataSize,
|
||||
headers: {
|
||||
'Content-Type': mimeType,
|
||||
'Content-Length': dataSize,
|
||||
},
|
||||
})
|
||||
.catch((reason: AxiosError) => {
|
||||
throw new sdk.RuntimeError(`Error uploading file stream: ${reason}`)
|
||||
})
|
||||
} else {
|
||||
const uploadResponse = await client.uploadFile({
|
||||
...uploadParams,
|
||||
content: data,
|
||||
})
|
||||
botpressFileId = uploadResponse.file.id
|
||||
}
|
||||
|
||||
return { botpressFileId }
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { RuntimeError, z } from '@botpress/sdk'
|
||||
import { baseDiscriminatedFileSchema } from './schemas'
|
||||
import { BaseFolderFile, BaseDiscriminatedFile, BaseNormalFile } from './types'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const _filesMapSchema = z.record(z.string(), baseDiscriminatedFileSchema)
|
||||
type FilesMap = z.infer<typeof _filesMapSchema>
|
||||
|
||||
export class FilesCache {
|
||||
private _map: FilesMap
|
||||
public constructor(
|
||||
private _client: bp.Client,
|
||||
private _ctx: bp.Context
|
||||
) {
|
||||
this._map = {}
|
||||
}
|
||||
|
||||
public clear() {
|
||||
this._map = {}
|
||||
}
|
||||
|
||||
public static async load({ client, ctx }: { client: bp.Client; ctx: bp.Context }): Promise<FilesCache> {
|
||||
const getStateResponse = await client.getOrSetState({
|
||||
id: ctx.integrationId,
|
||||
type: 'integration',
|
||||
name: 'filesCache',
|
||||
payload: {
|
||||
filesCache: FilesCache._getEmpty(),
|
||||
},
|
||||
})
|
||||
const cache = new FilesCache(client, ctx)
|
||||
cache._map = getStateResponse.state.payload.filesCache
|
||||
return cache
|
||||
}
|
||||
|
||||
public async save() {
|
||||
await this._client.setState({
|
||||
id: this._ctx.integrationId,
|
||||
type: 'integration',
|
||||
name: 'filesCache',
|
||||
payload: {
|
||||
filesCache: this._map,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
private static _getEmpty(): FilesMap {
|
||||
return {}
|
||||
}
|
||||
|
||||
public find(id: string): BaseDiscriminatedFile | undefined {
|
||||
return this._map[id]
|
||||
}
|
||||
|
||||
public set(file: BaseDiscriminatedFile) {
|
||||
this._map[file.id] = file
|
||||
}
|
||||
|
||||
public remove(id: string) {
|
||||
delete this._map[id]
|
||||
}
|
||||
|
||||
private _getGenericFile(id: string): BaseDiscriminatedFile {
|
||||
const file = this._map[id]
|
||||
if (!file) {
|
||||
throw new RuntimeError(`Couldn't get file from files map with ID=${id}`)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
public get(id: string): BaseDiscriminatedFile {
|
||||
return this._getGenericFile(id)
|
||||
}
|
||||
|
||||
public getAll(filterFn?: (file: BaseDiscriminatedFile) => boolean): BaseDiscriminatedFile[] {
|
||||
const allFiles = Object.values(this._map)
|
||||
return filterFn ? allFiles.filter(filterFn) : allFiles
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws {RuntimeError} ID must correspond to a file compatible with BaseNormalFile
|
||||
*/
|
||||
public getFile(id: string): BaseNormalFile {
|
||||
const file = this._getGenericFile(id)
|
||||
if (file.type !== 'normal') {
|
||||
throw new RuntimeError(`Attempted to get file with ID=${file.id} as a normal file but is type ${file.type}`)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws {RuntimeError} ID must correspond to a file compatible with BaseFolderFile
|
||||
*/
|
||||
public getFolder(id: string): BaseFolderFile {
|
||||
const file = this._getGenericFile(id)
|
||||
if (file.type !== 'folder') {
|
||||
throw new RuntimeError(`Attempted to get file with ID=${file.id} as a folder file but is type ${file.type}`)
|
||||
}
|
||||
return file
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { filesReadonlyListItemsInFolder } from './list-items-in-folder'
|
||||
import { filesReadonlyTransferFileToBotpress } from './transfer-file-to-botpress'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const filesReadonlyActions = {
|
||||
filesReadonlyListItemsInFolder,
|
||||
filesReadonlyTransferFileToBotpress,
|
||||
} as const satisfies Pick<
|
||||
bp.IntegrationProps['actions'],
|
||||
'filesReadonlyListItemsInFolder' | 'filesReadonlyTransferFileToBotpress'
|
||||
>
|
||||
@@ -0,0 +1,180 @@
|
||||
import { APP_GOOGLE_FOLDER_MIMETYPE, APP_GOOGLE_SHORTCUT_MIMETYPE } from 'src/mime-types'
|
||||
import { Client as DriveClient } from '../../client'
|
||||
import { GoogleDriveNodeTree, type GoogleDriveNode } from '../google-drive-file-tree'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const GOOGLE_DRIVE_TREE_FILE_KEY = 'google-drive-file-tree.json'
|
||||
const SYNTHETIC_NEXT_TOKEN_PREFIX = 'synthetic-tree-index:'
|
||||
const SYNTHETIC_BATCH_SIZE = 100
|
||||
|
||||
type FilesReadonlyListItemsInFolderReturn = bp.actions.Actions['filesReadonlyListItemsInFolder']['output']
|
||||
type FilesReadonlyListItemsInFolderProps = bp.ActionProps['filesReadonlyListItemsInFolder']
|
||||
|
||||
export const filesReadonlyListItemsInFolder: bp.IntegrationProps['actions']['filesReadonlyListItemsInFolder'] = async (
|
||||
props
|
||||
) => (props.input.folderId ? await _listItemsInSpecificFolder(props) : await _listItemsInRootFolder(props))
|
||||
|
||||
const _listItemsInSpecificFolder = async (
|
||||
props: FilesReadonlyListItemsInFolderProps
|
||||
): Promise<FilesReadonlyListItemsInFolderReturn> => {
|
||||
const nodeTree = await _loadNodeTree(props.client)
|
||||
const node = nodeTree.getNodeById(props.input.folderId!)
|
||||
|
||||
return _enumerateNodeChildren({ node, nextToken: props.input.nextToken })
|
||||
}
|
||||
|
||||
const _enumerateNodeChildren = ({ node, nextToken }: { node?: GoogleDriveNode; nextToken?: string }) => {
|
||||
const nodeChildren = node?.children ?? []
|
||||
const batchChildIndex = parseInt(nextToken?.slice(SYNTHETIC_NEXT_TOKEN_PREFIX.length) ?? '0', 10)
|
||||
|
||||
if (batchChildIndex >= nodeChildren.length) {
|
||||
return { items: [], meta: { nextToken: undefined } }
|
||||
}
|
||||
|
||||
const nextBatchIndex = batchChildIndex + SYNTHETIC_BATCH_SIZE
|
||||
const currentBatch = nodeChildren.slice(batchChildIndex, nextBatchIndex)
|
||||
const mappedBatchItems = currentBatch.map(_mapNodeToBatchItem)
|
||||
|
||||
return {
|
||||
items: mappedBatchItems,
|
||||
meta: {
|
||||
nextToken: nextBatchIndex < nodeChildren.length ? _getNextTokenForChildIndex(nextBatchIndex) : undefined,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const _mapNodeToBatchItem = (item: GoogleDriveNode) =>
|
||||
({
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
parentId: item.parents?.[0] ?? 'root',
|
||||
...(item.mimeType === APP_GOOGLE_FOLDER_MIMETYPE
|
||||
? {
|
||||
type: 'folder' as const,
|
||||
}
|
||||
: {
|
||||
type: 'file' as const,
|
||||
sizeInBytes: parseInt(item.size ?? '0', 10),
|
||||
lastModifiedDate: item.modifiedTime,
|
||||
contentHash: item.sha256Checksum ?? item.md5Checksum ?? item.version ?? undefined,
|
||||
}),
|
||||
}) as const
|
||||
|
||||
const _listItemsInRootFolder = async (
|
||||
props: FilesReadonlyListItemsInFolderProps
|
||||
): Promise<FilesReadonlyListItemsInFolderReturn> => {
|
||||
if (props.input.nextToken?.startsWith(SYNTHETIC_NEXT_TOKEN_PREFIX)) {
|
||||
return await _enumerateNodeTreeItems(props.client, props.input.nextToken)
|
||||
}
|
||||
|
||||
const driveClient = await DriveClient.create(props)
|
||||
const filterString = _buildFilterString(props.input.filters)
|
||||
|
||||
return props.input.nextToken
|
||||
? await _enumerateDriveItemsNextPage(driveClient, props.client, props.input.nextToken, filterString)
|
||||
: await _enumerateDriveItemsFirstPage(driveClient, props.client, filterString)
|
||||
}
|
||||
|
||||
const _enumerateNodeTreeItems = async (
|
||||
client: FilesReadonlyListItemsInFolderProps['client'],
|
||||
nextToken: string
|
||||
): Promise<FilesReadonlyListItemsInFolderReturn> => {
|
||||
const nodeTree = await _loadNodeTree(client)
|
||||
const rootNode = nodeTree.getRootNode()
|
||||
|
||||
return _enumerateNodeChildren({ node: rootNode, nextToken })
|
||||
}
|
||||
|
||||
const _loadNodeTree = async (client: FilesReadonlyListItemsInFolderProps['client']): Promise<GoogleDriveNodeTree> => {
|
||||
const { file } = await client.getFile({
|
||||
id: GOOGLE_DRIVE_TREE_FILE_KEY,
|
||||
})
|
||||
return GoogleDriveNodeTree.fromJSON(await fetch(file.url).then((res) => res.text()))
|
||||
}
|
||||
|
||||
const _getNextTokenForChildIndex = (childIndex: number): string => `${SYNTHETIC_NEXT_TOKEN_PREFIX}${childIndex}`
|
||||
|
||||
const _buildFilterString = (filters: FilesReadonlyListItemsInFolderProps['input']['filters']): string => {
|
||||
const query: string[] = []
|
||||
|
||||
if (filters?.itemType === 'file') {
|
||||
query.push(`mimeType != '${APP_GOOGLE_FOLDER_MIMETYPE}'`, `mimeType != '${APP_GOOGLE_SHORTCUT_MIMETYPE}'`)
|
||||
} else if (filters?.itemType === 'folder') {
|
||||
query.push(`mimeType = '${APP_GOOGLE_FOLDER_MIMETYPE}'`)
|
||||
} else {
|
||||
query.push(`mimeType != '${APP_GOOGLE_SHORTCUT_MIMETYPE}'`)
|
||||
}
|
||||
|
||||
if (filters?.maxSizeInBytes) {
|
||||
query.push(`size <= ${filters.maxSizeInBytes}`)
|
||||
}
|
||||
|
||||
if (filters?.modifiedAfter) {
|
||||
query.push(`modifiedTime > '${filters.modifiedAfter}'`)
|
||||
}
|
||||
|
||||
return query.join(' and ')
|
||||
}
|
||||
|
||||
const _enumerateDriveItemsFirstPage = async (
|
||||
driveClient: DriveClient,
|
||||
client: bp.Client,
|
||||
filters: string
|
||||
): Promise<FilesReadonlyListItemsInFolderReturn> => {
|
||||
const rootFolderId = await driveClient.getRootFolderId()
|
||||
const nodeTree = new GoogleDriveNodeTree({ rootFolderId })
|
||||
|
||||
return await _enumerateGoogleDriveAndBuildNodeTree({
|
||||
driveClient,
|
||||
client,
|
||||
nextToken: undefined,
|
||||
filters,
|
||||
nodeTree,
|
||||
})
|
||||
}
|
||||
|
||||
const _enumerateDriveItemsNextPage = async (
|
||||
driveClient: DriveClient,
|
||||
client: FilesReadonlyListItemsInFolderProps['client'],
|
||||
nextToken: string,
|
||||
filters: string
|
||||
): Promise<FilesReadonlyListItemsInFolderReturn> => {
|
||||
const nodeTree = await _loadNodeTree(client)
|
||||
|
||||
return await _enumerateGoogleDriveAndBuildNodeTree({ driveClient, client, nextToken, filters, nodeTree })
|
||||
}
|
||||
|
||||
const _enumerateGoogleDriveAndBuildNodeTree = async ({
|
||||
driveClient,
|
||||
client,
|
||||
nextToken,
|
||||
filters,
|
||||
nodeTree,
|
||||
}: {
|
||||
driveClient: DriveClient
|
||||
client: FilesReadonlyListItemsInFolderProps['client']
|
||||
nextToken: string | undefined
|
||||
filters: string
|
||||
nodeTree: GoogleDriveNodeTree
|
||||
}): Promise<FilesReadonlyListItemsInFolderReturn> => {
|
||||
const { files, nextToken: newNextToken } = await driveClient.getChildrenSubset({
|
||||
folderId: 'root',
|
||||
extraQuery: filters,
|
||||
nextToken,
|
||||
})
|
||||
|
||||
for (const item of files ?? []) {
|
||||
nodeTree.upsertNode(item as GoogleDriveNode)
|
||||
}
|
||||
|
||||
await _saveNodeTree(client, newNextToken ? nodeTree : nodeTree.removeAllEmptyFoldersRecursively())
|
||||
|
||||
return { items: [], meta: { nextToken: newNextToken ?? _getNextTokenForChildIndex(0) } }
|
||||
}
|
||||
|
||||
const _saveNodeTree = async (client: bp.Client, nodeTree: GoogleDriveNodeTree): Promise<void> => {
|
||||
await client.uploadFile({
|
||||
key: GOOGLE_DRIVE_TREE_FILE_KEY,
|
||||
content: nodeTree.toJSON(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { downloadToBotpress } from 'src/files-api-utils'
|
||||
import { Client as DriveClient } from '../../client'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const filesReadonlyTransferFileToBotpress: bp.IntegrationProps['actions']['filesReadonlyTransferFileToBotpress'] =
|
||||
async (props) => {
|
||||
const driveClient = await DriveClient.create(props)
|
||||
|
||||
const { botpressFileId } = await downloadToBotpress({
|
||||
botpressFileKey: props.input.fileKey,
|
||||
googleDriveFileId: props.input.file.id,
|
||||
client: props.client,
|
||||
driveClient,
|
||||
indexFile: props.input.shouldIndex,
|
||||
})
|
||||
|
||||
return { botpressFileId }
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
import { it, expect, describe } from 'vitest'
|
||||
import {
|
||||
GoogleDriveNodeTree,
|
||||
SHARED_DRIVES_ID,
|
||||
SHARED_WITH_ME_ID,
|
||||
type GoogleDriveNode,
|
||||
} from './google-drive-file-tree'
|
||||
import { APP_GOOGLE_FOLDER_MIMETYPE } from '../mime-types'
|
||||
|
||||
const DUMMY_ROOT_FOLDER_ID = '0AMKYlhzXYUfqUk9PVA'
|
||||
const DUMMY_SHARED_DRIVE_ID = '0AJcxkTsZHSqGUk9PVA'
|
||||
|
||||
const _createMockTree = () => new GoogleDriveNodeTree({ rootFolderId: DUMMY_ROOT_FOLDER_ID })
|
||||
|
||||
const _createMockFile = (overrides: Partial<GoogleDriveNode> = {}): GoogleDriveNode => ({
|
||||
id: 'example123',
|
||||
name: 'example.txt',
|
||||
mimeType: 'text/plain',
|
||||
trashed: false,
|
||||
version: '1',
|
||||
modifiedTime: '2025-01-01T00:00:00.000Z',
|
||||
shared: false,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const _createMockFolder = (overrides: Partial<GoogleDriveNode> = {}): GoogleDriveNode => ({
|
||||
id: 'folder123',
|
||||
name: 'Example Folder',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
trashed: false,
|
||||
version: '1',
|
||||
modifiedTime: '2025-01-01T00:00:00.000Z',
|
||||
shared: false,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
describe.concurrent('GoogleDriveFileTree', () => {
|
||||
it('should create a root node with the correct properties', () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
|
||||
// Act
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
id: DUMMY_ROOT_FOLDER_ID,
|
||||
name: 'My Drive',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
})
|
||||
})
|
||||
|
||||
it("should correctly add a folder that's a direct child of 'My Drive'", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const folder = _createMockFolder({
|
||||
parents: [DUMMY_ROOT_FOLDER_ID],
|
||||
id: '1subfolder123',
|
||||
name: 'Work Documents',
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(folder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root.children).toHaveLength(1)
|
||||
expect(root).toMatchObject({
|
||||
children: [folder],
|
||||
})
|
||||
})
|
||||
|
||||
it("should correctly add a folder that's a nested child of 'My Drive'", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const parentId = '1parent123'
|
||||
const nestedFolder = _createMockFolder({
|
||||
parents: [parentId],
|
||||
id: '1nested123',
|
||||
name: 'Nested Folder',
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(nestedFolder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: parentId,
|
||||
name: `[${parentId}]`,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [nestedFolder],
|
||||
}),
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it("should correctly add a folder that's a direct child of a shared drive", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const sharedDriveFolder = _createMockFolder({
|
||||
parents: [DUMMY_SHARED_DRIVE_ID],
|
||||
id: '1shared123',
|
||||
name: 'Team Documents',
|
||||
driveId: DUMMY_SHARED_DRIVE_ID,
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(sharedDriveFolder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: SHARED_DRIVES_ID,
|
||||
name: 'Shared drives',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: DUMMY_SHARED_DRIVE_ID,
|
||||
name: `[${DUMMY_SHARED_DRIVE_ID}]`,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [expect.objectContaining(sharedDriveFolder)],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it("should correctly add a folder that's a nested child of a shared drive", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const parentId = '1sharedparent123'
|
||||
const sharedDriveFolder = _createMockFolder({
|
||||
parents: [parentId],
|
||||
id: '1sharednested123',
|
||||
name: 'Project Files',
|
||||
driveId: DUMMY_SHARED_DRIVE_ID,
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(sharedDriveFolder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: SHARED_DRIVES_ID,
|
||||
name: 'Shared drives',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: DUMMY_SHARED_DRIVE_ID,
|
||||
name: `[${DUMMY_SHARED_DRIVE_ID}]`,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: parentId,
|
||||
name: `[${parentId}]`,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [expect.objectContaining(sharedDriveFolder)],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it("should correctly add a folder that's a direct child of 'Shared with me'", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const sharedWithMeFolder = _createMockFolder({
|
||||
id: '1sharedwithme123',
|
||||
name: 'Collaboration Folder',
|
||||
sharedWithMeTime: '2024-11-20T20:32:17.302Z',
|
||||
shared: true,
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(sharedWithMeFolder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: SHARED_WITH_ME_ID,
|
||||
name: 'Shared with me',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [sharedWithMeFolder],
|
||||
}),
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it("should correctly add a folder that's a nested child of 'Shared with me'", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const sharedWithMeParentFolder = _createMockFolder({
|
||||
id: '1sharedparent123',
|
||||
name: 'Main Project',
|
||||
sharedWithMeTime: '2024-11-20T20:32:17.302Z',
|
||||
shared: true,
|
||||
})
|
||||
const nestedSharedWithMeFolder = _createMockFolder({
|
||||
parents: [sharedWithMeParentFolder.id],
|
||||
id: '1sharednested123',
|
||||
name: 'Subfolder',
|
||||
shared: true,
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(sharedWithMeParentFolder)
|
||||
tree.upsertNode(nestedSharedWithMeFolder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: SHARED_WITH_ME_ID,
|
||||
name: 'Shared with me',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
...sharedWithMeParentFolder,
|
||||
children: [expect.objectContaining(nestedSharedWithMeFolder)],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
describe.concurrent('automatic rebalancing', () => {
|
||||
it("should correctly rebalance to 'Shared with me' when previously under 'My Drive'", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const sharedWithMeParentFolder = _createMockFolder({
|
||||
id: '1rebalanceparent123',
|
||||
name: 'Rebalance Test',
|
||||
sharedWithMeTime: '2024-11-20T20:32:17.302Z',
|
||||
shared: true,
|
||||
})
|
||||
const nestedSharedWithMeFolder = _createMockFolder({
|
||||
parents: [sharedWithMeParentFolder.id],
|
||||
id: '1rebalancenested123',
|
||||
name: 'Nested Test',
|
||||
shared: true,
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(nestedSharedWithMeFolder)
|
||||
tree.upsertNode(sharedWithMeParentFolder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: SHARED_WITH_ME_ID,
|
||||
name: 'Shared with me',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
...sharedWithMeParentFolder,
|
||||
children: [expect.objectContaining(nestedSharedWithMeFolder)],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it("should correctly rebalance from a stub subfolder under 'Shared drives' to a known folder", () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const parentId = '1stubparent123'
|
||||
const sharedDriveSubItem = _createMockFile({
|
||||
parents: [parentId],
|
||||
id: '1stubitem123',
|
||||
name: 'image.jpg',
|
||||
mimeType: 'image/jpeg',
|
||||
driveId: DUMMY_SHARED_DRIVE_ID,
|
||||
md5Checksum: 'abc123def456',
|
||||
sha256Checksum: 'def456abc123',
|
||||
size: '2995372',
|
||||
})
|
||||
const sharedDriveParentFolder = _createMockFolder({
|
||||
parents: [DUMMY_SHARED_DRIVE_ID],
|
||||
id: parentId,
|
||||
name: 'Media Files',
|
||||
driveId: DUMMY_SHARED_DRIVE_ID,
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(sharedDriveSubItem)
|
||||
tree.upsertNode(sharedDriveParentFolder)
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root).toMatchObject({
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: SHARED_DRIVES_ID,
|
||||
name: 'Shared drives',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
id: DUMMY_SHARED_DRIVE_ID,
|
||||
name: `[${DUMMY_SHARED_DRIVE_ID}]`,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
children: [
|
||||
expect.objectContaining({
|
||||
...sharedDriveParentFolder,
|
||||
children: [expect.objectContaining(sharedDriveSubItem)],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe.concurrent('empty folder pruning', () => {
|
||||
it('should remove nested empty folders when calling removeAllEmptyFoldersRecursively', () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
const parentEmptyFolder = _createMockFolder({
|
||||
parents: [DUMMY_ROOT_FOLDER_ID],
|
||||
id: '1parentempty123',
|
||||
name: 'Parent Empty Folder',
|
||||
})
|
||||
const nestedEmptyFolder = _createMockFolder({
|
||||
parents: [parentEmptyFolder.id],
|
||||
id: '1nestedempty123',
|
||||
name: 'Nested Empty Folder',
|
||||
})
|
||||
const nonEmptyFolder = _createMockFolder({
|
||||
parents: [DUMMY_ROOT_FOLDER_ID],
|
||||
id: '1nonempty123',
|
||||
name: 'Non-Empty Folder',
|
||||
})
|
||||
const fileInNonEmptyFolder = _createMockFile({
|
||||
parents: [nonEmptyFolder.id],
|
||||
id: '1file123',
|
||||
name: 'file.txt',
|
||||
})
|
||||
|
||||
// Act
|
||||
tree.upsertNode(parentEmptyFolder)
|
||||
tree.upsertNode(nestedEmptyFolder)
|
||||
tree.upsertNode(nonEmptyFolder)
|
||||
tree.upsertNode(fileInNonEmptyFolder)
|
||||
|
||||
// Pre-Assert
|
||||
expect(tree.getRootNode().children).toHaveLength(2)
|
||||
|
||||
tree.removeAllEmptyFoldersRecursively()
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root.children).toHaveLength(1)
|
||||
expect(root).toMatchObject({
|
||||
children: [expect.objectContaining(nonEmptyFolder)],
|
||||
})
|
||||
})
|
||||
|
||||
it('should not remove the root folder when calling removeAllEmptyFoldersRecursively', () => {
|
||||
// Arrange
|
||||
const tree = _createMockTree()
|
||||
|
||||
// Act
|
||||
tree.removeAllEmptyFoldersRecursively()
|
||||
const root = tree.getRootNode()
|
||||
|
||||
// Assert
|
||||
expect(root.id).toBe(DUMMY_ROOT_FOLDER_ID)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,303 @@
|
||||
import { APP_GOOGLE_FOLDER_MIMETYPE } from '../mime-types'
|
||||
|
||||
export type GoogleDriveNode = {
|
||||
id: string
|
||||
name: string
|
||||
mimeType: string
|
||||
parents?: string[]
|
||||
children?: GoogleDriveNode[]
|
||||
trashed?: boolean
|
||||
version?: string
|
||||
modifiedTime?: string
|
||||
shared?: boolean
|
||||
driveId?: string
|
||||
size?: string
|
||||
md5Checksum?: string
|
||||
sha256Checksum?: string
|
||||
sharedWithMeTime?: string
|
||||
}
|
||||
|
||||
export const SHARED_WITH_ME_ID = 'sharedWithMe'
|
||||
export const SHARED_DRIVES_ID = 'sharedDrives'
|
||||
|
||||
export class GoogleDriveNodeTree {
|
||||
private readonly _nodeByIdMap = new Map<string, GoogleDriveNode>()
|
||||
private readonly _childIdsByParentIdMap = new Map<string, Set<string>>()
|
||||
private readonly _rootFolderId: string
|
||||
|
||||
public constructor({ rootFolderId }: { rootFolderId: string }) {
|
||||
this._rootFolderId = rootFolderId
|
||||
this._createMyDriveRootNode()
|
||||
}
|
||||
|
||||
public getRootNode(): GoogleDriveNode {
|
||||
return this._buildNodeWithAllDescendants(this._rootFolderId)
|
||||
}
|
||||
|
||||
public getNodeById(nodeId: string): GoogleDriveNode | undefined {
|
||||
return this._nodeByIdMap.get(nodeId)
|
||||
}
|
||||
|
||||
public upsertNode(nodeToInsert: GoogleDriveNode): this {
|
||||
if (nodeToInsert.id !== this._rootFolderId) {
|
||||
const existingNodeWithSameId = this._nodeByIdMap.get(nodeToInsert.id)
|
||||
const existingChildIdsForThisNode = new Set(this._childIdsByParentIdMap.get(nodeToInsert.id))
|
||||
|
||||
this._removeNodeFromItsPreviousParent(existingNodeWithSameId)
|
||||
this._storeNodeInIdMap(nodeToInsert)
|
||||
this._addNodeToItsNewParent(nodeToInsert)
|
||||
this._preserveExistingChildrenForFolders(nodeToInsert, existingChildIdsForThisNode)
|
||||
}
|
||||
|
||||
this._upsertChildrenForNode(nodeToInsert)
|
||||
return this
|
||||
}
|
||||
|
||||
public removeAllEmptyFoldersRecursively(): this {
|
||||
const emptyFolderIds = this._findAllEmptyFolderIds()
|
||||
this._removeEmptyFoldersFromTheirParents(emptyFolderIds)
|
||||
this._deleteEmptyFoldersFromMaps(emptyFolderIds)
|
||||
return this
|
||||
}
|
||||
|
||||
private _upsertChildrenForNode(nodeToInsert: GoogleDriveNode): void {
|
||||
for (const child of nodeToInsert.children ?? []) {
|
||||
this.upsertNode(child)
|
||||
}
|
||||
}
|
||||
|
||||
private _createMyDriveRootNode(): void {
|
||||
this._nodeByIdMap.set(this._rootFolderId, {
|
||||
id: this._rootFolderId,
|
||||
name: 'My Drive',
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
})
|
||||
this._childIdsByParentIdMap.set(this._rootFolderId, new Set())
|
||||
}
|
||||
|
||||
private _removeNodeFromItsPreviousParent(existingNodeWithSameId: GoogleDriveNode | undefined): void {
|
||||
if (!existingNodeWithSameId) return
|
||||
const previousParentId = this._determineEffectiveParentId(existingNodeWithSameId)
|
||||
this._childIdsByParentIdMap.get(previousParentId)?.delete(existingNodeWithSameId.id)
|
||||
}
|
||||
|
||||
private _storeNodeInIdMap(nodeToStore: GoogleDriveNode): void {
|
||||
this._nodeByIdMap.set(nodeToStore.id, { ...nodeToStore })
|
||||
}
|
||||
|
||||
private _addNodeToItsNewParent(nodeToAdd: GoogleDriveNode): void {
|
||||
const newParentId = this._determineEffectiveParentId(nodeToAdd)
|
||||
this._ensureParentNodeExists(newParentId, nodeToAdd)
|
||||
this._ensureChildSetExistsForParent(newParentId)
|
||||
this._childIdsByParentIdMap.get(newParentId)!.add(nodeToAdd.id)
|
||||
}
|
||||
|
||||
private _preserveExistingChildrenForFolders(nodeToCheck: GoogleDriveNode, existingChildIds: Set<string>): void {
|
||||
if (nodeToCheck.mimeType !== APP_GOOGLE_FOLDER_MIMETYPE) return
|
||||
|
||||
this._ensureChildSetExistsForParent(nodeToCheck.id)
|
||||
if (existingChildIds.size > 0) {
|
||||
this._childIdsByParentIdMap.set(nodeToCheck.id, existingChildIds)
|
||||
}
|
||||
}
|
||||
|
||||
private _ensureChildSetExistsForParent(parentNodeId: string): void {
|
||||
if (!this._childIdsByParentIdMap.has(parentNodeId)) {
|
||||
this._childIdsByParentIdMap.set(parentNodeId, new Set())
|
||||
}
|
||||
}
|
||||
|
||||
private _findAllEmptyFolderIds(): Set<string> {
|
||||
const emptyFolderIds = new Set<string>()
|
||||
let foundNewEmptyFoldersInThisIteration = true
|
||||
|
||||
while (foundNewEmptyFoldersInThisIteration) {
|
||||
foundNewEmptyFoldersInThisIteration = false
|
||||
foundNewEmptyFoldersInThisIteration = this._findEmptyFoldersInCurrentIteration(emptyFolderIds)
|
||||
this._removeEmptyFoldersFromTheirParents(emptyFolderIds)
|
||||
}
|
||||
|
||||
return emptyFolderIds
|
||||
}
|
||||
|
||||
private _findEmptyFoldersInCurrentIteration(alreadyFoundEmptyFolderIds: Set<string>): boolean {
|
||||
let foundNewEmptyFolderInThisIteration = false
|
||||
|
||||
for (const [potentialParentId, childIdSet] of this._childIdsByParentIdMap.entries()) {
|
||||
const potentialParentNode = this._nodeByIdMap.get(potentialParentId)
|
||||
if (!potentialParentNode || alreadyFoundEmptyFolderIds.has(potentialParentId)) continue
|
||||
|
||||
if (this._isFolderEmptyOfChildren(potentialParentNode, childIdSet)) {
|
||||
alreadyFoundEmptyFolderIds.add(potentialParentId)
|
||||
foundNewEmptyFolderInThisIteration = true
|
||||
}
|
||||
}
|
||||
|
||||
return foundNewEmptyFolderInThisIteration
|
||||
}
|
||||
|
||||
private _isFolderEmptyOfChildren(nodeToCheck: GoogleDriveNode, childIdSet: Set<string>): boolean {
|
||||
return nodeToCheck.mimeType === APP_GOOGLE_FOLDER_MIMETYPE && childIdSet.size === 0
|
||||
}
|
||||
|
||||
private _removeEmptyFoldersFromTheirParents(emptyFolderIds: Set<string>): void {
|
||||
for (const emptyFolderId of emptyFolderIds) {
|
||||
if (emptyFolderId === this._rootFolderId) continue
|
||||
const emptyFolderNode = this._nodeByIdMap.get(emptyFolderId)
|
||||
if (!emptyFolderNode) continue
|
||||
|
||||
const parentIdOfEmptyFolder = this._determineEffectiveParentId(emptyFolderNode)
|
||||
this._childIdsByParentIdMap.get(parentIdOfEmptyFolder)?.delete(emptyFolderId)
|
||||
}
|
||||
}
|
||||
|
||||
private _deleteEmptyFoldersFromMaps(emptyFolderIds: Set<string>): void {
|
||||
for (const emptyFolderId of emptyFolderIds) {
|
||||
if (emptyFolderId === this._rootFolderId) continue
|
||||
const emptyFolderNode = this._nodeByIdMap.get(emptyFolderId)
|
||||
if (!emptyFolderNode) continue
|
||||
|
||||
const parentIdOfEmptyFolder = this._determineEffectiveParentId(emptyFolderNode)
|
||||
this._childIdsByParentIdMap.get(parentIdOfEmptyFolder)?.delete(emptyFolderId)
|
||||
|
||||
this._nodeByIdMap.delete(emptyFolderId)
|
||||
this._childIdsByParentIdMap.delete(emptyFolderId)
|
||||
}
|
||||
}
|
||||
|
||||
private _buildNodeWithAllDescendants(nodeId: string): GoogleDriveNode {
|
||||
const requestedNode = this._nodeByIdMap.get(nodeId)
|
||||
if (!requestedNode) {
|
||||
throw new Error(`Node ${nodeId} not found`)
|
||||
}
|
||||
|
||||
const allDescendantNodes = this._buildSortedDescendantNodes(nodeId)
|
||||
|
||||
return {
|
||||
...structuredClone(requestedNode),
|
||||
children: allDescendantNodes.length > 0 ? allDescendantNodes : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
private _buildSortedDescendantNodes(parentNodeId: string): GoogleDriveNode[] {
|
||||
const childIdSet = this._childIdsByParentIdMap.get(parentNodeId)
|
||||
if (!childIdSet || childIdSet.size === 0) return []
|
||||
|
||||
return Array.from(childIdSet)
|
||||
.map((childId) => this._buildNodeWithAllDescendants(childId))
|
||||
.sort((nodeA, nodeB) => nodeA.name.localeCompare(nodeB.name))
|
||||
}
|
||||
|
||||
private _determineEffectiveParentId(nodeToAnalyze: GoogleDriveNode): string {
|
||||
if (nodeToAnalyze.sharedWithMeTime) {
|
||||
return this._determineParentIdForSharedWithMeNode(nodeToAnalyze)
|
||||
}
|
||||
|
||||
if (nodeToAnalyze.driveId) {
|
||||
return this._determineParentIdForSharedDriveNode(nodeToAnalyze)
|
||||
}
|
||||
|
||||
return nodeToAnalyze.parents?.[0] ?? this._rootFolderId
|
||||
}
|
||||
|
||||
private _determineParentIdForSharedWithMeNode(sharedWithMeNode: GoogleDriveNode): string {
|
||||
this._ensureSpecialFolderNodeExists(SHARED_WITH_ME_ID, 'Shared with me')
|
||||
|
||||
if (sharedWithMeNode.parents?.[0]) {
|
||||
this._ensureParentExistsWithinSharedWithMeFolder(sharedWithMeNode.parents[0])
|
||||
return sharedWithMeNode.parents[0]
|
||||
}
|
||||
|
||||
return SHARED_WITH_ME_ID
|
||||
}
|
||||
|
||||
private _determineParentIdForSharedDriveNode(sharedDriveNode: GoogleDriveNode): string {
|
||||
this._ensureSpecialFolderNodeExists(SHARED_DRIVES_ID, 'Shared drives')
|
||||
this._ensureSharedDriveRootNodeExists(sharedDriveNode.driveId!, sharedDriveNode.driveId!)
|
||||
|
||||
return sharedDriveNode.parents?.[0] ?? sharedDriveNode.driveId!
|
||||
}
|
||||
|
||||
private _ensureParentExistsWithinSharedWithMeFolder(parentIdWithinSharedWithMe: string): void {
|
||||
if (this._nodeByIdMap.has(parentIdWithinSharedWithMe)) return
|
||||
|
||||
this._createPlaceholderFolderNode(parentIdWithinSharedWithMe, [SHARED_WITH_ME_ID])
|
||||
this._ensureChildSetExistsForParent(SHARED_WITH_ME_ID)
|
||||
this._childIdsByParentIdMap.get(SHARED_WITH_ME_ID)!.add(parentIdWithinSharedWithMe)
|
||||
}
|
||||
|
||||
private _ensureParentNodeExists(parentIdToCheck: string, childNodeRequiringParent: GoogleDriveNode): void {
|
||||
if (this._nodeByIdMap.has(parentIdToCheck)) return
|
||||
if (this._isSpecialSystemFolder(parentIdToCheck)) return
|
||||
|
||||
const placeholderParentId = this._determinePlaceholderParentIdForMissingNode(childNodeRequiringParent)
|
||||
this._createPlaceholderFolderNode(parentIdToCheck, [placeholderParentId])
|
||||
this._ensureChildSetExistsForParent(placeholderParentId)
|
||||
this._childIdsByParentIdMap.get(placeholderParentId)!.add(parentIdToCheck)
|
||||
}
|
||||
|
||||
private _isSpecialSystemFolder(nodeIdToCheck: string): boolean {
|
||||
return (
|
||||
nodeIdToCheck === SHARED_WITH_ME_ID || nodeIdToCheck === SHARED_DRIVES_ID || nodeIdToCheck === this._rootFolderId
|
||||
)
|
||||
}
|
||||
|
||||
private _determinePlaceholderParentIdForMissingNode(childNodeNeedingParent: GoogleDriveNode): string {
|
||||
if (childNodeNeedingParent.sharedWithMeTime) {
|
||||
this._ensureSpecialFolderNodeExists(SHARED_WITH_ME_ID, 'Shared with me')
|
||||
return SHARED_WITH_ME_ID
|
||||
}
|
||||
|
||||
if (childNodeNeedingParent.driveId) {
|
||||
this._ensureSpecialFolderNodeExists(SHARED_DRIVES_ID, 'Shared drives')
|
||||
this._ensureSharedDriveRootNodeExists(childNodeNeedingParent.driveId, childNodeNeedingParent.driveId)
|
||||
return childNodeNeedingParent.driveId
|
||||
}
|
||||
|
||||
return this._rootFolderId
|
||||
}
|
||||
|
||||
private _createPlaceholderFolderNode(placeholderNodeId: string, parentIds: string[]): void {
|
||||
this._nodeByIdMap.set(placeholderNodeId, {
|
||||
id: placeholderNodeId,
|
||||
name: `[${placeholderNodeId}]`,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
parents: parentIds,
|
||||
})
|
||||
this._childIdsByParentIdMap.set(placeholderNodeId, new Set())
|
||||
}
|
||||
|
||||
private _ensureSpecialFolderNodeExists(specialFolderId: string, displayName: string): void {
|
||||
if (this._nodeByIdMap.has(specialFolderId)) return
|
||||
|
||||
this._nodeByIdMap.set(specialFolderId, {
|
||||
id: specialFolderId,
|
||||
name: displayName,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
})
|
||||
this._childIdsByParentIdMap.set(specialFolderId, new Set())
|
||||
this._childIdsByParentIdMap.get(this._rootFolderId)!.add(specialFolderId)
|
||||
}
|
||||
|
||||
private _ensureSharedDriveRootNodeExists(sharedDriveRootNodeId: string, sharedDriveId: string): void {
|
||||
if (this._nodeByIdMap.has(sharedDriveRootNodeId)) return
|
||||
|
||||
this._nodeByIdMap.set(sharedDriveRootNodeId, {
|
||||
id: sharedDriveRootNodeId,
|
||||
name: `[${sharedDriveId}]`,
|
||||
mimeType: APP_GOOGLE_FOLDER_MIMETYPE,
|
||||
parents: [SHARED_DRIVES_ID],
|
||||
})
|
||||
this._childIdsByParentIdMap.set(sharedDriveRootNodeId, new Set())
|
||||
this._childIdsByParentIdMap.get(SHARED_DRIVES_ID)!.add(sharedDriveRootNodeId)
|
||||
}
|
||||
|
||||
public toJSON(): string {
|
||||
return JSON.stringify(this.getRootNode())
|
||||
}
|
||||
|
||||
public static fromJSON(json: string): GoogleDriveNodeTree {
|
||||
const rootNode = JSON.parse(json)
|
||||
return new GoogleDriveNodeTree({ rootFolderId: rootNode.id }).upsertNode(rootNode)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import * as oauthWizard from '@botpress/common/src/oauth-wizard'
|
||||
import * as sdk from '@botpress/sdk'
|
||||
import { getAccessToken, updateRefreshTokenFromAuthorizationCode } from './auth'
|
||||
import { Client } from './client'
|
||||
import { FileChannelsCache } from './file-channels-cache'
|
||||
import { FileEventHandler } from './file-event-handler'
|
||||
import { FilesCache } from './files-cache'
|
||||
import { NotificationHandler } from './notification-handler'
|
||||
import { notificationSchema } from './schemas'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export const handler: bp.IntegrationProps['handler'] = async (props) => {
|
||||
const { req, client, ctx, logger } = props
|
||||
|
||||
if (oauthWizard.isOAuthWizardUrl(req.path)) {
|
||||
return await _handleOAuthWizard(props)
|
||||
}
|
||||
|
||||
const notifParseResult = notificationSchema.safeParse(req)
|
||||
if (!notifParseResult.success) {
|
||||
console.error('Invalid request:', notifParseResult.error)
|
||||
return {
|
||||
status: 400,
|
||||
body: 'Invalid request',
|
||||
}
|
||||
}
|
||||
|
||||
const notification = notifParseResult.data
|
||||
if (!NotificationHandler.isSupported(notification)) {
|
||||
return
|
||||
}
|
||||
|
||||
const driveClient = await Client.create({ client, ctx, logger })
|
||||
const filesCache = await FilesCache.load({ client, ctx })
|
||||
const fileChannelsCache = await FileChannelsCache.load({ client, ctx })
|
||||
const fileEventHandler = new FileEventHandler(client, driveClient, filesCache, fileChannelsCache)
|
||||
const notificationHandler = new NotificationHandler(driveClient, filesCache, fileEventHandler)
|
||||
await notificationHandler.handle(notification)
|
||||
await filesCache.save()
|
||||
await fileChannelsCache.save()
|
||||
return
|
||||
}
|
||||
|
||||
const _handleOAuthWizard = async (props: bp.HandlerProps): Promise<sdk.Response> => {
|
||||
const { client, ctx } = props
|
||||
|
||||
const wizard = new oauthWizard.OAuthWizardBuilder(props)
|
||||
|
||||
.addStep({
|
||||
id: 'start',
|
||||
handler({ responses }) {
|
||||
return responses.displayButtons({
|
||||
pageTitle: 'Google Drive Integration',
|
||||
htmlOrMarkdownPageContents: `
|
||||
This wizard will reset your Google Drive integration. This means
|
||||
that the integration will cease to function until you complete the
|
||||
authorization process.
|
||||
|
||||
Do you wish to continue?
|
||||
`,
|
||||
buttons: [
|
||||
{
|
||||
action: 'external',
|
||||
label: 'Yes, continue',
|
||||
navigateToUrl: _getOAuthAuthorizationUri(ctx),
|
||||
buttonType: 'primary',
|
||||
},
|
||||
{ action: 'close', label: 'No, cancel', buttonType: 'secondary' },
|
||||
],
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
.addStep({
|
||||
id: 'oauth-callback',
|
||||
async handler({ query, responses }) {
|
||||
const authorizationCode = query.get('code')
|
||||
|
||||
if (!authorizationCode) {
|
||||
console.error('Error extracting code from url in OAuth handler')
|
||||
return responses.endWizard({
|
||||
success: false,
|
||||
errorMessage: 'Error extracting code from url in OAuth handler',
|
||||
})
|
||||
}
|
||||
|
||||
await updateRefreshTokenFromAuthorizationCode({ authorizationCode, client, ctx })
|
||||
|
||||
// Done in order to correctly display the authorization status in the UI (not used for webhooks)
|
||||
await client.configureIntegration({
|
||||
identifier: ctx.webhookId,
|
||||
})
|
||||
|
||||
return responses.redirectToStep('file-picker')
|
||||
},
|
||||
})
|
||||
|
||||
.addStep({
|
||||
id: 'file-picker',
|
||||
async handler({ responses, client, ctx }) {
|
||||
return responses.displayButtons({
|
||||
pageTitle: 'Google Drive Integration',
|
||||
htmlOrMarkdownPageContents: `
|
||||
You will now be asked to select the files and folders you wish to
|
||||
grant access to. This is necessary for the integration to work
|
||||
properly.
|
||||
|
||||
If you do not give access to any files or folders, the integration
|
||||
will only be able to access files that are created through the
|
||||
integration.
|
||||
|
||||
<script>
|
||||
function createPicker() {
|
||||
new google.picker.PickerBuilder()
|
||||
.addView(new google.picker.DocsView()
|
||||
.setIncludeFolders(true)
|
||||
.setSelectFolderEnabled(true)
|
||||
.setMode(google.picker.DocsViewMode.LIST))
|
||||
.enableFeature(google.picker.Feature.NAV_HIDDEN)
|
||||
.enableFeature(google.picker.Feature.MULTISELECT_ENABLED)
|
||||
.setTitle('Select the files and folders you wish to share with Botpress')
|
||||
.setOAuthToken('${await getAccessToken({ client, ctx })}')
|
||||
.setDeveloperKey('${bp.secrets.FILE_PICKER_API_KEY}')
|
||||
.setCallback((data) => {
|
||||
if (data[google.picker.Response.ACTION] == google.picker.Action.PICKED) {
|
||||
document.location.href = '${oauthWizard.getWizardStepUrl('end', ctx).href}';
|
||||
}})
|
||||
.setSize(640,790)
|
||||
.setAppId('${bp.secrets.CLIENT_ID.split('-')[0]}')
|
||||
.build().setVisible(true);
|
||||
}
|
||||
</script>
|
||||
<script async defer src="https://apis.google.com/js/api.js" onload="gapi.load('picker')"></script>
|
||||
`,
|
||||
buttons: [
|
||||
{
|
||||
action: 'javascript',
|
||||
label: 'Select files',
|
||||
callFunction: 'createPicker',
|
||||
buttonType: 'primary',
|
||||
},
|
||||
{
|
||||
action: 'navigate',
|
||||
label: 'Skip file selection',
|
||||
navigateToStep: 'end',
|
||||
buttonType: 'warning',
|
||||
},
|
||||
],
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
.addStep({
|
||||
id: 'end',
|
||||
handler({ responses }) {
|
||||
return responses.endWizard({
|
||||
success: true,
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
.build()
|
||||
|
||||
return await wizard.handleRequest()
|
||||
}
|
||||
|
||||
const _getOAuthAuthorizationUri = (ctx: { webhookId: string }) =>
|
||||
'https://accounts.google.com/o/oauth2/v2/auth?scope=' +
|
||||
'https%3A//www.googleapis.com/auth/drive.file&access_type=offline' +
|
||||
'&include_granted_scopes=true&response_type=code&prompt=consent&trigger_onepick=true' +
|
||||
`&state=${ctx.webhookId}&redirect_uri=${encodeURI(_getOAuthRedirectUri().href)}` +
|
||||
`&client_id=${bp.secrets.CLIENT_ID}`
|
||||
|
||||
const _getOAuthRedirectUri = () => oauthWizard.getWizardStepUrl('oauth-callback')
|
||||
@@ -0,0 +1,15 @@
|
||||
import { reporting } from '@botpress/sdk-addons'
|
||||
import actions from './actions'
|
||||
import { handler } from './handler'
|
||||
import { register, unregister } from './setup'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
const integration = new bp.Integration({
|
||||
register,
|
||||
unregister,
|
||||
actions,
|
||||
channels: {},
|
||||
handler,
|
||||
})
|
||||
|
||||
export default reporting.wrapIntegration(integration)
|
||||
@@ -0,0 +1,15 @@
|
||||
export const APP_PDF_MIMETYPE = 'application/pdf'
|
||||
export const TEXT_HTML_MIMETYPE = 'text/html'
|
||||
export const TEXT_MARKDOWN_MIMETYPE = 'text/markdown'
|
||||
export const TEXT_PLAIN_MIMETYPE = 'text/plain'
|
||||
|
||||
export const APP_GOOGLE_DOCS_MIMETYPE = 'application/vnd.google-apps.document'
|
||||
export const APP_GOOGLE_SHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet'
|
||||
export const APP_GOOGLE_SLIDES_MIMETYPE = 'application/vnd.google-apps.presentation'
|
||||
|
||||
export const APP_GOOGLE_FOLDER_MIMETYPE = 'application/vnd.google-apps.folder'
|
||||
export const APP_GOOGLE_SHORTCUT_MIMETYPE = 'application/vnd.google-apps.shortcut'
|
||||
|
||||
// Order of types in the array should reflect the priority of type when automatically choosing type for export
|
||||
// Prioritize types best suited for indexing
|
||||
export const INDEXABLE_MIMETYPES = [APP_PDF_MIMETYPE, TEXT_HTML_MIMETYPE, TEXT_MARKDOWN_MIMETYPE, TEXT_PLAIN_MIMETYPE]
|
||||
@@ -0,0 +1,59 @@
|
||||
import { Client } from './client'
|
||||
import { FileEventHandler } from './file-event-handler'
|
||||
import { deserializeToken, Token } from './file-notification-token'
|
||||
import { FilesCache } from './files-cache'
|
||||
import { Notification } from './types'
|
||||
import * as bp from '.botpress'
|
||||
|
||||
export class NotificationHandler {
|
||||
public constructor(
|
||||
private _driveClient: Client,
|
||||
private _filesCache: FilesCache,
|
||||
private _fileEventHandler: FileEventHandler
|
||||
) {}
|
||||
|
||||
public static isSupported(notification: Notification): boolean {
|
||||
const type = notification.headers['x-goog-resource-state']
|
||||
return type === 'update' || type === 'remove'
|
||||
}
|
||||
|
||||
public async handle(notification: Notification): Promise<void> {
|
||||
const type = notification.headers['x-goog-resource-state']
|
||||
const changes = notification.headers['x-goog-changed']
|
||||
const serializedToken = notification.headers['x-goog-channel-token']
|
||||
const token = deserializeToken(serializedToken, bp.secrets.WEBHOOK_SECRET)
|
||||
if (!token) {
|
||||
console.error('Invalid notification token:', token)
|
||||
return
|
||||
}
|
||||
if (type === 'update') {
|
||||
for (const change of changes) {
|
||||
if (change === 'children') {
|
||||
await this._handleUpdateChildrenNotif(token)
|
||||
}
|
||||
}
|
||||
} else if (type === 'remove') {
|
||||
await this._handleRemoveNotif(token)
|
||||
}
|
||||
}
|
||||
|
||||
private async _handleUpdateChildrenNotif(token: Token) {
|
||||
if (token.fileType !== 'folder') {
|
||||
return
|
||||
}
|
||||
const currentChildren = await this._driveClient.getChildren(token.fileId)
|
||||
for (const child of currentChildren) {
|
||||
if (!this._filesCache.find(child.id)) {
|
||||
await this._fileEventHandler.handleFileCreated(child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async _handleRemoveNotif(token: Token) {
|
||||
const baseFile = this._filesCache.find(token.fileId)
|
||||
if (!baseFile) {
|
||||
return
|
||||
}
|
||||
await this._fileEventHandler.handleFileDeleted(baseFile)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
import { z } from '@botpress/sdk'
|
||||
import { APP_GOOGLE_FOLDER_MIMETYPE, APP_GOOGLE_SHORTCUT_MIMETYPE } from './mime-types'
|
||||
|
||||
// Utility schemas
|
||||
export const fileIdSchema = z.string().min(1).describe('The ID of the Google Drive file')
|
||||
export const commonFileAttrSchema = z.object({
|
||||
id: fileIdSchema.title('File ID'),
|
||||
name: z.string().min(1).title('Name').describe('The name of the file'),
|
||||
parentId: z
|
||||
.string()
|
||||
.min(1)
|
||||
.optional()
|
||||
.title('Parent ID')
|
||||
.describe("The ID of the file that is the parent of this file. If not set, 'My Drive' is the parent"),
|
||||
mimeType: z.string().min(1).title('MIME Type').describe('The media type of the file'),
|
||||
})
|
||||
|
||||
export const baseNormalFileSchema = commonFileAttrSchema.extend({
|
||||
size: z.number().nonnegative().title('Size').describe('The size in bytes of the file'),
|
||||
contentHash: z
|
||||
.string()
|
||||
.optional()
|
||||
.title('Content Hash')
|
||||
.describe('The hash of the file content, or version/revision number'),
|
||||
lastModifiedDate: z
|
||||
.string()
|
||||
.datetime()
|
||||
.optional()
|
||||
.title('Last Modified Date')
|
||||
.describe('The last modified date of the file in RFC 3339 format'),
|
||||
})
|
||||
|
||||
export const baseFolderFileSchema = commonFileAttrSchema.extend({
|
||||
mimeType: z.literal(APP_GOOGLE_FOLDER_MIMETYPE).title('MIME Type').describe('The media type of the folder'),
|
||||
})
|
||||
|
||||
export const baseShortcutFileSchema = commonFileAttrSchema.extend({
|
||||
mimeType: z.literal(APP_GOOGLE_SHORTCUT_MIMETYPE).title('MIME Type').describe('The media type of the shortcut'),
|
||||
})
|
||||
|
||||
const _fileTypesArray = ['normal', 'folder', 'shortcut'] as const
|
||||
export const fileTypesEnumSchema = z.enum(_fileTypesArray)
|
||||
const _fileTypes = fileTypesEnumSchema.Enum
|
||||
export const fileTypesUnionSchema = z.union([
|
||||
z.literal(_fileTypes.normal),
|
||||
z.literal(_fileTypes.folder),
|
||||
z.literal(_fileTypes.shortcut),
|
||||
])
|
||||
|
||||
/* Used to represent a generic file, closer to what is received by the API.
|
||||
Type is added to enable discrimination and remove/add access to properties
|
||||
depending on file type. */
|
||||
export const baseDiscriminatedFileSchema = z.discriminatedUnion('type', [
|
||||
baseNormalFileSchema.extend({ type: z.literal(_fileTypes.normal).title('Type').describe('The type of the file') }),
|
||||
baseFolderFileSchema.extend({ type: z.literal(_fileTypes.folder).title('Type').describe('The type of the file') }),
|
||||
baseShortcutFileSchema.extend({
|
||||
type: z.literal(_fileTypes.shortcut).title('Type').describe('The type of the file'),
|
||||
}),
|
||||
])
|
||||
|
||||
export const baseChannelSchema = z.object({
|
||||
id: z.string().min(1).title('Channel ID').describe('The ID of the channel'),
|
||||
resourceId: z
|
||||
.string()
|
||||
.min(1)
|
||||
.title('Resource ID')
|
||||
.describe('The ID of the watched resource (different from the file ID)'),
|
||||
})
|
||||
|
||||
const notificationTypesSchema = z.union([
|
||||
z.literal('sync'),
|
||||
z.literal('add'),
|
||||
z.literal('remove'),
|
||||
z.literal('update'),
|
||||
z.literal('trash'),
|
||||
z.literal('untrash'),
|
||||
z.literal('change'),
|
||||
])
|
||||
const updateDetailTypesSchema = z.union([
|
||||
z.literal('content'),
|
||||
z.literal('properties'),
|
||||
z.literal('parents'),
|
||||
z.literal('children'),
|
||||
z.literal('permissions'),
|
||||
])
|
||||
export const notificationSchema = z.object({
|
||||
headers: z.object({
|
||||
'x-goog-resource-state': notificationTypesSchema,
|
||||
'x-goog-changed': z
|
||||
.string()
|
||||
.optional() // May be present on 'update' notifications
|
||||
.transform((details) => details?.split(',') ?? [])
|
||||
.pipe(z.array(updateDetailTypesSchema)),
|
||||
'x-goog-channel-token': z.string(),
|
||||
}),
|
||||
})
|
||||
|
||||
// Entities
|
||||
const computedFileAttrSchema = z.object({
|
||||
path: z
|
||||
.array(
|
||||
z
|
||||
.string()
|
||||
.min(1)
|
||||
.describe("A component of the path of the file. It corresponds to the name of one of it's parents.")
|
||||
)
|
||||
.title('Path')
|
||||
.describe("An array of the path's components sorted by level (root to leaf)"),
|
||||
})
|
||||
export const fileSchema = baseNormalFileSchema.merge(computedFileAttrSchema)
|
||||
export const folderSchema = baseFolderFileSchema.merge(computedFileAttrSchema)
|
||||
export const shortcutSchema = baseShortcutFileSchema.merge(computedFileAttrSchema)
|
||||
export const genericFileSchema = z.discriminatedUnion('type', [
|
||||
fileSchema.extend({ type: z.literal(_fileTypes.normal) }),
|
||||
folderSchema.extend({ type: z.literal(_fileTypes.folder) }),
|
||||
shortcutSchema.extend({ type: z.literal(_fileTypes.shortcut) }),
|
||||
])
|
||||
export const fileChannelSchema = baseChannelSchema.extend({
|
||||
fileId: fileIdSchema.title('File ID'),
|
||||
})
|
||||
|
||||
// Action args/outputs
|
||||
function createListOutputSchema<T extends z.ZodTypeAny>(itemSchema: T) {
|
||||
return z.object({
|
||||
items: z
|
||||
.array(itemSchema)
|
||||
.describe(
|
||||
'The list of items listed in Google Drive. Results may be paginated. If set, use nextToken to get additional results'
|
||||
),
|
||||
meta: z
|
||||
.object({
|
||||
nextToken: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe('The token to pass as input to the next call of the list action to list additional items'),
|
||||
})
|
||||
.describe('Metadata about the list results, including pagination information'),
|
||||
})
|
||||
}
|
||||
|
||||
export const createFileArgSchema = commonFileAttrSchema.omit({
|
||||
id: true, // Unknown at creation time
|
||||
})
|
||||
export const readFileArgSchema = z.object({ id: fileIdSchema.title('File ID') })
|
||||
export const updateFileArgSchema = commonFileAttrSchema.omit({
|
||||
mimeType: true, // Cannot be changed unless data is uploaded with a new type
|
||||
})
|
||||
export const deleteFileArgSchema = z.object({ id: fileIdSchema.title('File ID') })
|
||||
export const listItemsInputSchema = z.object({
|
||||
nextToken: z.string().optional().title('Next Token').describe('The token to use to get the next page of results'),
|
||||
})
|
||||
export const listItemsOutputSchema = createListOutputSchema(z.any())
|
||||
export const listFilesOutputSchema = createListOutputSchema(fileSchema)
|
||||
export const listFoldersOutputSchema = createListOutputSchema(folderSchema)
|
||||
|
||||
// URL is used instead of ID because an integration can't access all files of a bot
|
||||
// using the files API
|
||||
export const uploadFileDataArgSchema = z.object({
|
||||
id: z.string().min(1).title('File ID').describe('The ID of the Google Drive file to upload content to'),
|
||||
url: z
|
||||
.string()
|
||||
.min(1)
|
||||
.title('File URL')
|
||||
.describe('The URL used to access the file whose content will be uploaded to Google Drive'),
|
||||
mimeType: z
|
||||
.string()
|
||||
.min(1)
|
||||
.optional()
|
||||
.title('MIME Type')
|
||||
.describe('Media type of the uploaded content. This will override any previously set media type'),
|
||||
})
|
||||
export const downloadFileDataArgSchema = z.object({
|
||||
id: z.string().min(1).title('File ID').describe('The ID of the Google Drive file whose content will be downloaded'),
|
||||
index: z.boolean().title('Index File').describe('Indicates if the file is to be indexed or not'),
|
||||
})
|
||||
export const downloadFileDataOutputSchema = z.object({
|
||||
bpFileId: z
|
||||
.string()
|
||||
.min(1)
|
||||
.describe('The Botpress file ID corresponding to the file that was uploaded from Google Drive to the Files API'),
|
||||
})
|
||||
|
||||
export const fileDeletedEventSchema = z.object({
|
||||
id: fileIdSchema.title('File ID'),
|
||||
})
|
||||
export const folderDeletedEventSchema = z.object({
|
||||
id: fileIdSchema.title('Folder ID'),
|
||||
})
|
||||
@@ -0,0 +1,2 @@
|
||||
export const register = async () => {}
|
||||
export const unregister = async () => {}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { z } from '@botpress/sdk'
|
||||
import { google, drive_v3 } from 'googleapis'
|
||||
import {
|
||||
baseDiscriminatedFileSchema,
|
||||
baseNormalFileSchema,
|
||||
baseFolderFileSchema,
|
||||
baseShortcutFileSchema,
|
||||
fileSchema,
|
||||
folderSchema,
|
||||
commonFileAttrSchema,
|
||||
listFilesOutputSchema,
|
||||
listFoldersOutputSchema,
|
||||
createFileArgSchema,
|
||||
updateFileArgSchema,
|
||||
uploadFileDataArgSchema,
|
||||
downloadFileDataArgSchema,
|
||||
downloadFileDataOutputSchema,
|
||||
listItemsOutputSchema,
|
||||
listItemsInputSchema,
|
||||
fileChannelSchema,
|
||||
notificationSchema,
|
||||
fileTypesUnionSchema,
|
||||
baseChannelSchema,
|
||||
genericFileSchema,
|
||||
shortcutSchema,
|
||||
} from './schemas'
|
||||
|
||||
type Overwrite<T, NewT> = Omit<T, keyof NewT> & NewT
|
||||
|
||||
export type GoogleDriveClient = drive_v3.Drive
|
||||
export type UnvalidatedGoogleDriveFile = drive_v3.Schema$File
|
||||
export type UnvalidatedGoogleDriveChannel = drive_v3.Schema$Channel
|
||||
|
||||
export type CommonFileAttr = z.infer<typeof commonFileAttrSchema>
|
||||
export type BaseDiscriminatedFile = z.infer<typeof baseDiscriminatedFileSchema>
|
||||
export type BaseNormalFile = z.infer<typeof baseNormalFileSchema>
|
||||
export type BaseFolderFile = z.infer<typeof baseFolderFileSchema>
|
||||
export type BaseShortcutFile = z.infer<typeof baseShortcutFileSchema>
|
||||
export type BaseGenericFileUnion = BaseNormalFile | BaseFolderFile | BaseShortcutFile
|
||||
export type BaseFileChannel = z.infer<typeof baseChannelSchema>
|
||||
export type FileType = z.infer<typeof fileTypesUnionSchema>
|
||||
export type Notification = z.infer<typeof notificationSchema>
|
||||
|
||||
export type File = z.infer<typeof fileSchema>
|
||||
export type Folder = z.infer<typeof folderSchema>
|
||||
export type Shortcut = z.infer<typeof shortcutSchema>
|
||||
export type GenericFile = z.infer<typeof genericFileSchema>
|
||||
export type FileChannel = z.infer<typeof fileChannelSchema>
|
||||
|
||||
export type ListItemsInput = z.infer<typeof listItemsInputSchema>
|
||||
export type ListItemsOutput<T> = Overwrite<z.infer<typeof listItemsOutputSchema>, { items: T[] }>
|
||||
export type ListFilesOutput = z.infer<typeof listFilesOutputSchema>
|
||||
export type ListFoldersOutput = z.infer<typeof listFoldersOutputSchema>
|
||||
export type CreateFileArgs = z.infer<typeof createFileArgSchema>
|
||||
export type UpdateFileArgs = z.infer<typeof updateFileArgSchema>
|
||||
export type UploadFileDataArgs = z.infer<typeof uploadFileDataArgSchema>
|
||||
export type DownloadFileDataArgs = z.infer<typeof downloadFileDataArgSchema>
|
||||
export type DownloadFileDataOutput = z.infer<typeof downloadFileDataOutputSchema>
|
||||
|
||||
export type GoogleOAuth2Client = InstanceType<(typeof google.auth)['OAuth2']>
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Readable } from 'stream'
|
||||
import { ListItemsInput, ListItemsOutput } from './types'
|
||||
|
||||
export type ListFunction<T> = (input: ListItemsInput) => Promise<ListItemsOutput<T>>
|
||||
export type ListItemsInputWithArgs<T> = ListItemsInput & {
|
||||
args?: T
|
||||
}
|
||||
export type ListFunctionWithArgs<T, U> = (input: ListItemsInputWithArgs<U>) => Promise<ListItemsOutput<T>>
|
||||
|
||||
export const streamToBuffer = (stream: Readable, maxBufferSize: number): Promise<Buffer> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunkArray: Buffer[] = []
|
||||
let size = 0
|
||||
stream
|
||||
.on('data', (chunk: Buffer) => {
|
||||
size += chunk.length
|
||||
if (size > maxBufferSize) {
|
||||
reject(`Max buffer size exceeded while converting stream to buffer (${maxBufferSize})`)
|
||||
}
|
||||
chunkArray.push(chunk)
|
||||
})
|
||||
.on('end', () => {
|
||||
resolve(Buffer.concat(chunkArray))
|
||||
})
|
||||
.on('error', (err) => {
|
||||
reject(err)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export const listItemsAndProcess = async <T, U>(
|
||||
listFn: ListFunctionWithArgs<T, U>,
|
||||
processFn: (item: T) => Promise<void>,
|
||||
args?: U
|
||||
) => {
|
||||
let nextToken: string | undefined = undefined
|
||||
do {
|
||||
const { items, meta } = await listFn({ nextToken, args })
|
||||
for (const item of items) {
|
||||
await processFn(item)
|
||||
}
|
||||
nextToken = meta.nextToken
|
||||
} while (nextToken)
|
||||
}
|
||||
|
||||
export const listAllItems = async <T, U>(listFn: ListFunctionWithArgs<T, U>, args?: U): Promise<T[]> => {
|
||||
const items: T[] = []
|
||||
let nextToken: string | undefined = undefined
|
||||
do {
|
||||
const { items: currentItems, meta } = await listFn({ nextToken, args })
|
||||
items.push(...currentItems)
|
||||
nextToken = meta.nextToken
|
||||
} while (nextToken)
|
||||
return items
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import { RuntimeError } from '@botpress/sdk'
|
||||
import { APP_GOOGLE_FOLDER_MIMETYPE, APP_GOOGLE_SHORTCUT_MIMETYPE } from './mime-types'
|
||||
import { baseFolderFileSchema, baseNormalFileSchema, baseShortcutFileSchema } from './schemas'
|
||||
import {
|
||||
BaseFolderFile,
|
||||
BaseDiscriminatedFile,
|
||||
BaseNormalFile,
|
||||
BaseShortcutFile,
|
||||
CommonFileAttr,
|
||||
UnvalidatedGoogleDriveFile,
|
||||
FileType,
|
||||
UnvalidatedGoogleDriveChannel,
|
||||
BaseFileChannel,
|
||||
} from './types'
|
||||
|
||||
export const parseChannel = (channel: UnvalidatedGoogleDriveChannel): BaseFileChannel => {
|
||||
const { id, resourceId } = channel
|
||||
if (!resourceId) {
|
||||
throw new RuntimeError('Resource ID is missing in Schema$Channel from the API response')
|
||||
}
|
||||
|
||||
if (!id) {
|
||||
throw new RuntimeError('Channel ID is missing in Schema$Channel from the API response')
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
resourceId,
|
||||
}
|
||||
}
|
||||
|
||||
export const getFileTypeFromMimeType = (mimeType: string): FileType => {
|
||||
switch (mimeType) {
|
||||
case APP_GOOGLE_FOLDER_MIMETYPE:
|
||||
return 'folder'
|
||||
case APP_GOOGLE_SHORTCUT_MIMETYPE:
|
||||
return 'shortcut'
|
||||
default:
|
||||
return 'normal'
|
||||
}
|
||||
}
|
||||
|
||||
export const parseBaseGenerics = (files: UnvalidatedGoogleDriveFile[]): BaseDiscriminatedFile[] => {
|
||||
return files.map((f) => parseBaseGeneric(f))
|
||||
}
|
||||
|
||||
export const parseBaseGeneric = (unvalidatedFile: UnvalidatedGoogleDriveFile): BaseDiscriminatedFile => {
|
||||
const { mimeType } = parseCommonFileAttr(unvalidatedFile)
|
||||
let file: BaseDiscriminatedFile
|
||||
const type = getFileTypeFromMimeType(mimeType)
|
||||
switch (type) {
|
||||
case 'folder':
|
||||
file = {
|
||||
...parseBaseFolder(unvalidatedFile),
|
||||
type,
|
||||
}
|
||||
break
|
||||
case 'shortcut':
|
||||
file = {
|
||||
...parseBaseShortcut(unvalidatedFile),
|
||||
type,
|
||||
}
|
||||
break
|
||||
default:
|
||||
file = {
|
||||
...parseBaseNormal(unvalidatedFile),
|
||||
type,
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
|
||||
export const parseBaseNormal = (unvalidatedFile: UnvalidatedGoogleDriveFile): BaseNormalFile => {
|
||||
const commmonFileAttr = parseCommonFileAttr(unvalidatedFile)
|
||||
const { size: sizeStr, sha256Checksum, md5Checksum, version, modifiedTime } = unvalidatedFile
|
||||
|
||||
const size = parseInt(sizeStr ?? '0')
|
||||
if (isNaN(size)) {
|
||||
throw new RuntimeError(
|
||||
`Invalid size returned in Schema$File from the API response for file with name=${commmonFileAttr.name} (size=${sizeStr})`
|
||||
)
|
||||
}
|
||||
|
||||
const parseResult = baseNormalFileSchema.safeParse({
|
||||
...commmonFileAttr,
|
||||
size,
|
||||
contentHash: (sha256Checksum || md5Checksum || version) ?? undefined,
|
||||
lastModifiedDate: modifiedTime ?? undefined,
|
||||
})
|
||||
if (parseResult.error) {
|
||||
throw new RuntimeError('Error validating Schema$File received from the API response')
|
||||
}
|
||||
return parseResult.data
|
||||
}
|
||||
|
||||
const parseCommonFileAttr = (unvalidatedFile: UnvalidatedGoogleDriveFile): CommonFileAttr => {
|
||||
const { id, name, mimeType } = unvalidatedFile
|
||||
if (!id) {
|
||||
throw new RuntimeError('File ID is missing in Schema$File from the API response')
|
||||
}
|
||||
|
||||
if (!name) {
|
||||
throw new RuntimeError(
|
||||
`Name is missing in Schema$File from the API response for file with ID=${unvalidatedFile.id}`
|
||||
)
|
||||
}
|
||||
|
||||
if (!mimeType) {
|
||||
throw new RuntimeError(`MIME type is missing in Schema$File from the API response for file with name=${name}`)
|
||||
}
|
||||
|
||||
let parentId: string | undefined = undefined
|
||||
if (unvalidatedFile.parents) {
|
||||
parentId = unvalidatedFile.parents[0]
|
||||
if (!parentId) {
|
||||
throw new RuntimeError(`Empty parent ID array in Schema$File from the API response for file with name=${name}`)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
mimeType,
|
||||
parentId,
|
||||
}
|
||||
}
|
||||
|
||||
const parseBaseFolder = (unvalidatedFile: UnvalidatedGoogleDriveFile): BaseFolderFile => {
|
||||
const commmonFileAttr = parseCommonFileAttr(unvalidatedFile)
|
||||
const parseResult = baseFolderFileSchema.safeParse(commmonFileAttr)
|
||||
if (parseResult.error) {
|
||||
throw new RuntimeError('Error validating Schema$File received from the API response')
|
||||
}
|
||||
return parseResult.data
|
||||
}
|
||||
|
||||
const parseBaseShortcut = (unvalidatedFile: UnvalidatedGoogleDriveFile): BaseShortcutFile => {
|
||||
const commmonFileAttr = parseCommonFileAttr(unvalidatedFile)
|
||||
const parseResult = baseShortcutFileSchema.safeParse(commmonFileAttr)
|
||||
if (parseResult.error) {
|
||||
throw new RuntimeError('Error validating Schema$File received from the API response')
|
||||
}
|
||||
return parseResult.data
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"jsx": "react-jsx",
|
||||
"jsxImportSource": "preact",
|
||||
"types": ["preact"],
|
||||
"paths": { "*": ["./*"] },
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": [".botpress/**/*", "definitions/**/*", "src/**/*", "*.ts"]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
import config from '../../vitest.config'
|
||||
export default config
|
||||
Reference in New Issue
Block a user