chore: import upstream snapshot with attribution

2026-07-13 12:20:32 +08:00
commit d2375abf88
5575 changed files with 274444 additions and 0 deletions
@@ -0,0 +1,206 @@
+const { reqBody, multiUserMode, userFromSession } = require("../utils/http");
+const { handleFileUpload } = require("../utils/files/multer");
+const { validatedRequest } = require("../utils/middleware/validatedRequest");
+const { Telemetry } = require("../models/telemetry");
+const {
+  flexUserRoleValid,
+  ROLES,
+} = require("../utils/middleware/multiUserProtected");
+const { EventLogs } = require("../models/eventLogs");
+const { validWorkspaceSlug } = require("../utils/middleware/validWorkspace");
+const { CollectorApi } = require("../utils/collectorApi");
+const { WorkspaceThread } = require("../models/workspaceThread");
+const { WorkspaceParsedFiles } = require("../models/workspaceParsedFiles");
+
+function workspaceParsedFilesEndpoints(app) {
+  if (!app) return;
+
+  app.get(
+    "/workspace/:slug/parsed-files",
+    [validatedRequest, flexUserRoleValid([ROLES.all]), validWorkspaceSlug],
+    async (request, response) => {
+      try {
+        const threadSlug = request.query.threadSlug || null;
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+        const thread = threadSlug
+          ? await WorkspaceThread.get({ slug: String(threadSlug) })
+          : null;
+        const { files, contextWindow, currentContextTokenCount } =
+          await WorkspaceParsedFiles.getContextMetadataAndLimits(
+            workspace,
+            thread || null,
+            multiUserMode(response) ? user : null
+          );
+
+        return response
+          .status(200)
+          .json({ files, contextWindow, currentContextTokenCount });
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.delete(
+    "/workspace/:slug/delete-parsed-files",
+    [validatedRequest, flexUserRoleValid([ROLES.all]), validWorkspaceSlug],
+    async function (request, response) {
+      try {
+        const { fileIds = [] } = reqBody(request);
+        if (!fileIds.length) return response.sendStatus(400).end();
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+        const success = await WorkspaceParsedFiles.delete({
+          id: {
+            in: fileIds.map((id) => parseInt(id)),
+          },
+          ...(user ? { userId: user.id } : {}),
+          workspaceId: workspace.id,
+        });
+        return response.status(success ? 200 : 403).end();
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.post(
+    "/workspace/:slug/embed-parsed-file/:fileId",
+    [
+      validatedRequest,
+      // Embed is still an admin/manager only feature
+      flexUserRoleValid([ROLES.admin, ROLES.manager]),
+      validWorkspaceSlug,
+    ],
+    async function (request, response) {
+      const { fileId = null } = request.params;
+      try {
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+
+        if (!fileId) return response.sendStatus(400).end();
+        const { success, error, document } =
+          await WorkspaceParsedFiles.moveToDocumentsAndEmbed(
+            user,
+            fileId,
+            workspace
+          );
+
+        if (!success) {
+          return response.status(500).json({
+            success: false,
+            error: error || "Failed to embed file",
+          });
+        }
+
+        await Telemetry.sendTelemetry("document_embedded");
+        await EventLogs.logEvent(
+          "document_embedded",
+          {
+            documentName: document?.name || "unknown",
+            workspaceId: workspace.id,
+          },
+          user?.id
+        );
+
+        return response.status(200).json({
+          success: true,
+          error: null,
+          document,
+        });
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.post(
+    "/workspace/:slug/parse",
+    [
+      validatedRequest,
+      flexUserRoleValid([ROLES.all]),
+      handleFileUpload,
+      validWorkspaceSlug,
+    ],
+    async function (request, response) {
+      try {
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+        const Collector = new CollectorApi();
+        const { originalname } = request.file;
+        const processingOnline = await Collector.online();
+
+        if (!processingOnline) {
+          return response.status(500).json({
+            success: false,
+            error: `Document processing API is not online. Document ${originalname} will not be parsed.`,
+          });
+        }
+
+        const { success, reason, documents } =
+          await Collector.parseDocument(originalname);
+        if (!success || !documents?.[0]) {
+          return response.status(500).json({
+            success: false,
+            error: reason || "No document returned from collector",
+          });
+        }
+
+        // Get thread ID if we have a slug
+        const { threadSlug = null } = reqBody(request);
+        const thread = threadSlug
+          ? await WorkspaceThread.get({
+              slug: String(threadSlug),
+              workspace_id: workspace.id,
+              user_id: user?.id || null,
+            })
+          : null;
+        const files = await Promise.all(
+          documents.map(async (doc) => {
+            const metadata = { ...doc };
+            // Strip out pageContent
+            delete metadata.pageContent;
+            const filename = `${originalname}-${doc.id}.json`;
+            const { file, error: dbError } = await WorkspaceParsedFiles.create({
+              filename,
+              workspaceId: workspace.id,
+              userId: user?.id || null,
+              threadId: thread?.id || null,
+              metadata: JSON.stringify(metadata),
+              tokenCountEstimate: doc.token_count_estimate || 0,
+            });
+
+            if (dbError) throw new Error(dbError);
+            return file;
+          })
+        );
+
+        Collector.log(`Document ${originalname} parsed successfully.`);
+        await EventLogs.logEvent(
+          "document_uploaded_to_chat",
+          {
+            documentName: originalname,
+            workspace: workspace.slug,
+            thread: thread?.name || null,
+          },
+          user?.id
+        );
+
+        return response.status(200).json({
+          success: true,
+          error: null,
+          files,
+        });
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      }
+    }
+  );
+}
+
+module.exports = { workspaceParsedFilesEndpoints };