chore: import upstream snapshot with attribution

api/server/services/Threads/index.js

@@ -0,0 +1,5 @@
+const manage = require('./manage');
+
+module.exports = {
+  ...manage,
+};

api/server/services/Threads/manage.js

@@ -0,0 +1,690 @@
+const path = require('path');
+const { v4 } = require('uuid');
+const { countTokens } = require('@librechat/api');
+const { escapeRegExp } = require('@librechat/data-schemas');
+const {
+  Constants,
+  ContentTypes,
+  AnnotationTypes,
+  defaultOrderQuery,
+} = require('librechat-data-provider');
+const { recordMessage, getMessages, spendTokens, saveConvo } = require('~/models');
+const { retrieveAndProcessFile } = require('~/server/services/Files/process');
+
+/**
+ * Initializes a new thread or adds messages to an existing thread.
+ *
+ * @param {Object} params - The parameters for initializing a thread.
+ * @param {OpenAIClient} params.openai - The OpenAI client instance.
+ * @param {Object} params.body - The body of the request.
+ * @param {ThreadMessage[]} params.body.messages - A list of messages to start the thread with.
+ * @param {Object} [params.body.metadata] - Optional metadata for the thread.
+ * @param {string} [params.thread_id] - Optional existing thread ID. If provided, a message will be added to this thread.
+ * @return {Promise<Thread>} A promise that resolves to the newly created thread object or the updated thread object.
+ */
+async function initThread({ openai, body, thread_id: _thread_id }) {
+  let thread = {};
+  const messages = [];
+  if (_thread_id) {
+    const message = await openai.beta.threads.messages.create(_thread_id, body.messages[0]);
+    messages.push(message);
+  } else {
+    thread = await openai.beta.threads.create(body);
+  }
+
+  const thread_id = _thread_id || thread.id;
+  return { messages, thread_id, ...thread };
+}
+
+/**
+ * Saves a user message to the DB in the Assistants endpoint format.
+ *
+ * @param {Object} req - The request object.
+ * @param {Object} params - The parameters of the user message
+ * @param {string} params.user - The user's ID.
+ * @param {string} params.text - The user's prompt.
+ * @param {string} params.messageId - The user message Id.
+ * @param {string} params.model - The model used by the assistant.
+ * @param {string} params.assistant_id - The current assistant Id.
+ * @param {string} params.thread_id - The thread Id.
+ * @param {string} params.conversationId - The message's conversationId
+ * @param {string} params.endpoint - The conversation endpoint
+ * @param {string} [params.parentMessageId] - Optional if initial message.
+ * Defaults to Constants.NO_PARENT.
+ * @param {string} [params.instructions] - Optional: from preset for `instructions` field.
+ * Overrides the instructions of the assistant.
+ * @param {string} [params.promptPrefix] - Optional: from preset for `additional_instructions` field.
+ * @param {import('librechat-data-provider').TFile[]} [params.files] - Optional. List of Attached File Objects.
+ * @param {string[]} [params.file_ids] - Optional. List of File IDs attached to the userMessage.
+ * @return {Promise<Run>} A promise that resolves to the created run object.
+ */
+async function saveUserMessage(req, params) {
+  const tokenCount = await countTokens(params.text);
+
+  const userMessage = {
+    user: params.user,
+    endpoint: params.endpoint,
+    messageId: params.messageId,
+    conversationId: params.conversationId,
+    parentMessageId: params.parentMessageId ?? Constants.NO_PARENT,
+    /* For messages, use the assistant_id instead of model */
+    model: params.assistant_id,
+    thread_id: params.thread_id,
+    sender: 'User',
+    text: params.text,
+    isCreatedByUser: true,
+    tokenCount,
+  };
+
+  const convo = {
+    endpoint: params.endpoint,
+    conversationId: params.conversationId,
+    promptPrefix: params.promptPrefix,
+    instructions: params.instructions,
+    assistant_id: params.assistant_id,
+    model: params.model,
+  };
+
+  if (params.files?.length) {
+    userMessage.files = params.files.map(({ file_id }) => ({ file_id }));
+    convo.file_ids = params.file_ids;
+  }
+
+  const message = await recordMessage(userMessage);
+  await saveConvo(
+    {
+      userId: req?.user?.id,
+      isTemporary: req?.body?.isTemporary,
+      interfaceConfig: req?.config?.interfaceConfig,
+    },
+    convo,
+    { context: 'api/server/services/Threads/manage.js #saveUserMessage' },
+  );
+  return message;
+}
+
+/**
+ * Saves an Assistant message to the DB in the Assistants endpoint format.
+ *
+ * @param {Object} req - The request object.
+ * @param {Object} params - The parameters of the Assistant message
+ * @param {string} params.user - The user's ID.
+ * @param {string} params.messageId - The message Id.
+ * @param {string} params.text - The concatenated text of the message.
+ * @param {string} params.assistant_id - The assistant Id.
+ * @param {string} params.thread_id - The thread Id.
+ * @param {string} params.model - The model used by the assistant.
+ * @param {ContentPart[]} params.content - The message content parts.
+ * @param {string} params.conversationId - The message's conversationId
+ * @param {string} params.endpoint - The conversation endpoint
+ * @param {string} params.parentMessageId - The latest user message that triggered this response.
+ * @param {string} [params.instructions] - Optional: from preset for `instructions` field.
+ * @param {string} [params.spec] - Optional: Model spec identifier.
+ * @param {string} [params.iconURL]
+ * Overrides the instructions of the assistant.
+ * @param {string} [params.promptPrefix] - Optional: from preset for `additional_instructions` field.
+ * @return {Promise<Run>} A promise that resolves to the created run object.
+ */
+async function saveAssistantMessage(req, params) {
+  // const tokenCount = // TODO: need to count each content part
+
+  const message = await recordMessage({
+    user: params.user,
+    endpoint: params.endpoint,
+    messageId: params.messageId,
+    conversationId: params.conversationId,
+    parentMessageId: params.parentMessageId,
+    thread_id: params.thread_id,
+    /* For messages, use the assistant_id instead of model */
+    model: params.assistant_id,
+    content: params.content,
+    sender: 'Assistant',
+    isCreatedByUser: false,
+    text: params.text,
+    unfinished: false,
+    // tokenCount,
+    iconURL: params.iconURL,
+    spec: params.spec,
+  });
+
+  await saveConvo(
+    {
+      userId: req?.user?.id,
+      isTemporary: req?.body?.isTemporary,
+      interfaceConfig: req?.config?.interfaceConfig,
+    },
+    {
+      endpoint: params.endpoint,
+      conversationId: params.conversationId,
+      promptPrefix: params.promptPrefix,
+      instructions: params.instructions,
+      assistant_id: params.assistant_id,
+      model: params.model,
+      iconURL: params.iconURL,
+      spec: params.spec,
+    },
+    { context: 'api/server/services/Threads/manage.js #saveAssistantMessage' },
+  );
+
+  return message;
+}
+
+/**
+ * Records LibreChat messageId to all response messages' metadata
+ *
+ * @param {Object} params - The parameters for initializing a thread.
+ * @param {OpenAIClient} params.openai - The OpenAI client instance.
+ * @param {string} params.thread_id - Response thread ID.
+ * @param {string} params.messageId - The response `messageId` generated by LibreChat.
+ * @param {StepMessage[] | Message[]} params.messages - A list of messages to start the thread with.
+ * @return {Promise<ThreadMessage[]>} A promise that resolves to the updated messages
+ */
+async function addThreadMetadata({ openai, thread_id, messageId, messages }) {
+  const promises = [];
+  for (const message of messages) {
+    promises.push(
+      openai.beta.threads.messages.update(message.id, {
+        thread_id,
+        metadata: {
+          messageId,
+        },
+      }),
+    );
+  }
+
+  return await Promise.all(promises);
+}
+
+/**
+ * Synchronizes LibreChat messages to Thread Messages.
+ * Updates the LibreChat DB with any missing Thread Messages and
+ * updates the missing Thread Messages' metadata with their corresponding db messageId's.
+ *
+ * Also updates the existing conversation's file_ids with any new file_ids.
+ *
+ * @param {Object} params - The parameters for synchronizing messages.
+ * @param {OpenAIClient} params.openai - The OpenAI client instance.
+ * @param {string} params.endpoint - The current endpoint.
+ * @param {string} params.thread_id - The current thread ID.
+ * @param {TMessage[]} params.dbMessages - The LibreChat DB messages.
+ * @param {ThreadMessage[]} params.apiMessages - The thread messages from the API.
+ * @param {string} [params.assistant_id] - The current assistant ID.
+ * @param {string} params.conversationId - The current conversation ID.
+ * @return {Promise<TMessage[]>} A promise that resolves to the updated messages
+ */
+async function syncMessages({
+  openai,
+  endpoint,
+  thread_id,
+  dbMessages,
+  apiMessages,
+  assistant_id,
+  conversationId,
+}) {
+  let result = [];
+  let dbMessageMap = new Map(dbMessages.map((msg) => [msg.messageId, msg]));
+
+  const modifyPromises = [];
+  const recordPromises = [];
+
+  /**
+   *
+   * Modify API message and save newMessage to DB
+   *
+   * @param {Object} params - The parameters object
+   * @param {TMessage} params.dbMessage
+   * @param {dbMessage} params.apiMessage
+   */
+  const processNewMessage = async ({ dbMessage, apiMessage }) => {
+    recordPromises.push(recordMessage({ ...dbMessage, user: openai.req.user.id }));
+
+    if (!apiMessage.id.includes('msg_')) {
+      return;
+    }
+
+    if (dbMessage.aggregateMessages?.length > 1) {
+      modifyPromises.push(
+        addThreadMetadata({
+          openai,
+          thread_id,
+          messageId: dbMessage.messageId,
+          messages: dbMessage.aggregateMessages,
+        }),
+      );
+      return;
+    }
+
+    modifyPromises.push(
+      openai.beta.threads.messages.update(apiMessage.id, {
+        thread_id,
+        metadata: {
+          messageId: dbMessage.messageId,
+        },
+      }),
+    );
+  };
+
+  let lastMessage = null;
+
+  for (let i = 0; i < apiMessages.length; i++) {
+    const apiMessage = apiMessages[i];
+
+    // Check if the message exists in the database based on metadata
+    const dbMessageId = apiMessage.metadata && apiMessage.metadata.messageId;
+    let dbMessage = dbMessageMap.get(dbMessageId);
+
+    if (dbMessage) {
+      // If message exists in DB, use its messageId and update parentMessageId
+      dbMessage.parentMessageId = lastMessage ? lastMessage.messageId : Constants.NO_PARENT;
+      lastMessage = dbMessage;
+      result.push(dbMessage);
+      continue;
+    }
+
+    if (apiMessage.role === 'assistant' && lastMessage && lastMessage.role === 'assistant') {
+      // Aggregate assistant messages
+      lastMessage.content = [...lastMessage.content, ...apiMessage.content];
+      lastMessage.files = [...(lastMessage.files ?? []), ...(apiMessage.files ?? [])];
+      lastMessage.aggregateMessages.push({ id: apiMessage.id });
+    } else {
+      // Handle new or missing message
+      const newMessage = {
+        thread_id,
+        conversationId,
+        messageId: v4(),
+        endpoint,
+        parentMessageId: lastMessage ? lastMessage.messageId : Constants.NO_PARENT,
+        role: apiMessage.role,
+        isCreatedByUser: apiMessage.role === 'user',
+        // TODO: process generated files in content parts
+        content: apiMessage.content,
+        aggregateMessages: [{ id: apiMessage.id }],
+        model: apiMessage.role === 'user' ? null : apiMessage.assistant_id,
+        user: openai.req.user.id,
+        unfinished: false,
+      };
+
+      if (apiMessage.file_ids?.length) {
+        // TODO: retrieve file objects from API
+        newMessage.files = apiMessage.file_ids.map((file_id) => ({ file_id }));
+      }
+
+      /* Assign assistant_id if defined */
+      if (assistant_id && apiMessage.role === 'assistant' && !newMessage.model) {
+        apiMessage.model = assistant_id;
+        newMessage.model = assistant_id;
+      }
+
+      result.push(newMessage);
+      lastMessage = newMessage;
+
+      if (apiMessage.role === 'user') {
+        processNewMessage({ dbMessage: newMessage, apiMessage });
+        continue;
+      }
+    }
+
+    const nextMessage = apiMessages[i + 1];
+    const processAssistant = !nextMessage || nextMessage.role === 'user';
+
+    if (apiMessage.role === 'assistant' && processAssistant) {
+      processNewMessage({ dbMessage: lastMessage, apiMessage });
+    }
+  }
+
+  const attached_file_ids = apiMessages.reduce((acc, msg) => {
+    if (msg.role === 'user' && msg.file_ids?.length) {
+      return [...acc, ...msg.file_ids];
+    }
+
+    return acc;
+  }, []);
+
+  await Promise.all(modifyPromises);
+  await Promise.all(recordPromises);
+
+  await saveConvo(
+    {
+      userId: openai.req?.user?.id,
+      isTemporary: openai.req?.body?.isTemporary,
+      interfaceConfig: openai.req?.config?.interfaceConfig,
+    },
+    {
+      conversationId,
+      file_ids: attached_file_ids,
+    },
+    { context: 'api/server/services/Threads/manage.js #syncMessages' },
+  );
+
+  return result;
+}
+
+/**
+ * Maps messages to their corresponding steps. Steps with message creation will be paired with their messages,
+ * while steps without message creation will be returned as is.
+ *
+ * @param {RunStep[]} steps - An array of steps from the run.
+ * @param {Message[]} messages - An array of message objects.
+ * @returns {(StepMessage | RunStep)[]} An array where each element is either a step with its corresponding message (StepMessage) or a step without a message (RunStep).
+ */
+function mapMessagesToSteps(steps, messages) {
+  // Create a map of messages indexed by their IDs for efficient lookup
+  const messageMap = messages.reduce((acc, msg) => {
+    acc[msg.id] = msg;
+    return acc;
+  }, {});
+
+  // Map each step to its corresponding message, or return the step as is if no message ID is present
+  return steps
+    .sort((a, b) => a.created_at - b.created_at)
+    .map((step) => {
+      const messageId = step.step_details?.message_creation?.message_id;
+
+      if (messageId && messageMap[messageId]) {
+        return { step, message: messageMap[messageId] };
+      }
+      return step;
+    });
+}
+
+/**
+ * Checks for any missing messages; if missing,
+ * synchronizes LibreChat messages to Thread Messages
+ *
+ * @param {Object} params - The parameters for initializing a thread.
+ * @param {OpenAIClient} params.openai - The OpenAI client instance.
+ * @param {string} params.endpoint - The current endpoint.
+ * @param {string} [params.latestMessageId] - Optional: The latest message ID from LibreChat.
+ * @param {string} params.thread_id - Response thread ID.
+ * @param {string} params.run_id - Response Run ID.
+ * @param {string} params.conversationId - LibreChat conversation ID.
+ * @return {Promise<TMessage[]>} A promise that resolves to the updated messages
+ */
+async function checkMessageGaps({
+  openai,
+  endpoint,
+  latestMessageId,
+  thread_id,
+  run_id,
+  conversationId,
+}) {
+  const promises = [];
+  promises.push(openai.beta.threads.messages.list(thread_id, defaultOrderQuery));
+  promises.push(openai.beta.threads.runs.steps.list(run_id, { thread_id }));
+  /** @type {[{ data: ThreadMessage[] }, { data: RunStep[] }]} */
+  const [response, stepsResponse] = await Promise.all(promises);
+
+  const steps = mapMessagesToSteps(stepsResponse.data, response.data);
+  /** @type {ThreadMessage} */
+  const currentMessage = {
+    id: v4(),
+    content: [],
+    assistant_id: null,
+    created_at: Math.floor(new Date().getTime() / 1000),
+    object: 'thread.message',
+    role: 'assistant',
+    run_id,
+    thread_id,
+    endpoint,
+    metadata: {
+      messageId: latestMessageId,
+    },
+  };
+
+  for (const step of steps) {
+    if (!currentMessage.assistant_id && step.assistant_id) {
+      currentMessage.assistant_id = step.assistant_id;
+    }
+    if (step.message) {
+      currentMessage.id = step.message.id;
+      currentMessage.created_at = step.message.created_at;
+      currentMessage.content = currentMessage.content.concat(step.message.content);
+    } else if (step.step_details?.type === 'tool_calls' && step.step_details?.tool_calls?.length) {
+      currentMessage.content = currentMessage.content.concat(
+        step.step_details?.tool_calls.map((toolCall) => ({
+          [ContentTypes.TOOL_CALL]: {
+            ...toolCall,
+            progress: 2,
+          },
+          type: ContentTypes.TOOL_CALL,
+        })),
+      );
+    }
+  }
+
+  let addedCurrentMessage = false;
+  const apiMessages = response.data
+    .map((msg) => {
+      if (msg.id === currentMessage.id) {
+        addedCurrentMessage = true;
+        return currentMessage;
+      }
+      return msg;
+    })
+    .sort((a, b) => new Date(a.created_at) - new Date(b.created_at));
+
+  if (!addedCurrentMessage) {
+    apiMessages.push(currentMessage);
+  }
+
+  const dbMessages = await getMessages({ conversationId, user: openai.req.user.id });
+  const assistant_id = dbMessages?.[0]?.model;
+
+  const syncedMessages = await syncMessages({
+    openai,
+    endpoint,
+    thread_id,
+    dbMessages,
+    apiMessages,
+    assistant_id,
+    conversationId,
+  });
+
+  return Object.values(
+    [...dbMessages, ...syncedMessages].reduce(
+      (acc, message) => ({ ...acc, [message.messageId]: message }),
+      {},
+    ),
+  );
+}
+
+/**
+ * Records token usage for a given completion request.
+ * @param {Object} params - The parameters for initializing a thread.
+ * @param {number} params.prompt_tokens - The number of prompt tokens used.
+ * @param {number} params.completion_tokens - The number of completion tokens used.
+ * @param {string} params.model - The model used by the assistant run.
+ * @param {string} params.user - The user's ID.
+ * @param {string} params.conversationId - LibreChat conversation ID.
+ * @param {string} [params.context='message'] - The context of the usage. Defaults to 'message'.
+ * @return {Promise<TMessage[]>} A promise that resolves to the updated messages
+ */
+const recordUsage = async ({
+  prompt_tokens,
+  completion_tokens,
+  model,
+  user,
+  conversationId,
+  context = 'message',
+}) => {
+  await spendTokens(
+    {
+      user,
+      model,
+      context,
+      conversationId,
+    },
+    { promptTokens: prompt_tokens, completionTokens: completion_tokens },
+  );
+};
+
+const uniqueCitationStart = '^====||===';
+const uniqueCitationEnd = '==|||||^';
+
+/**
+ * Sorts, processes, and flattens messages to a single string.
+ *
+ * @param {object} params - The parameters for processing messages.
+ * @param {OpenAIClient} params.openai - The OpenAI client instance.
+ * @param {RunClient} params.client - The LibreChat client that manages the run: either refers to `OpenAI` or `StreamRunManager`.
+ * @param {ThreadMessage[]} params.messages - An array of messages.
+ * @returns {Promise<{messages: ThreadMessage[], text: string, edited: boolean}>} The sorted messages, the flattened text, and whether it was edited.
+ */
+async function processMessages({ openai, client, messages = [] }) {
+  const sorted = messages.sort((a, b) => a.created_at - b.created_at);
+
+  let text = '';
+  let edited = false;
+  const sources = new Map();
+  const fileRetrievalPromises = [];
+
+  for (const message of sorted) {
+    message.files = [];
+    for (const content of message.content) {
+      const type = content.type;
+      const contentType = content[type];
+      const currentFileId = contentType?.file_id;
+
+      if (type === ContentTypes.IMAGE_FILE && !client.processedFileIds.has(currentFileId)) {
+        fileRetrievalPromises.push(
+          retrieveAndProcessFile({
+            openai,
+            client,
+            file_id: currentFileId,
+            basename: `${currentFileId}.png`,
+          })
+            .then((file) => {
+              client.processedFileIds.add(currentFileId);
+              message.files.push(file);
+            })
+            .catch((error) => {
+              console.error(`Failed to retrieve file: ${error.message}`);
+            }),
+        );
+        continue;
+      }
+
+      let currentText = contentType?.value ?? '';
+
+      /** @type {{ annotations: Annotation[] }} */
+      const { annotations } = contentType ?? {};
+
+      if (!annotations?.length) {
+        text += currentText;
+        continue;
+      }
+
+      const replacements = [];
+      const annotationPromises = annotations.map(async (annotation) => {
+        const type = annotation.type;
+        const annotationType = annotation[type];
+        const file_id = annotationType?.file_id;
+        const alreadyProcessed = client.processedFileIds.has(file_id);
+
+        let file;
+        let replacementText = '';
+
+        try {
+          if (alreadyProcessed) {
+            file = await retrieveAndProcessFile({ openai, client, file_id, unknownType: true });
+          } else if (type === AnnotationTypes.FILE_PATH) {
+            const basename = path.basename(annotation.text);
+            file = await retrieveAndProcessFile({
+              openai,
+              client,
+              file_id,
+              basename,
+            });
+            replacementText = file.filepath;
+          } else if (type === AnnotationTypes.FILE_CITATION && file_id) {
+            file = await retrieveAndProcessFile({
+              openai,
+              client,
+              file_id,
+              unknownType: true,
+            });
+            if (file && file.filename) {
+              if (!sources.has(file.filename)) {
+                sources.set(file.filename, sources.size + 1);
+              }
+              replacementText = `${uniqueCitationStart}${sources.get(
+                file.filename,
+              )}${uniqueCitationEnd}`;
+            }
+          }
+
+          if (file && replacementText) {
+            replacements.push({
+              start: annotation.start_index,
+              end: annotation.end_index,
+              text: replacementText,
+            });
+            edited = true;
+            if (!alreadyProcessed) {
+              client.processedFileIds.add(file_id);
+              message.files.push(file);
+            }
+          }
+        } catch (error) {
+          console.error(`Failed to process annotation: ${error.message}`);
+        }
+      });
+
+      await Promise.all(annotationPromises);
+
+      // Apply replacements in reverse order
+      replacements.sort((a, b) => b.start - a.start);
+      for (const { start, end, text: replacementText } of replacements) {
+        currentText = currentText.slice(0, start) + replacementText + currentText.slice(end);
+      }
+
+      text += currentText;
+    }
+  }
+
+  await Promise.all(fileRetrievalPromises);
+
+  // Handle adjacent identical citations with the unique format
+  const adjacentCitationRegex = new RegExp(
+    `${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(
+      uniqueCitationEnd,
+    )}(\\s*)${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(uniqueCitationEnd)}`,
+    'g',
+  );
+  text = text.replace(adjacentCitationRegex, (match, num1, space, num2) => {
+    return num1 === num2
+      ? `${uniqueCitationStart}${num1}${uniqueCitationEnd}`
+      : `${uniqueCitationStart}${num1}${uniqueCitationEnd}${space}${uniqueCitationStart}${num2}${uniqueCitationEnd}`;
+  });
+
+  // Remove any remaining adjacent identical citations
+  const remainingAdjacentRegex = new RegExp(
+    `(${escapeRegExp(uniqueCitationStart)}(\\d+)${escapeRegExp(uniqueCitationEnd)})\\s*\\1+`,
+    'g',
+  );
+  text = text.replace(remainingAdjacentRegex, '$1');
+
+  // Replace the unique citation format with the final format
+  text = text.replace(new RegExp(escapeRegExp(uniqueCitationStart), 'g'), '^');
+  text = text.replace(new RegExp(escapeRegExp(uniqueCitationEnd), 'g'), '^');
+
+  if (sources.size) {
+    text += '\n\n';
+    Array.from(sources.entries()).forEach(([source, index], arrayIndex) => {
+      text += `^${index}.^ ${source}${arrayIndex === sources.size - 1 ? '' : '\n'}`;
+    });
+  }
+
+  return { messages: sorted, text, edited };
+}
+
+module.exports = {
+  initThread,
+  recordUsage,
+  processMessages,
+  saveUserMessage,
+  checkMessageGaps,
+  addThreadMetadata,
+  mapMessagesToSteps,
+  saveAssistantMessage,
+};

api/server/services/Threads/processMessages.spec.js

@@ -0,0 +1,983 @@
+const { retrieveAndProcessFile } = require('~/server/services/Files/process');
+const { processMessages } = require('./manage');
+
+jest.mock('~/server/services/Files/process', () => ({
+  retrieveAndProcessFile: jest.fn(),
+}));
+
+describe('processMessages', () => {
+  let openai, client;
+
+  beforeEach(() => {
+    openai = {};
+    client = {
+      processedFileIds: new Set(),
+    };
+    jest.clearAllMocks();
+    retrieveAndProcessFile.mockReset();
+  });
+
+  test('handles normal case with single source', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'This is a test ^1^ and another^1^',
+              annotations: [
+                {
+                  type: 'file_citation',
+                  start_index: 15,
+                  end_index: 18,
+                  file_citation: { file_id: 'file1' },
+                },
+                {
+                  type: 'file_citation',
+                  start_index: 30,
+                  end_index: 33,
+                  file_citation: { file_id: 'file1' },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValue({ filename: 'test.txt' });
+
+    const result = await processMessages({ openai, client, messages });
+
+    expect(result.text).toBe('This is a test ^1^ and another^1^\n\n^1.^ test.txt');
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles multiple different sources', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'This is a test ^1^ and another^2^',
+              annotations: [
+                {
+                  type: 'file_citation',
+                  start_index: 15,
+                  end_index: 18,
+                  file_citation: { file_id: 'file1' },
+                },
+                {
+                  type: 'file_citation',
+                  start_index: 30,
+                  end_index: 33,
+                  file_citation: { file_id: 'file2' },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile
+      .mockResolvedValueOnce({ filename: 'test1.txt' })
+      .mockResolvedValueOnce({ filename: 'test2.txt' });
+
+    const result = await processMessages({ openai, client, messages });
+
+    expect(result.text).toBe('This is a test ^1^ and another^2^\n\n^1.^ test1.txt\n^2.^ test2.txt');
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles file retrieval failure', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'This is a test ^1^',
+              annotations: [
+                {
+                  type: 'file_citation',
+                  start_index: 15,
+                  end_index: 18,
+                  file_citation: { file_id: 'file1' },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockRejectedValue(new Error('File not found'));
+
+    const result = await processMessages({ openai, client, messages });
+
+    expect(result.text).toBe('This is a test ^1^');
+    expect(result.edited).toBe(false);
+  });
+
+  test('handles citations without file ids', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'This is a test ^1^',
+              annotations: [{ type: 'file_citation', start_index: 15, end_index: 18 }],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    const result = await processMessages({ openai, client, messages });
+
+    expect(result.text).toBe('This is a test ^1^');
+    expect(result.edited).toBe(false);
+  });
+
+  test('handles mixed valid and invalid citations', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'This is a test ^1^ and ^2^ and ^3^',
+              annotations: [
+                {
+                  type: 'file_citation',
+                  start_index: 15,
+                  end_index: 18,
+                  file_citation: { file_id: 'file1' },
+                },
+                { type: 'file_citation', start_index: 23, end_index: 26 },
+                {
+                  type: 'file_citation',
+                  start_index: 31,
+                  end_index: 34,
+                  file_citation: { file_id: 'file3' },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile
+      .mockResolvedValueOnce({ filename: 'test1.txt' })
+      .mockResolvedValueOnce({ filename: 'test3.txt' });
+
+    const result = await processMessages({ openai, client, messages });
+
+    expect(result.text).toBe(
+      'This is a test ^1^ and ^2^ and ^2^\n\n^1.^ test1.txt\n^2.^ test3.txt',
+    );
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles adjacent identical citations', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'This is a test ^1^^1^ and ^1^ ^1^',
+              annotations: [
+                {
+                  type: 'file_citation',
+                  start_index: 15,
+                  end_index: 18,
+                  file_citation: { file_id: 'file1' },
+                },
+                {
+                  type: 'file_citation',
+                  start_index: 18,
+                  end_index: 21,
+                  file_citation: { file_id: 'file1' },
+                },
+                {
+                  type: 'file_citation',
+                  start_index: 26,
+                  end_index: 29,
+                  file_citation: { file_id: 'file1' },
+                },
+                {
+                  type: 'file_citation',
+                  start_index: 30,
+                  end_index: 33,
+                  file_citation: { file_id: 'file1' },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValue({ filename: 'test.txt' });
+
+    const result = await processMessages({ openai, client, messages });
+
+    expect(result.text).toBe('This is a test ^1^ and ^1^\n\n^1.^ test.txt');
+    expect(result.edited).toBe(true);
+  });
+  test('handles real data with multiple adjacent citations', async () => {
+    const messages = [
+      {
+        id: 'msg_XXXXXXXXXXXXXXXXXXXX',
+        object: 'thread.message',
+        created_at: 1722980324,
+        assistant_id: 'asst_XXXXXXXXXXXXXXXXXXXX',
+        thread_id: 'thread_XXXXXXXXXXXXXXXXXXXX',
+        run_id: 'run_XXXXXXXXXXXXXXXXXXXX',
+        status: 'completed',
+        incomplete_details: null,
+        incomplete_at: null,
+        completed_at: 1722980331,
+        role: 'assistant',
+        content: [
+          {
+            type: 'text',
+            text: {
+              value:
+                "The text you have uploaded is from the book \"Harry Potter and the Philosopher's Stone\" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:\n\n1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry【11:2†source】【11:4†source】.\n\n2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander's【11:9†source】【11:14†source】.\n\n3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background【11:12†source】【11:18†source】.\n\n4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy【11:16†source】.\n\n5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher's Stone and its connection to the dark wizard Voldemort【11:1†source】【11:10†source】【11:7†source】.\n\nThese points highlight Harry's initial experiences in the magical world and set the stage for his adventures at Hogwarts.",
+              annotations: [
+                {
+                  type: 'file_citation',
+                  text: '【11:2†source】',
+                  start_index: 420,
+                  end_index: 433,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:4†source】',
+                  start_index: 433,
+                  end_index: 446,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:9†source】',
+                  start_index: 578,
+                  end_index: 591,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:14†source】',
+                  start_index: 591,
+                  end_index: 605,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:12†source】',
+                  start_index: 767,
+                  end_index: 781,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:18†source】',
+                  start_index: 781,
+                  end_index: 795,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:16†source】',
+                  start_index: 935,
+                  end_index: 949,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:1†source】',
+                  start_index: 1114,
+                  end_index: 1127,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:10†source】',
+                  start_index: 1127,
+                  end_index: 1141,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:7†source】',
+                  start_index: 1141,
+                  end_index: 1154,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        attachments: [],
+        metadata: {},
+        files: [
+          {
+            object: 'file',
+            id: 'file-XXXXXXXXXXXXXXXXXXXX',
+            purpose: 'assistants',
+            filename: 'hp1.txt',
+            bytes: 439742,
+            created_at: 1722962139,
+            status: 'processed',
+            status_details: null,
+            type: 'text/plain',
+            file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+            filepath:
+              'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-XXXXXXXXXXXXXXXXXXXX/hp1.txt',
+            usage: 1,
+            user: 'XXXXXXXXXXXXXXXXXXXX',
+            context: 'assistants',
+            source: 'openai',
+            model: 'gpt-4o',
+          },
+        ],
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValue({ filename: 'hp1.txt' });
+
+    const result = await processMessages({
+      openai: {},
+      client: { processedFileIds: new Set() },
+      messages,
+    });
+
+    const expectedText = `The text you have uploaded is from the book "Harry Potter and the Philosopher's Stone" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:
+
+1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry^1^.
+
+2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander's^1^.
+
+3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background^1^.
+
+4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy^1^.
+
+5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher's Stone and its connection to the dark wizard Voldemort^1^.
+
+These points highlight Harry's initial experiences in the magical world and set the stage for his adventures at Hogwarts.
+
+^1.^ hp1.txt`;
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles real data with multiple adjacent citations with multiple sources', async () => {
+    const messages = [
+      {
+        id: 'msg_XXXXXXXXXXXXXXXXXXXX',
+        object: 'thread.message',
+        created_at: 1722980324,
+        assistant_id: 'asst_XXXXXXXXXXXXXXXXXXXX',
+        thread_id: 'thread_XXXXXXXXXXXXXXXXXXXX',
+        run_id: 'run_XXXXXXXXXXXXXXXXXXXX',
+        status: 'completed',
+        incomplete_details: null,
+        incomplete_at: null,
+        completed_at: 1722980331,
+        role: 'assistant',
+        content: [
+          {
+            type: 'text',
+            text: {
+              value:
+                "The text you have uploaded is from the book \"Harry Potter and the Philosopher's Stone\" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:\n\n1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry【11:2†source】【11:4†source】.\n\n2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander's【11:9†source】【11:14†source】.\n\n3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background【11:12†source】【11:18†source】.\n\n4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy【11:16†source】.\n\n5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher's Stone and its connection to the dark wizard Voldemort【11:1†source】【11:10†source】【11:7†source】.\n\nThese points highlight Harry's initial experiences in the magical world and set the stage for his adventures at Hogwarts.",
+              annotations: [
+                {
+                  type: 'file_citation',
+                  text: '【11:2†source】',
+                  start_index: 420,
+                  end_index: 433,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:4†source】',
+                  start_index: 433,
+                  end_index: 446,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:9†source】',
+                  start_index: 578,
+                  end_index: 591,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:14†source】',
+                  start_index: 591,
+                  end_index: 605,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:12†source】',
+                  start_index: 767,
+                  end_index: 781,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:18†source】',
+                  start_index: 781,
+                  end_index: 795,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:16†source】',
+                  start_index: 935,
+                  end_index: 949,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:1†source】',
+                  start_index: 1114,
+                  end_index: 1127,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:10†source】',
+                  start_index: 1127,
+                  end_index: 1141,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '【11:7†source】',
+                  start_index: 1141,
+                  end_index: 1154,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        attachments: [],
+        metadata: {},
+        files: [
+          {
+            object: 'file',
+            id: 'file-XXXXXXXXXXXXXXXXXXXX',
+            purpose: 'assistants',
+            filename: 'hp1.txt',
+            bytes: 439742,
+            created_at: 1722962139,
+            status: 'processed',
+            status_details: null,
+            type: 'text/plain',
+            file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+            filepath:
+              'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-XXXXXXXXXXXXXXXXXXXX/hp1.txt',
+            usage: 1,
+            user: 'XXXXXXXXXXXXXXXXXXXX',
+            context: 'assistants',
+            source: 'openai',
+            model: 'gpt-4o',
+          },
+        ],
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValue({ filename: 'hp1.txt' });
+
+    const result = await processMessages({
+      openai: {},
+      client: { processedFileIds: new Set() },
+      messages,
+    });
+
+    const expectedText = `The text you have uploaded is from the book "Harry Potter and the Philosopher's Stone" by J.K. Rowling. It follows the story of a young boy named Harry Potter who discovers that he is a wizard on his eleventh birthday. Here are some key points of the narrative:
+
+1. **Discovery and Invitation to Hogwarts**: Harry learns that he is a wizard and receives an invitation to attend Hogwarts School of Witchcraft and Wizardry^1^.
+
+2. **Shopping for Supplies**: Hagrid takes Harry to Diagon Alley to buy his school supplies, including his wand from Ollivander's^1^.
+
+3. **Introduction to Hogwarts**: Harry is introduced to Hogwarts, the magical school where he will learn about magic and discover more about his own background^1^.
+
+4. **Meeting Friends and Enemies**: At Hogwarts, Harry makes friends like Ron Weasley and Hermione Granger, and enemies like Draco Malfoy^1^.
+
+5. **Uncovering the Mystery**: Harry, along with Ron and Hermione, uncovers the mystery of the Philosopher's Stone and its connection to the dark wizard Voldemort^1^.
+
+These points highlight Harry's initial experiences in the magical world and set the stage for his adventures at Hogwarts.
+
+^1.^ hp1.txt`;
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles edge case with pre-existing citation-like text', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value:
+                "This is a test ^1^ with pre-existing citation-like text. Here's a real citation【11:2†source】.",
+              annotations: [
+                {
+                  type: 'file_citation',
+                  text: '【11:2†source】',
+                  start_index: 79,
+                  end_index: 92,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValue({ filename: 'test.txt' });
+
+    const result = await processMessages({
+      openai: {},
+      client: { processedFileIds: new Set() },
+      messages,
+    });
+
+    const expectedText =
+      "This is a test ^1^ with pre-existing citation-like text. Here's a real citation^1^.\n\n^1.^ test.txt";
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles FILE_PATH annotation type', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'Here is a file path: [file_path]',
+              annotations: [
+                {
+                  type: 'file_path',
+                  text: '[file_path]',
+                  start_index: 21,
+                  end_index: 32,
+                  file_path: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValue({
+      filename: 'test.txt',
+      filepath: '/path/to/test.txt',
+    });
+
+    const result = await processMessages({
+      openai: {},
+      client: { processedFileIds: new Set() },
+      messages,
+    });
+
+    const expectedText = 'Here is a file path: /path/to/test.txt';
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles FILE_CITATION annotation type', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'Here is a citation: [citation]',
+              annotations: [
+                {
+                  type: 'file_citation',
+                  text: '[citation]',
+                  start_index: 20,
+                  end_index: 30,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValue({ filename: 'test.txt' });
+
+    const result = await processMessages({
+      openai: {},
+      client: { processedFileIds: new Set() },
+      messages,
+    });
+
+    const expectedText = 'Here is a citation: ^1^\n\n^1.^ test.txt';
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles multiple annotation types in a single message', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value:
+                'File path: [file_path]. Citation: [citation1]. Another citation: [citation2].',
+              annotations: [
+                {
+                  type: 'file_path',
+                  text: '[file_path]',
+                  start_index: 11,
+                  end_index: 22,
+                  file_path: {
+                    file_id: 'file-XXXXXXXXXXXXXXXX1',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '[citation1]',
+                  start_index: 34,
+                  end_index: 45,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXX2',
+                  },
+                },
+                {
+                  type: 'file_citation',
+                  text: '[citation2]',
+                  start_index: 65,
+                  end_index: 76,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXX3',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockResolvedValueOnce({
+      filename: 'file1.txt',
+      filepath: '/path/to/file1.txt',
+    });
+    retrieveAndProcessFile.mockResolvedValueOnce({ filename: 'file2.txt' });
+    retrieveAndProcessFile.mockResolvedValueOnce({ filename: 'file3.txt' });
+
+    const result = await processMessages({
+      openai: {},
+      client: { processedFileIds: new Set() },
+      messages,
+    });
+
+    const expectedText =
+      'File path: /path/to/file1.txt. Citation: ^1^. Another citation: ^2^.\n\n^1.^ file2.txt\n^2.^ file3.txt';
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(true);
+  });
+
+  test('handles annotation processing failure', async () => {
+    const messages = [
+      {
+        content: [
+          {
+            type: 'text',
+            text: {
+              value: 'This citation will fail: [citation]',
+              annotations: [
+                {
+                  type: 'file_citation',
+                  text: '[citation]',
+                  start_index: 25,
+                  end_index: 35,
+                  file_citation: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        created_at: 1,
+      },
+    ];
+
+    retrieveAndProcessFile.mockRejectedValue(new Error('File not found'));
+
+    const result = await processMessages({
+      openai: {},
+      client: { processedFileIds: new Set() },
+      messages,
+    });
+
+    const expectedText = 'This citation will fail: [citation]';
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(false);
+  });
+
+  test('handles multiple FILE_PATH annotations with sandbox links', async () => {
+    const messages = [
+      {
+        id: 'msg_XXXXXXXXXXXXXXXXXXXX',
+        object: 'thread.message',
+        created_at: 1722983745,
+        assistant_id: 'asst_XXXXXXXXXXXXXXXXXXXX',
+        thread_id: 'thread_XXXXXXXXXXXXXXXXXXXX',
+        run_id: 'run_XXXXXXXXXXXXXXXXXXXX',
+        status: 'completed',
+        incomplete_details: null,
+        incomplete_at: null,
+        completed_at: 1722983747,
+        role: 'assistant',
+        content: [
+          {
+            type: 'text',
+            text: {
+              value:
+                'I have generated three dummy CSV files for you. You can download them using the links below:\n\n1. [Download Dummy Data 1](sandbox:/mnt/data/dummy_data1.csv)\n2. [Download Dummy Data 2](sandbox:/mnt/data/dummy_data2.csv)\n3. [Download Dummy Data 3](sandbox:/mnt/data/dummy_data3.csv)',
+              annotations: [
+                {
+                  type: 'file_path',
+                  text: 'sandbox:/mnt/data/dummy_data1.csv',
+                  start_index: 121,
+                  end_index: 154,
+                  file_path: {
+                    file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+                  },
+                },
+                {
+                  type: 'file_path',
+                  text: 'sandbox:/mnt/data/dummy_data2.csv',
+                  start_index: 183,
+                  end_index: 216,
+                  file_path: {
+                    file_id: 'file-YYYYYYYYYYYYYYYYYYYY',
+                  },
+                },
+                {
+                  type: 'file_path',
+                  text: 'sandbox:/mnt/data/dummy_data3.csv',
+                  start_index: 245,
+                  end_index: 278,
+                  file_path: {
+                    file_id: 'file-ZZZZZZZZZZZZZZZZZZZZ',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+        attachments: [
+          {
+            file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+            tools: [
+              {
+                type: 'code_interpreter',
+              },
+            ],
+          },
+          {
+            file_id: 'file-YYYYYYYYYYYYYYYYYYYY',
+            tools: [
+              {
+                type: 'code_interpreter',
+              },
+            ],
+          },
+          {
+            file_id: 'file-ZZZZZZZZZZZZZZZZZZZZ',
+            tools: [
+              {
+                type: 'code_interpreter',
+              },
+            ],
+          },
+        ],
+        metadata: {},
+        files: [
+          {
+            object: 'file',
+            id: 'file-XXXXXXXXXXXXXXXXXXXX',
+            purpose: 'assistants_output',
+            filename: 'dummy_data1.csv',
+            bytes: 1925,
+            created_at: 1722983746,
+            status: 'processed',
+            status_details: null,
+            type: 'text/csv',
+            file_id: 'file-XXXXXXXXXXXXXXXXXXXX',
+            filepath:
+              'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-XXXXXXXXXXXXXXXXXXXX/dummy_data1.csv',
+            usage: 1,
+            user: 'XXXXXXXXXXXXXXXXXXXX',
+            context: 'assistants_output',
+            source: 'openai',
+            model: 'gpt-4o-mini',
+          },
+          {
+            object: 'file',
+            id: 'file-YYYYYYYYYYYYYYYYYYYY',
+            purpose: 'assistants_output',
+            filename: 'dummy_data2.csv',
+            bytes: 4221,
+            created_at: 1722983746,
+            status: 'processed',
+            status_details: null,
+            type: 'text/csv',
+            file_id: 'file-YYYYYYYYYYYYYYYYYYYY',
+            filepath:
+              'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-YYYYYYYYYYYYYYYYYYYY/dummy_data2.csv',
+            usage: 1,
+            user: 'XXXXXXXXXXXXXXXXXXXX',
+            context: 'assistants_output',
+            source: 'openai',
+            model: 'gpt-4o-mini',
+          },
+          {
+            object: 'file',
+            id: 'file-ZZZZZZZZZZZZZZZZZZZZ',
+            purpose: 'assistants_output',
+            filename: 'dummy_data3.csv',
+            bytes: 3534,
+            created_at: 1722983747,
+            status: 'processed',
+            status_details: null,
+            type: 'text/csv',
+            file_id: 'file-ZZZZZZZZZZZZZZZZZZZZ',
+            filepath:
+              'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-ZZZZZZZZZZZZZZZZZZZZ/dummy_data3.csv',
+            usage: 1,
+            user: 'XXXXXXXXXXXXXXXXXXXX',
+            context: 'assistants_output',
+            source: 'openai',
+            model: 'gpt-4o-mini',
+          },
+        ],
+      },
+    ];
+
+    const mockClient = {
+      processedFileIds: new Set(),
+    };
+
+    // Mock the retrieveAndProcessFile function for each file
+    retrieveAndProcessFile.mockImplementation(({ file_id }) => {
+      const fileMap = {
+        'file-XXXXXXXXXXXXXXXXXXXX': {
+          filename: 'dummy_data1.csv',
+          filepath:
+            'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-XXXXXXXXXXXXXXXXXXXX/dummy_data1.csv',
+        },
+        'file-YYYYYYYYYYYYYYYYYYYY': {
+          filename: 'dummy_data2.csv',
+          filepath:
+            'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-YYYYYYYYYYYYYYYYYYYY/dummy_data2.csv',
+        },
+        'file-ZZZZZZZZZZZZZZZZZZZZ': {
+          filename: 'dummy_data3.csv',
+          filepath:
+            'https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-ZZZZZZZZZZZZZZZZZZZZ/dummy_data3.csv',
+        },
+      };
+
+      return Promise.resolve(fileMap[file_id]);
+    });
+
+    const result = await processMessages({ openai: {}, client: mockClient, messages });
+
+    const expectedText =
+      'I have generated three dummy CSV files for you. You can download them using the links below:\n\n1. [Download Dummy Data 1](https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-XXXXXXXXXXXXXXXXXXXX/dummy_data1.csv)\n2. [Download Dummy Data 2](https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-YYYYYYYYYYYYYYYYYYYY/dummy_data2.csv)\n3. [Download Dummy Data 3](https://api.openai.com/v1/files/XXXXXXXXXXXXXXXXXXXX/file-ZZZZZZZZZZZZZZZZZZZZ/dummy_data3.csv)';
+
+    expect(result.text).toBe(expectedText);
+    expect(result.edited).toBe(true);
+  });
+});