AnythingLLM | Your personal LLM trained on anything

. + const content = + root.querySelector("article") || + root.querySelector("main") || + root.querySelector('[role="main"]'); + if (content) root = content; + + const junkSelectors = [ + "script", + "style", + "noscript", + "nav", + "footer", + "header", + "aside", + "iframe", + "svg", + '[role="navigation"]', + '[role="banner"]', + '[role="contentinfo"]', + '[aria-hidden="true"]', + "[hidden]", + ]; + for (const sel of junkSelectors) { + root.querySelectorAll(sel).forEach((el) => el.remove()); + } + + for (const el of root.querySelectorAll("[style]")) { + const style = el.getAttribute("style") || ""; + if ( + /display\s*:\s*none/i.test(style) || + /visibility\s*:\s*hidden/i.test(style) + ) { + el.remove(); + } + } + + if (baseUrl) { + resolveUrls(root, baseUrl); + } + + stripCitations(root); + + const cleanedHtml = root.toString(); + + const turndown = new TurndownService({ + headingStyle: "atx", + codeBlockStyle: "fenced", + bulletListMarker: "-", + }); + turndown.remove(["script", "style", "noscript", "iframe", "svg"]); + + turndown.addRule("compactLinks", { + filter: "a", + replacement: function (content, node) { + const href = node.getAttribute("href"); + if (!href) return content; + const text = content.replace(/\s+/g, " ").trim(); + if (!text) return ""; + return `[${text}](${href})`; + }, + }); + + let markdown = turndown.turndown(cleanedHtml); + + // Strip links and images with excessively long URLs (200+ chars) + markdown = markdown.replace(/!\[[^\]]*\]$[^)]{200,}$\s*/g, ""); + markdown = markdown.replace(/\[[^\]]*\]$[^)]{200,}$/g, (match) => { + const textMatch = match.match(/\[([^\]]*)\]/); + return textMatch ? textMatch[1] : ""; + }); + + markdown = markdown.replace(/\[[\d]+\]/g, ""); + markdown = markdown.replace(/\[#cite[^\]]*\]/g, ""); + markdown = markdown.replace(/\[edit\]/gi, ""); + + markdown = markdown.replace(/\n{4,}/g, "\n\n\n").trim(); + return markdown; + } catch (error) { + console.error("htmlToMarkdown failed:", error); + try { + return parse(html).text.trim(); + } catch { + return ""; + } + } +} + +/** + * Resolve relative URLs for links and images. Strips base64 and + * ignored-basepath images entirely. Gives alt-less images a filename-based alt. + * @param {import('node-html-parser').HTMLElement} root + * @param {string} baseUrl + */ +function resolveUrls(root, baseUrl) { + for (const a of root.querySelectorAll("a[href]")) { + const href = a.getAttribute("href"); + if (!href || /^(https?:|mailto:|tel:|javascript:|#)/i.test(href)) continue; + try { + a.setAttribute("href", new URL(href, baseUrl).toString()); + } catch {} + } + + for (const img of root.querySelectorAll("img[src]")) { + const src = img.getAttribute("src"); + if (!src) { + img.remove(); + continue; + } + if (src.startsWith("data:")) { + img.remove(); + continue; + } + + const resolvedSrc = /^https?:/i.test(src) + ? src + : (() => { + try { + return new URL(src, baseUrl).toString(); + } catch { + return src; + } + })(); + + if (IGNORED_IMG_BASEPATHS.some((base) => resolvedSrc.startsWith(base))) { + img.remove(); + continue; + } + + if (!/^https?:/i.test(src)) { + try { + img.setAttribute("src", new URL(src, baseUrl).toString()); + } catch {} + } + + const alt = (img.getAttribute("alt") || "").trim(); + if (!alt) { + try { + const pathname = new URL(img.getAttribute("src")).pathname; + const filename = pathname.split("/").pop() || "image"; + img.setAttribute("alt", filename); + } catch { + img.setAttribute("alt", "image"); + } + } + } +} + +/** + * Strip citation/reference superscripts and reference sections + * commonly found on Wikipedia and similar sites. + * @param {import('node-html-parser').HTMLElement} root + */ +function stripCitations(root) { + for (const sup of root.querySelectorAll("sup.reference, sup.noprint")) { + sup.remove(); + } + + for (const sel of [ + ".reflist", + ".references", + ".refbegin", + "#References", + ".catlinks", + ".mw-authority-control", + ]) { + root.querySelectorAll(sel).forEach((el) => el.remove()); + } +} + +module.exports = { htmlToMarkdown }; diff --git a/collector/processLink/helpers/index.js b/collector/processLink/helpers/index.js new file mode 100644 index 0000000..dc00f8f --- /dev/null +++ b/collector/processLink/helpers/index.js @@ -0,0 +1,190 @@ +const path = require("path"); +const { validURL } = require("../../utils/url"); +const { processSingleFile } = require("../../processSingleFile"); +const { downloadURIToFile } = require("../../utils/downloadURIToFile"); +const { ACCEPTED_MIMES } = require("../../utils/constants"); +const { validYoutubeVideoUrl } = require("../../utils/url"); + +/** + * Parse a Content-Type header value and return the MIME type without charset or other parameters. + * @param {string|null} contentTypeHeader - The raw Content-Type header value + * @returns {string|null} - The MIME type (e.g., "application/pdf") or null + */ +function parseContentType(contentTypeHeader) { + if (!contentTypeHeader) return null; + return contentTypeHeader.toLowerCase().split(";")[0].trim() || null; +} + +/** + * Get the content type of a resource + * - Sends a HEAD request to the URL and returns the Content-Type header with a 5 second timeout + * @param {string} url - The URL to get the content type of + * @returns {Promise<{success: boolean, reason: string|null, contentType: string|null}>} - The content type of the resource + */ +async function getContentTypeFromURL(url) { + try { + if (!url || typeof url !== "string" || !validURL(url)) + return { success: false, reason: "Not a valid URL.", contentType: null }; + + const abortController = new AbortController(); + const timeout = setTimeout(() => { + abortController.abort(); + console.error("Timeout fetching content type for URL:", url.toString()); + }, 5_000); + + const res = await fetch(url, { + method: "HEAD", + signal: abortController.signal, + }).finally(() => clearTimeout(timeout)); + + if (!res.ok) + return { + success: false, + reason: `HTTP ${res.status}: ${res.statusText}`, + contentType: null, + }; + + const contentTypeWithoutCharset = parseContentType( + res.headers.get("Content-Type") + ); + if (!contentTypeWithoutCharset) + return { + success: false, + reason: "No Content-Type found.", + contentType: null, + }; + return { + success: true, + reason: null, + contentType: contentTypeWithoutCharset, + }; + } catch (error) { + return { + success: false, + reason: `Error: ${error.message}`, + contentType: null, + }; + } +} + +/** + * Normalize the result object based on the saveAsDocument flag + * @param {Object} result - The result object to normalize + * @param {boolean} result.success - Whether the result is successful + * @param {string|null} result.reason - The reason for the result + * @param {Object[]} result.documents - The documents from the result + * @param {string|null} result.content - The content of the result + * @param {boolean} result.saveAsDocument - Whether to save the content as a document. Default is true + * @returns {{success: boolean, reason: string|null, documents: Object[], content: string|null}} - The normalized result object + */ +function returnResult({ + success, + reason, + documents, + content, + saveAsDocument = true, +} = {}) { + if (!saveAsDocument) { + return { + success, + content, + }; + } else return { success, reason, documents }; +} + +/** + * Determine the content type of a link - should be a URL + * @param {string} uri - The link to determine the content type of + * @returns {Promise<{contentType: string|null, processVia: 'web' | 'file' | 'youtube'}>} - The content type of the link + */ +async function determineContentType(uri) { + let processVia = "web"; + + // Dont check for content type if it is a YouTube video URL + if (validYoutubeVideoUrl(uri)) + return { contentType: "text/html", processVia: "youtube" }; + + return await getContentTypeFromURL(uri) + .then((result) => { + if (!!result.reason) console.error(result.reason); + + // If the content type is not text/html or text/plain, and it is in the ACCEPTED_MIMES, + // then we can process it as a file + if ( + !!result.contentType && + !["text/html", "text/plain"].includes(result.contentType) && + result.contentType in ACCEPTED_MIMES + ) + processVia = "file"; + + return { contentType: result.contentType, processVia }; + }) + .catch((error) => { + console.error("Error getting content type from URL", error); + return { contentType: null, processVia }; + }); +} + +/** + * Process a link as a file + * @param {string} uri - The link to process as a file + * @param {boolean} saveAsDocument - Whether to save the content as a document. Default is true + * @returns {Promise<{success: boolean, reason: string|null, documents: Object[], content: string|null, saveAsDocument: boolean}>} - The content of the file + */ +async function processAsFile({ uri, saveAsDocument = true }) { + const fileContentResult = await downloadURIToFile(uri); + if (!fileContentResult.success) + return returnResult({ + success: false, + reason: fileContentResult.reason, + documents: [], + content: null, + saveAsDocument, + }); + + const fileFilePath = fileContentResult.fileLocation; + const targetFilename = path.basename(fileFilePath); + + /** + * If the saveAsDocument is false, we are only interested in the text content + * and can ignore the file as a document by using `parseOnly` in the options. + * This will send the file to the Direct Uploads folder instead of the Documents folder. + * that will be deleted by the cleanup-orphan-documents job that runs frequently. The trade off + * is that since it still is in FS we can debug its output or even potentially reuse it for other purposes. + * + * TODO: Improve this process via a new option that will instantly delete the file after processing + * if we find we dont need this file ever after processing. + */ + const processSingleFileResult = await processSingleFile(targetFilename, { + parseOnly: saveAsDocument === false, + }); + if (!processSingleFileResult.success) { + return returnResult({ + success: false, + reason: processSingleFileResult.reason, + documents: [], + content: null, + saveAsDocument, + }); + } + + // If we intend to return only the text content, return the content from the file + // and then delete the file - otherwise it will be saved as a document + if (!saveAsDocument) { + return returnResult({ + success: true, + content: processSingleFileResult.documents[0].pageContent, + saveAsDocument, + }); + } + + return processSingleFileResult; +} + +module.exports = { + parseContentType, + returnResult, + getContentTypeFromURL, + determineContentType, + processAsFile, +}; diff --git a/collector/processLink/index.js b/collector/processLink/index.js new file mode 100644 index 0000000..ad71ca5 --- /dev/null +++ b/collector/processLink/index.js @@ -0,0 +1,47 @@ +const { validURL } = require("../utils/url"); +const { scrapeGenericUrl } = require("./convert/generic"); +const { validateURL } = require("../utils/url"); + +/** + * Process a link and return the text content. This util will save the link as a document + * so it can be used for embedding later. + * @param {string} link - The link to process + * @param {{[key: string]: string}} scraperHeaders - Custom headers to apply when scraping the link + * @param {Object} metadata - Optional metadata to attach to the document + * @returns {Promise<{success: boolean, content: string}>} - Response from collector + */ +async function processLink(link, scraperHeaders = {}, metadata = {}) { + const validatedLink = validateURL(link); + if (!validURL(validatedLink)) + return { success: false, reason: "Not a valid URL." }; + return await scrapeGenericUrl({ + link: validatedLink, + captureAs: "text", + scraperHeaders, + metadata, + saveAsDocument: true, + }); +} + +/** + * Get the text content of a link - does not save the link as a document + * Mostly used in agentic flows/tools calls to get the text content of a link + * @param {string} link - The link to get the text content of + * @param {('html' | 'text' | 'json')} captureAs - The format to capture the page content as + * @returns {Promise<{success: boolean, content: string}>} - Response from collector + */ +async function getLinkText(link, captureAs = "text") { + const validatedLink = validateURL(link); + if (!validURL(validatedLink)) + return { success: false, reason: "Not a valid URL." }; + return await scrapeGenericUrl({ + link: validatedLink, + captureAs, + saveAsDocument: false, + }); +} + +module.exports = { + processLink, + getLinkText, +}; diff --git a/collector/processRawText/index.js b/collector/processRawText/index.js new file mode 100644 index 0000000..f035ed9 --- /dev/null +++ b/collector/processRawText/index.js @@ -0,0 +1,83 @@ +const { v4 } = require("uuid"); +const { writeToServerDocuments } = require("../utils/files"); +const { tokenizeString } = require("../utils/tokenizer"); +const { default: slugify } = require("slugify"); + +// Will remove the last .extension from the input +// and stringify the input + move to lowercase. +function stripAndSlug(input) { + if (!input.includes(".")) return slugify(input, { lower: true }); + return slugify(input.split(".").slice(0, -1).join("-"), { lower: true }); +} + +const METADATA_KEYS = { + possible: { + url: ({ url, title }) => { + let validUrl; + try { + const u = new URL(url); + validUrl = ["https:", "http:"].includes(u.protocol); + } catch {} + + if (validUrl) return `web://${url.toLowerCase()}.website`; + return `file://${stripAndSlug(title)}.txt`; + }, + title: ({ title }) => `${stripAndSlug(title)}.txt`, + docAuthor: ({ docAuthor }) => { + return typeof docAuthor === "string" ? docAuthor : "no author specified"; + }, + description: ({ description }) => { + return typeof description === "string" + ? description + : "no description found"; + }, + docSource: ({ docSource }) => { + return typeof docSource === "string" ? docSource : "no source set"; + }, + chunkSource: ({ chunkSource, title }) => { + return typeof chunkSource === "string" + ? chunkSource + : `${stripAndSlug(title)}.txt`; + }, + published: ({ published }) => { + if (isNaN(Number(published))) return new Date().toLocaleString(); + return new Date(Number(published)).toLocaleString(); + }, + }, +}; + +async function processRawText(textContent, metadata) { + console.log(`-- Working Raw Text doc ${metadata.title} --`); + if (!textContent || textContent.length === 0) { + return { + success: false, + reason: "textContent was empty - nothing to process.", + documents: [], + }; + } + + const data = { + id: v4(), + url: METADATA_KEYS.possible.url(metadata), + title: METADATA_KEYS.possible.title(metadata), + docAuthor: METADATA_KEYS.possible.docAuthor(metadata), + description: METADATA_KEYS.possible.description(metadata), + docSource: METADATA_KEYS.possible.docSource(metadata), + chunkSource: METADATA_KEYS.possible.chunkSource(metadata), + published: METADATA_KEYS.possible.published(metadata), + wordCount: textContent.split(" ").length, + pageContent: textContent, + token_count_estimate: tokenizeString(textContent), + }; + + const document = writeToServerDocuments({ + data, + filename: `raw-${stripAndSlug(metadata.title)}-${data.id}`, + }); + console.log( + `[SUCCESS]: Raw text and metadata saved & ready for embedding.\n` + ); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = { processRawText }; diff --git a/collector/processSingleFile/convert/asAudio.js b/collector/processSingleFile/convert/asAudio.js new file mode 100644 index 0000000..5106106 --- /dev/null +++ b/collector/processSingleFile/convert/asAudio.js @@ -0,0 +1,83 @@ +const { v4 } = require("uuid"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../utils/files"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { default: slugify } = require("slugify"); +const { LocalWhisper } = require("../../utils/WhisperProviders/localWhisper"); +const { OpenAiWhisper } = require("../../utils/WhisperProviders/OpenAiWhisper"); +const { + GenericOpenAiWhisper, +} = require("../../utils/WhisperProviders/GenericOpenAiWhisper"); + +const WHISPER_PROVIDERS = { + openai: OpenAiWhisper, + "generic-openai": GenericOpenAiWhisper, + local: LocalWhisper, +}; + +async function asAudio({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + const WhisperProvider = WHISPER_PROVIDERS.hasOwnProperty( + options?.whisperProvider + ) + ? WHISPER_PROVIDERS[options?.whisperProvider] + : WHISPER_PROVIDERS.local; + + console.log(`-- Working ${filename} --`); + const whisper = new WhisperProvider({ options }); + const { content, error } = await whisper.processFile(fullFilePath, filename); + + if (!!error) { + console.error(`Error encountered for parsing of ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: error, + documents: [], + }; + } + + if (!content?.length) { + console.error(`Resulting text content was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No text content found in ${filename}.`, + documents: [], + }; + } + + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: metadata.title || filename, + docAuthor: metadata.docAuthor || "no author found", + description: metadata.description || "No description found.", + docSource: metadata.docSource || "audio file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}`, + options: { parseOnly: options.parseOnly }, + }); + if (!options.absolutePath) trashFile(fullFilePath); + console.log( + `[SUCCESS]: ${filename} transcribed, converted & ready for embedding.\n` + ); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = asAudio; diff --git a/collector/processSingleFile/convert/asDocx.js b/collector/processSingleFile/convert/asDocx.js new file mode 100644 index 0000000..2080756 --- /dev/null +++ b/collector/processSingleFile/convert/asDocx.js @@ -0,0 +1,63 @@ +const { v4 } = require("uuid"); +const { DocxLoader } = require("langchain/document_loaders/fs/docx"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../utils/files"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { default: slugify } = require("slugify"); + +async function asDocX({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + const loader = new DocxLoader(fullFilePath); + + console.log(`-- Working ${filename} --`); + let pageContent = []; + const docs = await loader.load(); + for (const doc of docs) { + console.log(`-- Parsing content from docx page --`); + if (!doc.pageContent.length) continue; + pageContent.push(doc.pageContent); + } + + if (!pageContent.length) { + console.error(`Resulting text content was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No text content found in ${filename}.`, + documents: [], + }; + } + + const content = pageContent.join(""); + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: metadata.title || filename, + docAuthor: metadata.docAuthor || "no author found", + description: metadata.description || "No description found.", + docSource: metadata.docSource || "docx file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}`, + options: { parseOnly: options.parseOnly }, + }); + if (!options.absolutePath) trashFile(fullFilePath); + console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = asDocX; diff --git a/collector/processSingleFile/convert/asEPub.js b/collector/processSingleFile/convert/asEPub.js new file mode 100644 index 0000000..49d3945 --- /dev/null +++ b/collector/processSingleFile/convert/asEPub.js @@ -0,0 +1,61 @@ +const { v4 } = require("uuid"); +const { EPubLoader } = require("langchain/document_loaders/fs/epub"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../utils/files"); +const { default: slugify } = require("slugify"); + +async function asEPub({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + let content = ""; + try { + const loader = new EPubLoader(fullFilePath, { splitChapters: false }); + const docs = await loader.load(); + docs.forEach((doc) => (content += doc.pageContent)); + } catch (err) { + console.error("Could not read epub file!", err); + } + + if (!content?.length) { + console.error(`Resulting text content was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No text content found in ${filename}.`, + documents: [], + }; + } + + console.log(`-- Working ${filename} --`); + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: metadata.title || filename, + docAuthor: metadata.docAuthor || "Unknown", + description: metadata.description || "Unknown", + docSource: metadata.docSource || "epub file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}`, + options: { parseOnly: options.parseOnly }, + }); + if (!options.absolutePath) trashFile(fullFilePath); + console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = asEPub; diff --git a/collector/processSingleFile/convert/asImage.js b/collector/processSingleFile/convert/asImage.js new file mode 100644 index 0000000..85e941c --- /dev/null +++ b/collector/processSingleFile/convert/asImage.js @@ -0,0 +1,56 @@ +const { v4 } = require("uuid"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../utils/files"); +const OCRLoader = require("../../utils/OCRLoader"); +const { default: slugify } = require("slugify"); + +async function asImage({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + let content = await new OCRLoader({ + targetLanguages: options?.ocr?.langList, + }).ocrImage(fullFilePath); + + if (!content?.length) { + console.error(`Resulting text content was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No text content found in ${filename}.`, + documents: [], + }; + } + + console.log(`-- Working ${filename} --`); + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: metadata.title || filename, + docAuthor: metadata.docAuthor || "Unknown", + description: metadata.description || "Unknown", + docSource: metadata.docSource || "image file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}`, + options: { parseOnly: options.parseOnly }, + }); + if (!options.absolutePath) trashFile(fullFilePath); + console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = asImage; diff --git a/collector/processSingleFile/convert/asMbox.js b/collector/processSingleFile/convert/asMbox.js new file mode 100644 index 0000000..2954e68 --- /dev/null +++ b/collector/processSingleFile/convert/asMbox.js @@ -0,0 +1,83 @@ +const { v4 } = require("uuid"); +const fs = require("fs"); +const { mboxParser } = require("mbox-parser"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../utils/files"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { default: slugify } = require("slugify"); + +async function asMbox({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + console.log(`-- Working ${filename} --`); + + const mails = await mboxParser(fs.createReadStream(fullFilePath)) + .then((mails) => mails) + .catch((error) => { + console.log(`Could not parse mail items`, error); + return []; + }); + + if (!mails.length) { + console.error(`Resulting mail items was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No mail items found in ${filename}.`, + documents: [], + }; + } + + let item = 1; + const documents = []; + for (const mail of mails) { + if (!mail.hasOwnProperty("text")) continue; + + const content = mail.text; + if (!content) continue; + console.log( + `-- Working on message "${mail.subject || "Unknown subject"}" --` + ); + + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: + metadata.title || + (mail?.subject + ? slugify(mail?.subject?.replace(".", "")) + ".mbox" + : `msg_${item}-${filename}`), + docAuthor: metadata.docAuthor || mail?.from?.text, + description: metadata.description || "No description found.", + docSource: + metadata.docSource || "Mbox message file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + item++; + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}-msg-${item}`, + options: { parseOnly: options.parseOnly }, + }); + documents.push(document); + } + + if (!options.absolutePath) trashFile(fullFilePath); + console.log( + `[SUCCESS]: ${filename} messages converted & ready for embedding.\n` + ); + return { success: true, reason: null, documents }; +} + +module.exports = asMbox; diff --git a/collector/processSingleFile/convert/asOfficeMime.js b/collector/processSingleFile/convert/asOfficeMime.js new file mode 100644 index 0000000..00895ba --- /dev/null +++ b/collector/processSingleFile/convert/asOfficeMime.js @@ -0,0 +1,59 @@ +const { v4 } = require("uuid"); +const officeParser = require("officeparser"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../utils/files"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { default: slugify } = require("slugify"); + +async function asOfficeMime({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + console.log(`-- Working ${filename} --`); + let content = ""; + try { + content = await officeParser.parseOfficeAsync(fullFilePath); + } catch (error) { + console.error(`Could not parse office or office-like file`, error); + } + + if (!content.length) { + console.error(`Resulting text content was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No text content found in ${filename}.`, + documents: [], + }; + } + + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: metadata.title || filename, + docAuthor: metadata.docAuthor || "no author found", + description: metadata.description || "No description found.", + docSource: metadata.docSource || "Office file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}`, + options: { parseOnly: options.parseOnly }, + }); + if (!options.absolutePath) trashFile(fullFilePath); + console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = asOfficeMime; diff --git a/collector/processSingleFile/convert/asPDF/PDFLoader/index.js b/collector/processSingleFile/convert/asPDF/PDFLoader/index.js new file mode 100644 index 0000000..26bcf2b --- /dev/null +++ b/collector/processSingleFile/convert/asPDF/PDFLoader/index.js @@ -0,0 +1,97 @@ +const fs = require("fs").promises; + +class PDFLoader { + constructor(filePath, { splitPages = true } = {}) { + this.filePath = filePath; + this.splitPages = splitPages; + } + + async load() { + const buffer = await fs.readFile(this.filePath); + const { getDocument, version } = await this.getPdfJS(); + + const pdf = await getDocument({ + data: new Uint8Array(buffer), + useWorkerFetch: false, + isEvalSupported: false, + useSystemFonts: true, + }).promise; + + const meta = await pdf.getMetadata().catch(() => null); + const documents = []; + + for (let i = 1; i <= pdf.numPages; i += 1) { + const page = await pdf.getPage(i); + const content = await page.getTextContent(); + + if (content.items.length === 0) { + continue; + } + + let lastY; + const textItems = []; + for (const item of content.items) { + if ("str" in item) { + if (lastY === item.transform[5] || !lastY) { + textItems.push(item.str); + } else { + textItems.push(`\n${item.str}`); + } + lastY = item.transform[5]; + } + } + + const text = textItems.join(""); + documents.push({ + pageContent: text.trim(), + metadata: { + source: this.filePath, + pdf: { + version, + info: meta?.info, + metadata: meta?.metadata, + totalPages: pdf.numPages, + }, + loc: { pageNumber: i }, + }, + }); + } + + if (this.splitPages) { + return documents; + } + + if (documents.length === 0) { + return []; + } + + return [ + { + pageContent: documents.map((doc) => doc.pageContent).join("\n\n"), + metadata: { + source: this.filePath, + pdf: { + version, + info: meta?.info, + metadata: meta?.metadata, + totalPages: pdf.numPages, + }, + }, + }, + ]; + } + + async getPdfJS() { + try { + const pdfjs = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js"); + return { getDocument: pdfjs.getDocument, version: pdfjs.version }; + } catch (e) { + console.error(e); + throw new Error( + "Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`." + ); + } + } +} + +module.exports = PDFLoader; diff --git a/collector/processSingleFile/convert/asPDF/index.js b/collector/processSingleFile/convert/asPDF/index.js new file mode 100644 index 0000000..59d26f5 --- /dev/null +++ b/collector/processSingleFile/convert/asPDF/index.js @@ -0,0 +1,86 @@ +const { v4 } = require("uuid"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../../utils/files"); +const { tokenizeString } = require("../../../utils/tokenizer"); +const { default: slugify } = require("slugify"); +const PDFLoader = require("./PDFLoader"); +const OCRLoader = require("../../../utils/OCRLoader"); + +async function asPdf({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + const pdfLoader = new PDFLoader(fullFilePath, { + splitPages: true, + }); + + console.log(`-- Working ${filename} --`); + const pageContent = []; + let docs = await pdfLoader.load(); + + if (docs.length === 0) { + console.log( + `[asPDF] No text content found for ${filename}. Will attempt OCR parse.` + ); + docs = await new OCRLoader({ + targetLanguages: options?.ocr?.langList, + }).ocrPDF(fullFilePath); + } + + for (const doc of docs) { + console.log( + `-- Parsing content from pg ${ + doc.metadata?.loc?.pageNumber || "unknown" + } --` + ); + if (!doc.pageContent || !doc.pageContent.length) continue; + pageContent.push(doc.pageContent); + } + + if (!pageContent.length) { + console.error(`[asPDF] Resulting text content was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No text content found in ${filename}.`, + documents: [], + }; + } + + const content = pageContent.join(""); + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: metadata.title || filename, + docAuthor: + metadata.docAuthor || + docs[0]?.metadata?.pdf?.info?.Creator || + "no author found", + description: + metadata.description || + docs[0]?.metadata?.pdf?.info?.Title || + "No description found.", + docSource: metadata.docSource || "pdf file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}`, + options: { parseOnly: options.parseOnly }, + }); + if (!options.absolutePath) trashFile(fullFilePath); + console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = asPdf; diff --git a/collector/processSingleFile/convert/asTxt.js b/collector/processSingleFile/convert/asTxt.js new file mode 100644 index 0000000..62e8e40 --- /dev/null +++ b/collector/processSingleFile/convert/asTxt.js @@ -0,0 +1,59 @@ +const { v4 } = require("uuid"); +const fs = require("fs"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { + createdDate, + trashFile, + writeToServerDocuments, +} = require("../../utils/files"); +const { default: slugify } = require("slugify"); + +async function asTxt({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + let content = ""; + try { + content = fs.readFileSync(fullFilePath, "utf8"); + } catch (err) { + console.error("Could not read file!", err); + } + + if (!content?.length) { + console.error(`Resulting text content was empty for ${filename}.`); + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `No text content found in ${filename}.`, + documents: [], + }; + } + + console.log(`-- Working ${filename} --`); + const data = { + id: v4(), + url: "file://" + fullFilePath, + title: metadata.title || filename, + docAuthor: metadata.docAuthor || "Unknown", + description: metadata.description || "Unknown", + docSource: metadata.docSource || "a text file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data, + filename: `${slugify(filename)}-${data.id}`, + options: { parseOnly: options.parseOnly }, + }); + if (!options.absolutePath) trashFile(fullFilePath); + console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); + return { success: true, reason: null, documents: [document] }; +} + +module.exports = asTxt; diff --git a/collector/processSingleFile/convert/asXlsx.js b/collector/processSingleFile/convert/asXlsx.js new file mode 100644 index 0000000..a316d1e --- /dev/null +++ b/collector/processSingleFile/convert/asXlsx.js @@ -0,0 +1,193 @@ +const { v4 } = require("uuid"); +const xlsx = require("node-xlsx").default; +const path = require("path"); +const fs = require("fs"); +const { + createdDate, + trashFile, + writeToServerDocuments, + documentsFolder, +} = require("../../utils/files"); +const { tokenizeString } = require("../../utils/tokenizer"); +const { default: slugify } = require("slugify"); + +function convertToCSV(data) { + return data + .map((row) => + row + .map((cell) => { + if (cell === null || cell === undefined) return ""; + if (typeof cell === "string" && cell.includes(",")) + return `"${cell}"`; + return cell; + }) + .join(",") + ) + .join("\n"); +} + +async function asXlsx({ + fullFilePath = "", + filename = "", + options = {}, + metadata = {}, +}) { + const documents = []; + + try { + const workSheetsFromFile = xlsx.parse(fullFilePath); + + if (options.parseOnly) { + const allSheetContents = []; + let totalWordCount = 0; + const sheetNames = []; + + for (const sheet of workSheetsFromFile) { + const processed = processSheet(sheet); + if (!processed) continue; + + const { name, content, wordCount } = processed; + sheetNames.push(name); + allSheetContents.push(`\nSheet: ${name}\n${content}`); + totalWordCount += wordCount; + } + + if (allSheetContents.length === 0) { + console.log(`No valid sheets found in ${filename}.`); + return { + success: false, + reason: `No valid sheets found in ${filename}.`, + documents: [], + }; + } + + const combinedContent = allSheetContents.join("\n"); + const sheetListText = + sheetNames.length > 1 + ? ` (Sheets: ${sheetNames.join(", ")})` + : ` (Sheet: ${sheetNames[0]})`; + + const combinedData = { + id: v4(), + url: `file://${fullFilePath}`, + title: metadata.title || `${filename}${sheetListText}`, + docAuthor: metadata.docAuthor || "Unknown", + description: + metadata.description || + `Spreadsheet data from ${filename} containing ${sheetNames.length} ${ + sheetNames.length === 1 ? "sheet" : "sheets" + }`, + docSource: metadata.docSource || "an xlsx file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: totalWordCount, + pageContent: combinedContent, + token_count_estimate: tokenizeString(combinedContent), + }; + + const document = writeToServerDocuments({ + data: combinedData, + filename: `${slugify(path.basename(filename))}-${combinedData.id}`, + destinationOverride: null, + options: { parseOnly: true }, + }); + documents.push(document); + console.log(`[SUCCESS]: ${filename} converted & ready for embedding.`); + } else { + const folderName = slugify( + `${path.basename(filename)}-${v4().slice(0, 4)}`, + { + lower: true, + trim: true, + } + ); + const outFolderPath = path.resolve(documentsFolder, folderName); + if (!fs.existsSync(outFolderPath)) + fs.mkdirSync(outFolderPath, { recursive: true }); + + for (const sheet of workSheetsFromFile) { + const processed = processSheet(sheet); + if (!processed) continue; + + const { name, content, wordCount } = processed; + const sheetData = { + id: v4(), + url: `file://${path.join(outFolderPath, `${slugify(name)}.csv`)}`, + title: metadata.title || `${filename} - Sheet:${name}`, + docAuthor: metadata.docAuthor || "Unknown", + description: + metadata.description || `Spreadsheet data from sheet: ${name}`, + docSource: metadata.docSource || "an xlsx file uploaded by the user.", + chunkSource: metadata.chunkSource || "", + published: createdDate(fullFilePath), + wordCount: wordCount, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + const document = writeToServerDocuments({ + data: sheetData, + filename: `sheet-${slugify(name)}`, + destinationOverride: outFolderPath, + options: { parseOnly: options.parseOnly }, + }); + documents.push(document); + console.log( + `[SUCCESS]: Sheet "${name}" converted & ready for embedding.` + ); + } + } + } catch (err) { + console.error("Could not process xlsx file!", err); + return { + success: false, + reason: `Error processing ${filename}: ${err.message}`, + documents: [], + }; + } finally { + if (!options.absolutePath) trashFile(fullFilePath); + } + + if (documents.length === 0) { + console.error(`No valid sheets found in ${filename}.`); + return { + success: false, + reason: `No valid sheets found in ${filename}.`, + documents: [], + }; + } + + console.log( + `[SUCCESS]: ${filename} fully processed. Created ${documents.length} document(s).\n` + ); + return { success: true, reason: null, documents }; +} + +/** + * Processes a single sheet and returns its content and metadata + * @param {{name: string, data: Array>}} sheet - Parsed sheet with name and 2D array of cell values + * @returns {{name: string, content: string, wordCount: number}|null} - Object with name, CSV content, and word count, or null if sheet is empty + */ +function processSheet(sheet) { + try { + const { name, data } = sheet; + const content = convertToCSV(data); + + if (!content?.length) { + console.log(`Sheet "${name}" is empty. Skipping.`); + return null; + } + + console.log(`-- Processing sheet: ${name} --`); + return { + name, + content, + wordCount: content.split(/\s+/).length, + }; + } catch (err) { + console.error(`Error processing sheet "${sheet.name}":`, err); + return null; + } +} + +module.exports = asXlsx; diff --git a/collector/processSingleFile/index.js b/collector/processSingleFile/index.js new file mode 100644 index 0000000..4f2defb --- /dev/null +++ b/collector/processSingleFile/index.js @@ -0,0 +1,95 @@ +const path = require("path"); +const fs = require("fs"); +const { + WATCH_DIRECTORY, + SUPPORTED_FILETYPE_CONVERTERS, +} = require("../utils/constants"); +const { + trashFile, + isTextType, + normalizePath, + isWithin, +} = require("../utils/files"); +const RESERVED_FILES = ["__HOTDIR__.md"]; + +/** + * Process a single file and return the documents + * @param {string} targetFilename - The filename to process + * @param {Object} options - The options for the file processing + * @param {boolean} options.parseOnly - If true, the file will not be saved as a document even when `writeToServerDocuments` is called in the handler. Must be explicitly set to true to use. + * @param {string} options.absolutePath - If provided, use this absolute path instead of resolving relative to WATCH_DIRECTORY. For internal use only. + * @param {Object} metadata - The metadata for the file processing + * @returns {Promise<{success: boolean, reason: string, documents: Object[]}>} - The documents from the file processing + */ +async function processSingleFile(targetFilename, options = {}, metadata = {}) { + const fullFilePath = normalizePath( + options.absolutePath || path.resolve(WATCH_DIRECTORY, targetFilename) + ); + + // If absolute path is not provided, check if the file is within the watch directory + // to prevent unauthorized paths from being processed. + if ( + !options.absolutePath && + !isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath) + ) + return { + success: false, + reason: "Filename is a not a valid path to process.", + documents: [], + }; + + if (RESERVED_FILES.includes(targetFilename)) + return { + success: false, + reason: "Filename is a reserved filename and cannot be processed.", + documents: [], + }; + + if (!fs.existsSync(fullFilePath)) + return { + success: false, + reason: "File does not exist in upload directory.", + documents: [], + }; + + const fileExtension = path.extname(fullFilePath).toLowerCase(); + if (fullFilePath.includes(".") && !fileExtension) { + return { + success: false, + reason: `No file extension found. This file cannot be processed.`, + documents: [], + }; + } + + let processFileAs = fileExtension; + if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) { + if (isTextType(fullFilePath)) { + console.log( + `\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.` + ); + processFileAs = ".txt"; + } else { + // If absolute path is provided, do NOT trash the file since it is a user provided path. + if (!options.absolutePath) trashFile(fullFilePath); + return { + success: false, + reason: `File extension ${fileExtension} not supported for parsing and cannot be assumed as text file type.`, + documents: [], + }; + } + } + + const FileTypeProcessor = require(SUPPORTED_FILETYPE_CONVERTERS[ + processFileAs + ]); + return await FileTypeProcessor({ + fullFilePath, + filename: targetFilename, + options, + metadata, + }); +} + +module.exports = { + processSingleFile, +}; diff --git a/collector/storage/.gitignore b/collector/storage/.gitignore new file mode 100644 index 0000000..845482b --- /dev/null +++ b/collector/storage/.gitignore @@ -0,0 +1,2 @@ +tmp/* +!tmp/.placeholder \ No newline at end of file diff --git a/collector/storage/tmp/.placeholder b/collector/storage/tmp/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/collector/utils/EncryptionWorker/index.js b/collector/utils/EncryptionWorker/index.js new file mode 100644 index 0000000..ddc2773 --- /dev/null +++ b/collector/utils/EncryptionWorker/index.js @@ -0,0 +1,77 @@ +const crypto = require("crypto"); + +// Differs from EncryptionManager in that is does not set or define the keys that will be used +// to encrypt or read data and it must be told the key (as base64 string) explicitly that will be used and is provided to +// the class on creation. This key should be the same `key` that is used by the EncryptionManager class. +class EncryptionWorker { + constructor(presetKeyBase64 = "") { + this.key = Buffer.from(presetKeyBase64, "base64"); + this.algorithm = "aes-256-cbc"; + this.separator = ":"; + } + + log(text, ...args) { + console.log(`\x1b[36m[EncryptionManager]\x1b[0m ${text}`, ...args); + } + + /** + * Give a chunk source, parse its payload query param and expand that object back into the URL + * as additional query params + * @param {string} chunkSource + * @returns {URL} Javascript URL object with query params decrypted from payload query param. + */ + expandPayload(chunkSource = "") { + try { + const url = new URL(chunkSource); + if (!url.searchParams.has("payload")) return url; + + const decryptedPayload = this.decrypt(url.searchParams.get("payload")); + const encodedParams = JSON.parse(decryptedPayload); + url.searchParams.delete("payload"); // remove payload prop + + // Add all query params needed to replay as query params + Object.entries(encodedParams).forEach(([key, value]) => + url.searchParams.append(key, value) + ); + return url; + } catch (e) { + console.error(e); + } + return new URL(chunkSource); + } + + encrypt(plainTextString = null) { + try { + if (!plainTextString) + throw new Error("Empty string is not valid for this method."); + const iv = crypto.randomBytes(16); + const cipher = crypto.createCipheriv(this.algorithm, this.key, iv); + const encrypted = cipher.update(plainTextString, "utf8", "hex"); + return [ + encrypted + cipher.final("hex"), + Buffer.from(iv).toString("hex"), + ].join(this.separator); + } catch (e) { + this.log(e); + return null; + } + } + + decrypt(encryptedString) { + try { + const [encrypted, iv] = encryptedString.split(this.separator); + if (!iv) throw new Error("IV not found"); + const decipher = crypto.createDecipheriv( + this.algorithm, + this.key, + Buffer.from(iv, "hex") + ); + return decipher.update(encrypted, "hex", "utf8") + decipher.final("utf8"); + } catch (e) { + this.log(e); + return null; + } + } +} + +module.exports = { EncryptionWorker }; diff --git a/collector/utils/OCRLoader/index.js b/collector/utils/OCRLoader/index.js new file mode 100644 index 0000000..8f5ae99 --- /dev/null +++ b/collector/utils/OCRLoader/index.js @@ -0,0 +1,355 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { VALID_LANGUAGE_CODES } = require("./validLangs"); + +class OCRLoader { + /** + * The language code(s) to use for the OCR. + * @type {string[]} + */ + language; + /** + * The cache directory for the OCR. + * @type {string} + */ + cacheDir; + + /** + * The constructor for the OCRLoader. + * @param {Object} options - The options for the OCRLoader. + * @param {string} options.targetLanguages - The target languages to use for the OCR as a comma separated string. eg: "eng,deu,..." + */ + constructor({ targetLanguages = "eng" } = {}) { + this.language = this.parseLanguages(targetLanguages); + this.cacheDir = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, `models`, `tesseract`) + : path.resolve(__dirname, `../../../server/storage/models/tesseract`) + ); + + // Ensure the cache directory exists or else Tesseract will persist the cache in the default location. + if (!fs.existsSync(this.cacheDir)) + fs.mkdirSync(this.cacheDir, { recursive: true }); + this.log( + `OCRLoader initialized with language support for:`, + this.language.map((lang) => VALID_LANGUAGE_CODES[lang]).join(", ") + ); + } + + /** + * Parses the language code from a provided comma separated string of language codes. + * @param {string} language - The language code to parse. + * @returns {string[]} The parsed language code. + */ + parseLanguages(language = null) { + try { + if (!language || typeof language !== "string") return ["eng"]; + const langList = language + .split(",") + .map((lang) => (lang.trim() !== "" ? lang.trim() : null)) + .filter(Boolean) + .filter((lang) => VALID_LANGUAGE_CODES.hasOwnProperty(lang)); + if (langList.length === 0) return ["eng"]; + return langList; + } catch (e) { + this.log(`Error parsing languages: ${e.message}`, e.stack); + return ["eng"]; + } + } + + log(text, ...args) { + console.log(`\x1b[36m[OCRLoader]\x1b[0m ${text}`, ...args); + } + + /** + * Loads a PDF file and returns an array of documents. + * This function is reserved to parsing for SCANNED documents - digital documents are not supported in this function + * @returns {Promise<{pageContent: string, metadata: object}[]>} An array of documents with page content and metadata. + */ + async ocrPDF( + filePath, + { maxExecutionTime = 300_000, batchSize = 10, maxWorkers = null } = {} + ) { + if ( + !filePath || + !fs.existsSync(filePath) || + !fs.statSync(filePath).isFile() + ) { + this.log(`File ${filePath} does not exist. Skipping OCR.`); + return []; + } + + const documentTitle = path.basename(filePath); + this.log(`Starting OCR of ${documentTitle}`); + const pdfjs = await import("pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js"); + let buffer = fs.readFileSync(filePath); + + const pdfDocument = await pdfjs.getDocument({ data: buffer }); + + const documents = []; + const meta = await pdfDocument.getMetadata().catch(() => null); + const metadata = { + source: filePath, + pdf: { + version: "v2.0.550", + info: meta?.info, + metadata: meta?.metadata, + totalPages: pdfDocument.numPages, + }, + }; + + const pdfSharp = new PDFSharp({ + validOps: [ + pdfjs.OPS.paintJpegXObject, + pdfjs.OPS.paintImageXObject, + pdfjs.OPS.paintInlineImageXObject, + ], + }); + await pdfSharp.init(); + + const { createWorker, OEM } = require("tesseract.js"); + const BATCH_SIZE = batchSize; + const MAX_EXECUTION_TIME = maxExecutionTime; + const NUM_WORKERS = maxWorkers ?? Math.min(os.cpus().length, 4); + const totalPages = pdfDocument.numPages; + const workerPool = await Promise.all( + Array(NUM_WORKERS) + .fill(0) + .map(() => + createWorker(this.language, OEM.LSTM_ONLY, { + cachePath: this.cacheDir, + }) + ) + ); + + const startTime = Date.now(); + try { + this.log("Bootstrapping OCR completed successfully!", { + MAX_EXECUTION_TIME_MS: MAX_EXECUTION_TIME, + BATCH_SIZE, + MAX_CONCURRENT_WORKERS: NUM_WORKERS, + TOTAL_PAGES: totalPages, + }); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject( + new Error( + `OCR job took too long to complete (${ + MAX_EXECUTION_TIME / 1000 + } seconds)` + ) + ); + }, MAX_EXECUTION_TIME); + }); + + const processPages = async () => { + for ( + let startPage = 1; + startPage <= totalPages; + startPage += BATCH_SIZE + ) { + const endPage = Math.min(startPage + BATCH_SIZE - 1, totalPages); + const pageNumbers = Array.from( + { length: endPage - startPage + 1 }, + (_, i) => startPage + i + ); + this.log(`Working on pages ${startPage} - ${endPage}`); + + const pageQueue = [...pageNumbers]; + const results = []; + const workerPromises = workerPool.map(async (worker, workerIndex) => { + while (pageQueue.length > 0) { + const pageNum = pageQueue.shift(); + this.log( + `\x1b[34m[Worker ${ + workerIndex + 1 + }]\x1b[0m assigned pg${pageNum}` + ); + const page = await pdfDocument.getPage(pageNum); + const imageBuffer = await pdfSharp.pageToBuffer({ page }); + if (!imageBuffer) continue; + const { data } = await worker.recognize(imageBuffer, {}, "text"); + this.log( + `✅ \x1b[34m[Worker ${ + workerIndex + 1 + }]\x1b[0m completed pg${pageNum}` + ); + results.push({ + pageContent: data.text, + metadata: { + ...metadata, + loc: { pageNumber: pageNum }, + }, + }); + } + }); + + await Promise.all(workerPromises); + documents.push( + ...results.sort( + (a, b) => a.metadata.loc.pageNumber - b.metadata.loc.pageNumber + ) + ); + } + return documents; + }; + + await Promise.race([timeoutPromise, processPages()]); + } catch (e) { + this.log(`Error: ${e.message}`, e.stack); + } finally { + global.Image = undefined; + await Promise.all(workerPool.map((worker) => worker.terminate())); + } + + this.log(`Completed OCR of ${documentTitle}!`, { + documentsParsed: documents.length, + totalPages: totalPages, + executionTime: `${((Date.now() - startTime) / 1000).toFixed(2)}s`, + }); + return documents; + } + + /** + * Loads an image file and returns the OCRed text. + * @param {string} filePath - The path to the image file. + * @param {Object} options - The options for the OCR. + * @param {number} options.maxExecutionTime - The maximum execution time of the OCR in milliseconds. + * @returns {Promise} The OCRed text. + */ + async ocrImage(filePath, { maxExecutionTime = 300_000 } = {}) { + let content = ""; + let worker = null; + if ( + !filePath || + !fs.existsSync(filePath) || + !fs.statSync(filePath).isFile() + ) { + this.log(`File ${filePath} does not exist. Skipping OCR.`); + return null; + } + + const documentTitle = path.basename(filePath); + try { + this.log(`Starting OCR of ${documentTitle}`); + const startTime = Date.now(); + const { createWorker, OEM } = require("tesseract.js"); + worker = await createWorker(this.language, OEM.LSTM_ONLY, { + cachePath: this.cacheDir, + }); + + // Race the timeout with the OCR + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject( + new Error( + `OCR job took too long to complete (${ + maxExecutionTime / 1000 + } seconds)` + ) + ); + }, maxExecutionTime); + }); + + const processImage = async () => { + const { data } = await worker.recognize(filePath, {}, "text"); + content = data.text; + }; + + await Promise.race([timeoutPromise, processImage()]); + this.log(`Completed OCR of ${documentTitle}!`, { + executionTime: `${((Date.now() - startTime) / 1000).toFixed(2)}s`, + }); + + return content; + } catch (e) { + this.log(`Error: ${e.message}`); + return null; + } finally { + //eslint-disable-next-line + if (!worker) return; + await worker.terminate(); + } + } +} + +/** + * Converts a PDF page to a buffer using Sharp. + * @param {Object} options - The options for the Sharp PDF page object. + * @param {Object} options.page - The PDFJS page proxy object. + * @returns {Promise} The buffer of the page. + */ +class PDFSharp { + constructor({ validOps = [] } = {}) { + this.sharp = null; + this.validOps = validOps; + } + + log(text, ...args) { + console.log(`\x1b[36m[PDFSharp]\x1b[0m ${text}`, ...args); + } + + async init() { + this.sharp = (await import("sharp")).default; + } + + /** + * Converts a PDF page to a buffer. + * @param {Object} options - The options for the Sharp PDF page object. + * @param {Object} options.page - The PDFJS page proxy object. + * @returns {Promise} The buffer of the page. + */ + async pageToBuffer({ page }) { + if (!this.sharp) await this.init(); + try { + this.log(`Converting page ${page.pageNumber} to image...`); + const ops = await page.getOperatorList(); + const pageImages = ops.fnArray.length; + + for (let i = 0; i < pageImages; i++) { + try { + if (!this.validOps.includes(ops.fnArray[i])) continue; + + const name = ops.argsArray[i][0]; + const img = await page.objs.get(name); + const { width, height } = img; + const size = img.data.length; + const channels = size / width / height; + const targetDPI = 70; + const targetWidth = Math.floor(width * (targetDPI / 72)); + const targetHeight = Math.floor(height * (targetDPI / 72)); + + const image = this.sharp(img.data, { + raw: { width, height, channels }, + density: targetDPI, + }) + .resize({ + width: targetWidth, + height: targetHeight, + fit: "fill", + }) + .withMetadata({ + density: targetDPI, + resolution: targetDPI, + }) + .png(); + + // For debugging purposes + // await image.toFile(path.resolve(__dirname, `../../storage/`, `pg${page.pageNumber}.png`)); + return await image.toBuffer(); + } catch (error) { + this.log(`Iteration error: ${error.message}`, error.stack); + continue; + } + } + this.log(`No valid images found on page ${page.pageNumber}`); + return null; + } catch (error) { + this.log(`Error: ${error.message}`, error.stack); + return null; + } + } +} + +module.exports = OCRLoader; diff --git a/collector/utils/OCRLoader/validLangs.js b/collector/utils/OCRLoader/validLangs.js new file mode 100644 index 0000000..5bc807a --- /dev/null +++ b/collector/utils/OCRLoader/validLangs.js @@ -0,0 +1,155 @@ +/* + +To get the list of valid language codes - do the following: +Open the following URL in your browser: https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html + +Check this element is the proper table tbody with all the codes via console: +document.getElementsByTagName('table').item(0).children.item(1) + +Now, copy the following code and paste it into the console: +function parseLangs() { +let langs = {}; + Array.from(document.getElementsByTagName('table').item(0).children.item(1).children).forEach((el) => { + const [codeEl, languageEl, ...rest] = el.children + const code = codeEl.innerText.trim() + const language = languageEl.innerText.trim() + if (!!code && !!language) langs[code] = language + }) + return langs; +} + +now, run the function: +copy(parseLangs()) +*/ + +const VALID_LANGUAGE_CODES = { + afr: "Afrikaans", + amh: "Amharic", + ara: "Arabic", + asm: "Assamese", + aze: "Azerbaijani", + aze_cyrl: "Azerbaijani - Cyrilic", + bel: "Belarusian", + ben: "Bengali", + bod: "Tibetan", + bos: "Bosnian", + bre: "Breton", + bul: "Bulgarian", + cat: "Catalan; Valencian", + ceb: "Cebuano", + ces: "Czech", + chi_sim: "Chinese - Simplified", + chi_tra: "Chinese - Traditional", + chr: "Cherokee", + cos: "Corsican", + cym: "Welsh", + dan: "Danish", + dan_frak: "Danish - Fraktur (contrib)", + deu: "German", + deu_frak: "German - Fraktur (contrib)", + deu_latf: "German (Fraktur Latin)", + dzo: "Dzongkha", + ell: "Greek, Modern (1453-)", + eng: "English", + enm: "English, Middle (1100-1500)", + epo: "Esperanto", + equ: "Math / equation detection module", + est: "Estonian", + eus: "Basque", + fao: "Faroese", + fas: "Persian", + fil: "Filipino (old - Tagalog)", + fin: "Finnish", + fra: "French", + frk: "German - Fraktur (now deu_latf)", + frm: "French, Middle (ca.1400-1600)", + fry: "Western Frisian", + gla: "Scottish Gaelic", + gle: "Irish", + glg: "Galician", + grc: "Greek, Ancient (to 1453) (contrib)", + guj: "Gujarati", + hat: "Haitian; Haitian Creole", + heb: "Hebrew", + hin: "Hindi", + hrv: "Croatian", + hun: "Hungarian", + hye: "Armenian", + iku: "Inuktitut", + ind: "Indonesian", + isl: "Icelandic", + ita: "Italian", + ita_old: "Italian - Old", + jav: "Javanese", + jpn: "Japanese", + kan: "Kannada", + kat: "Georgian", + kat_old: "Georgian - Old", + kaz: "Kazakh", + khm: "Central Khmer", + kir: "Kirghiz; Kyrgyz", + kmr: "Kurmanji (Kurdish - Latin Script)", + kor: "Korean", + kor_vert: "Korean (vertical)", + kur: "Kurdish (Arabic Script)", + lao: "Lao", + lat: "Latin", + lav: "Latvian", + lit: "Lithuanian", + ltz: "Luxembourgish", + mal: "Malayalam", + mar: "Marathi", + mkd: "Macedonian", + mlt: "Maltese", + mon: "Mongolian", + mri: "Maori", + msa: "Malay", + mya: "Burmese", + nep: "Nepali", + nld: "Dutch; Flemish", + nor: "Norwegian", + oci: "Occitan (post 1500)", + ori: "Oriya", + osd: "Orientation and script detection module", + pan: "Panjabi; Punjabi", + pol: "Polish", + por: "Portuguese", + pus: "Pushto; Pashto", + que: "Quechua", + ron: "Romanian; Moldavian; Moldovan", + rus: "Russian", + san: "Sanskrit", + sin: "Sinhala; Sinhalese", + slk: "Slovak", + slk_frak: "Slovak - Fraktur (contrib)", + slv: "Slovenian", + snd: "Sindhi", + spa: "Spanish; Castilian", + spa_old: "Spanish; Castilian - Old", + sqi: "Albanian", + srp: "Serbian", + srp_latn: "Serbian - Latin", + sun: "Sundanese", + swa: "Swahili", + swe: "Swedish", + syr: "Syriac", + tam: "Tamil", + tat: "Tatar", + tel: "Telugu", + tgk: "Tajik", + tgl: "Tagalog (new - Filipino)", + tha: "Thai", + tir: "Tigrinya", + ton: "Tonga", + tur: "Turkish", + uig: "Uighur; Uyghur", + ukr: "Ukrainian", + urd: "Urdu", + uzb: "Uzbek", + uzb_cyrl: "Uzbek - Cyrilic", + vie: "Vietnamese", + yid: "Yiddish", + yor: "Yoruba", +}; + +module.exports.VALID_LANGUAGE_CODES = VALID_LANGUAGE_CODES; diff --git a/collector/utils/WhisperProviders/GenericOpenAiWhisper.js b/collector/utils/WhisperProviders/GenericOpenAiWhisper.js new file mode 100644 index 0000000..d1b51f2 --- /dev/null +++ b/collector/utils/WhisperProviders/GenericOpenAiWhisper.js @@ -0,0 +1,51 @@ +const fs = require("fs"); + +class GenericOpenAiWhisper { + constructor({ options }) { + const { OpenAI: OpenAIApi } = require("openai"); + if (!options.WhisperGenericOpenAiBaseUrl) + throw new Error("No base URL was set."); + + this.openai = new OpenAIApi({ + baseURL: options.WhisperGenericOpenAiBaseUrl, + apiKey: options.WhisperGenericOpenAiApiKey || null, + }); + this.model = options.WhisperGenericOpenAiModel || "whisper-small"; + this.temperature = 0; + this.#log("Initialized."); + } + + #log(text, ...args) { + console.log(`\x1b[32m[GenericOpenAiWhisper]\x1b[0m ${text}`, ...args); + } + + async processFile(fullFilePath) { + return await this.openai.audio.transcriptions + .create({ + file: fs.createReadStream(fullFilePath), + model: this.model, + temperature: this.temperature, + }) + .then((response) => { + if (!response) { + return { + content: "", + error: "No content was able to be transcribed.", + }; + } + + return { content: response.text, error: null }; + }) + .catch((error) => { + this.#log( + `Could not get any response from openai compatible whisper endpoint`, + error.message + ); + return { content: "", error: error.message }; + }); + } +} + +module.exports = { + GenericOpenAiWhisper, +}; diff --git a/collector/utils/WhisperProviders/OpenAiWhisper.js b/collector/utils/WhisperProviders/OpenAiWhisper.js new file mode 100644 index 0000000..e2405b6 --- /dev/null +++ b/collector/utils/WhisperProviders/OpenAiWhisper.js @@ -0,0 +1,49 @@ +const fs = require("fs"); + +class OpenAiWhisper { + constructor({ options }) { + const { OpenAI: OpenAIApi } = require("openai"); + if (!options.openAiKey) throw new Error("No OpenAI API key was set."); + + this.openai = new OpenAIApi({ + apiKey: options.openAiKey, + }); + this.model = "whisper-1"; + this.temperature = 0; + this.#log("Initialized."); + } + + #log(text, ...args) { + console.log(`\x1b[32m[OpenAiWhisper]\x1b[0m ${text}`, ...args); + } + + async processFile(fullFilePath) { + return await this.openai.audio.transcriptions + .create({ + file: fs.createReadStream(fullFilePath), + model: this.model, + temperature: this.temperature, + }) + .then((response) => { + if (!response) { + return { + content: "", + error: "No content was able to be transcribed.", + }; + } + + return { content: response.text, error: null }; + }) + .catch((error) => { + this.#log( + `Could not get any response from openai whisper`, + error.message + ); + return { content: "", error: error.message }; + }); + } +} + +module.exports = { + OpenAiWhisper, +}; diff --git a/collector/utils/WhisperProviders/ffmpeg/index.js b/collector/utils/WhisperProviders/ffmpeg/index.js new file mode 100644 index 0000000..7e0ed75 --- /dev/null +++ b/collector/utils/WhisperProviders/ffmpeg/index.js @@ -0,0 +1,114 @@ +const fs = require("fs"); +const path = require("path"); +const { execSync, spawnSync } = require("child_process"); +const { patchShellEnvironmentPath } = require("../../shell"); +/** + * Custom FFMPEG wrapper class for audio file conversion. + * Replaces deprecated fluent-ffmpeg package. + * Locates ffmpeg binary and converts audio files to required + * WAV format (16k hz mono 32f) for Whisper transcription. + * + * @class FFMPEGWrapper + */ +class FFMPEGWrapper { + static _instance; + + constructor() { + if (FFMPEGWrapper._instance) return FFMPEGWrapper._instance; + FFMPEGWrapper._instance = this; + this._ffmpegPath = null; + } + + log(text, ...args) { + console.log(`\x1b[35m[FFMPEG]\x1b[0m ${text}`, ...args); + } + + /** + * Locates ffmpeg binary. + * Uses fix-path on non-Windows platforms to ensure we can find ffmpeg. + * + * @returns {Promise} Path to ffmpeg binary + * @throws {Error} + */ + async ffmpegPath() { + if (this._ffmpegPath) return this._ffmpegPath; + await patchShellEnvironmentPath(); + + try { + const which = process.platform === "win32" ? "where" : "which"; + const result = execSync(`${which} ffmpeg`, { encoding: "utf8" }).trim(); + const candidatePath = result?.split("\n")?.[0]?.trim(); + if (!candidatePath) throw new Error("FFMPEG candidate path not found."); + if (!this.isValidFFMPEG(candidatePath)) + throw new Error("FFMPEG candidate path is not valid ffmpeg binary."); + + this.log(`Found FFMPEG binary at ${candidatePath}`); + this._ffmpegPath = candidatePath; + return this._ffmpegPath; + } catch (error) { + this.log(error.message); + } + + throw new Error("FFMPEG binary not found."); + } + + /** + * Validates that path points to a valid ffmpeg binary. + * Runs ffmpeg -version command. + * + * @param {string} pathToTest - Path of ffmpeg binary + * @returns {boolean} + */ + isValidFFMPEG(pathToTest) { + try { + if (!pathToTest || !fs.existsSync(pathToTest)) return false; + execSync(`"${pathToTest}" -version`, { encoding: "utf8", stdio: "pipe" }); + return true; + } catch { + return false; + } + } + + /** + * Converts audio file to WAV format with required parameters for Whisper. + * Output: 16k hz, mono, 32bit float. + * + * @param {string} inputPath - Input path for audio file (any format supported by ffmpeg) + * @param {string} outputPath - Output path for converted file + * @returns {Promise} + * @throws {Error} If ffmpeg binary cannot be found or conversion fails + */ + async convertAudioToWav(inputPath, outputPath) { + if (!fs.existsSync(inputPath)) + throw new Error(`Input file ${inputPath} does not exist.`); + const outputDir = path.dirname(outputPath); + if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true }); + + this.log(`Converting ${path.basename(inputPath)} to WAV format...`); + // Convert to 16k hz mono 32f + const result = spawnSync( + await this.ffmpegPath(), + [ + "-i", + inputPath, + "-ar", + "16000", + "-ac", + "1", + "-acodec", + "pcm_f32le", + "-y", + outputPath, + ], + { encoding: "utf8" } + ); + + // ffmpeg writes progress to stderr + if (result.stderr) this.log(result.stderr.trim()); + if (result.status !== 0) throw new Error(`FFMPEG conversion failed`); + this.log(`Conversion complete: ${path.basename(outputPath)}`); + return true; + } +} + +module.exports = { FFMPEGWrapper }; diff --git a/collector/utils/WhisperProviders/localWhisper.js b/collector/utils/WhisperProviders/localWhisper.js new file mode 100644 index 0000000..6ee124f --- /dev/null +++ b/collector/utils/WhisperProviders/localWhisper.js @@ -0,0 +1,197 @@ +const fs = require("fs"); +const path = require("path"); +const { v4 } = require("uuid"); +const defaultWhisper = "Xenova/whisper-small"; // Model Card: https://huggingface.co/Xenova/whisper-small +const fileSize = { + "Xenova/whisper-small": "250mb", + "Xenova/whisper-large": "1.56GB", +}; + +class LocalWhisper { + constructor({ options }) { + this.model = options?.WhisperModelPref ?? defaultWhisper; + this.fileSize = fileSize[this.model]; + this.cacheDir = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, `models`) + : path.resolve(__dirname, `../../../server/storage/models`) + ); + + this.modelPath = path.resolve(this.cacheDir, ...this.model.split("/")); + // Make directory when it does not exist in existing installations + if (!fs.existsSync(this.cacheDir)) + fs.mkdirSync(this.cacheDir, { recursive: true }); + + this.#log("Initialized."); + } + + #log(text, ...args) { + console.log(`\x1b[32m[LocalWhisper]\x1b[0m ${text}`, ...args); + } + + #validateAudioFile(wavFile) { + const sampleRate = wavFile.fmt.sampleRate; + const duration = wavFile.data.samples / sampleRate; + + // Most speech recognition systems expect minimum 8kHz + // But we'll set it lower to be safe + if (sampleRate < 4000) { + // 4kHz minimum + throw new Error( + "Audio file sample rate is too low for accurate transcription. Minimum required is 4kHz." + ); + } + + // Typical audio file duration limits + const MAX_DURATION_SECONDS = 4 * 60 * 60; // 4 hours + if (duration > MAX_DURATION_SECONDS) { + throw new Error("Audio file duration exceeds maximum limit of 4 hours."); + } + + // Check final sample count after upsampling to prevent memory issues + const targetSampleRate = 16000; + const upsampledSamples = duration * targetSampleRate; + const MAX_SAMPLES = 230_400_000; // ~4 hours at 16kHz + + if (upsampledSamples > MAX_SAMPLES) { + throw new Error("Audio file exceeds maximum allowed length."); + } + + return true; + } + + async #convertToWavAudioData(sourcePath) { + try { + let buffer; + const wavefile = require("wavefile"); + const { FFMPEGWrapper } = require("./ffmpeg"); + const ffmpeg = new FFMPEGWrapper(); + const outFolder = path.resolve(__dirname, `../../storage/tmp`); + if (!fs.existsSync(outFolder)) + fs.mkdirSync(outFolder, { recursive: true }); + + const outputFile = path.resolve(outFolder, `${v4()}.wav`); + const success = await ffmpeg.convertAudioToWav(sourcePath, outputFile); + if (!success) + throw new Error( + "[Conversion Failed]: Could not convert file to .wav format!" + ); + + buffer = fs.readFileSync(outputFile); + fs.rmSync(outputFile); + + const wavFile = new wavefile.WaveFile(buffer); + try { + this.#validateAudioFile(wavFile); + } catch (error) { + this.#log(`Audio validation failed: ${error.message}`); + throw new Error(`Invalid audio file: ${error.message}`); + } + + // Although we use ffmpeg to convert to the correct format (16k hz 32f), + // different versions of ffmpeg produce different results based on the + // environment. To ensure consistency, we convert to the correct format again. + wavFile.toBitDepth("32f"); + wavFile.toSampleRate(16000); + + let audioData = wavFile.getSamples(); + if (Array.isArray(audioData)) { + if (audioData.length > 1) { + const SCALING_FACTOR = Math.sqrt(2); + + // Merge channels into first channel to save memory + for (let i = 0; i < audioData[0].length; ++i) { + audioData[0][i] = + (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2; + } + } + audioData = audioData[0]; + } + + return audioData; + } catch (error) { + console.error(`convertToWavAudioData`, error); + return null; + } + } + + async client() { + if (!fs.existsSync(this.modelPath)) { + this.#log( + `The native whisper model has never been run and will be downloaded right now. Subsequent runs will be faster. (~${this.fileSize})` + ); + } + + try { + // Convert ESM to CommonJS via import so we can load this library. + const pipeline = (...args) => + import("@xenova/transformers").then(({ pipeline }) => { + return pipeline(...args); + }); + return await pipeline("automatic-speech-recognition", this.model, { + cache_dir: this.cacheDir, + ...(!fs.existsSync(this.modelPath) + ? { + // Show download progress if we need to download any files + progress_callback: (data) => { + if (!data.hasOwnProperty("progress")) return; + console.log( + `\x1b[34m[ONNXWhisper - Downloading Model Files]\x1b[0m ${ + data.file + } ${~~data?.progress}%` + ); + }, + } + : {}), + }); + } catch (error) { + let errMsg = error.message; + if (errMsg.includes("Could not locate file")) { + errMsg = + "The native whisper model failed to download from the huggingface.co CDN. Your internet connection may be unstable or blocked by Huggingface.co - you will need to download the model manually and place it in the storage/models folder to use local Whisper transcription."; + } + + this.#log( + `Failed to load the native whisper model: ${errMsg}`, + error.stack + ); + throw new Error(errMsg); + } + } + + async processFile(fullFilePath, filename) { + try { + const audioDataPromise = new Promise((resolve) => + this.#convertToWavAudioData(fullFilePath).then((audioData) => + resolve(audioData) + ) + ); + const [audioData, transcriber] = await Promise.all([ + audioDataPromise, + this.client(), + ]); + + if (!audioData) { + this.#log(`Failed to parse content from ${filename}.`); + return { + content: null, + error: `Failed to parse content from ${filename}.`, + }; + } + + this.#log(`Transcribing audio data to text...`); + const { text } = await transcriber(audioData, { + chunk_length_s: 30, + stride_length_s: 5, + }); + + return { content: text, error: null }; + } catch (error) { + return { content: null, error: error.message }; + } + } +} + +module.exports = { + LocalWhisper, +}; diff --git a/collector/utils/comKey/index.js b/collector/utils/comKey/index.js new file mode 100644 index 0000000..a2e2f52 --- /dev/null +++ b/collector/utils/comKey/index.js @@ -0,0 +1,54 @@ +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); +const keyPath = + process.env.NODE_ENV === "development" + ? path.resolve(__dirname, `../../../server/storage/comkey`) + : path.resolve( + process.env.STORAGE_DIR ?? + path.resolve(__dirname, `../../../server/storage`), + `comkey` + ); + +class CommunicationKey { + #pubKeyName = "ipc-pub.pem"; + #storageLoc = keyPath; + + constructor() {} + + log(text, ...args) { + console.log(`\x1b[36m[CommunicationKeyVerify]\x1b[0m ${text}`, ...args); + } + + #readPublicKey() { + return fs.readFileSync(path.resolve(this.#storageLoc, this.#pubKeyName)); + } + + // Given a signed payload from private key from /app/server/ this signature should + // decode to match the textData provided. This class does verification only in collector. + // Note: The textData is typically the JSON stringified body sent to the document processor API. + verify(signature = "", textData = "") { + try { + let data = textData; + if (typeof textData !== "string") data = JSON.stringify(data); + return crypto.verify( + "RSA-SHA256", + Buffer.from(data), + this.#readPublicKey(), + Buffer.from(signature, "hex") + ); + } catch {} + return false; + } + + // Use the rolling public-key to decrypt arbitrary data that was encrypted via the private key on the server side CommunicationKey class + // that we know was done with the same key-pair and the given input is in base64 format already. + // Returns plaintext string of the data that was encrypted. + decrypt(base64String = "") { + return crypto + .publicDecrypt(this.#readPublicKey(), Buffer.from(base64String, "base64")) + .toString(); + } +} + +module.exports = { CommunicationKey }; diff --git a/collector/utils/constants.js b/collector/utils/constants.js new file mode 100644 index 0000000..4813e6e --- /dev/null +++ b/collector/utils/constants.js @@ -0,0 +1,92 @@ +const WATCH_DIRECTORY = require("path").resolve(__dirname, "../hotdir"); + +const ACCEPTED_MIMES = { + "text/plain": [".txt", ".md", ".org", ".adoc", ".rst"], + "text/html": [".html"], + "text/csv": [".csv"], + "application/json": [".json"], + // TODO: Create asDoc.js that works for standard MS Word files. + // "application/msword": [".doc"], + + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [ + ".docx", + ], + "application/vnd.openxmlformats-officedocument.presentationml.presentation": [ + ".pptx", + ], + + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [ + ".xlsx", + ], + + "application/vnd.oasis.opendocument.text": [".odt"], + "application/vnd.oasis.opendocument.presentation": [".odp"], + + "application/pdf": [".pdf"], + "application/mbox": [".mbox"], + + "audio/wav": [".wav"], + "audio/mpeg": [".mp3"], + "audio/ogg": [".ogg", ".oga"], + "audio/opus": [".opus"], + "audio/mp4": [".m4a"], + "audio/x-m4a": [".m4a"], + "audio/webm": [".webm"], + + "video/mp4": [".mp4"], + "video/mpeg": [".mpeg"], + "application/epub+zip": [".epub"], + "image/png": [".png"], + "image/jpeg": [".jpg"], + "image/jpg": [".jpg"], + "image/webp": [".webp"], +}; + +const SUPPORTED_FILETYPE_CONVERTERS = { + ".txt": "./convert/asTxt.js", + ".md": "./convert/asTxt.js", + ".org": "./convert/asTxt.js", + ".adoc": "./convert/asTxt.js", + ".rst": "./convert/asTxt.js", + ".csv": "./convert/asTxt.js", + ".json": "./convert/asTxt.js", + + ".html": "./convert/asTxt.js", + ".pdf": "./convert/asPDF/index.js", + + ".docx": "./convert/asDocx.js", + // TODO: Create asDoc.js that works for standard MS Word files. + // ".doc": "./convert/asDoc.js", + + ".pptx": "./convert/asOfficeMime.js", + + ".odt": "./convert/asOfficeMime.js", + ".odp": "./convert/asOfficeMime.js", + + ".xlsx": "./convert/asXlsx.js", + + ".mbox": "./convert/asMbox.js", + + ".epub": "./convert/asEPub.js", + + ".mp3": "./convert/asAudio.js", + ".wav": "./convert/asAudio.js", + ".mp4": "./convert/asAudio.js", + ".mpeg": "./convert/asAudio.js", + ".ogg": "./convert/asAudio.js", + ".oga": "./convert/asAudio.js", + ".opus": "./convert/asAudio.js", + ".m4a": "./convert/asAudio.js", + ".webm": "./convert/asAudio.js", + + ".png": "./convert/asImage.js", + ".jpg": "./convert/asImage.js", + ".jpeg": "./convert/asImage.js", + ".webp": "./convert/asImage.js", +}; + +module.exports = { + SUPPORTED_FILETYPE_CONVERTERS, + WATCH_DIRECTORY, + ACCEPTED_MIMES, +}; diff --git a/collector/utils/downloadURIToFile/index.js b/collector/utils/downloadURIToFile/index.js new file mode 100644 index 0000000..ce91264 --- /dev/null +++ b/collector/utils/downloadURIToFile/index.js @@ -0,0 +1,90 @@ +const { WATCH_DIRECTORY, ACCEPTED_MIMES } = require("../constants"); +const fs = require("fs"); +const path = require("path"); +const { pipeline } = require("stream/promises"); +const { validURL } = require("../url"); +const { default: slugify } = require("slugify"); + +// Add a custom slugify extension for slashing to handle URLs with paths. +slugify.extend({ "/": "-" }); + +/** + * Maps a MIME type to the preferred file extension using ACCEPTED_MIMES. + * Returns null if the MIME type is not recognized or if there are no possible extensions. + * @param {string} mimeType - The MIME type to resolve (e.g., "application/pdf") + * @returns {string|null} - The file extension (e.g., ".pdf") or null + */ +function mimeToExtension(mimeType) { + if (!mimeType || !ACCEPTED_MIMES.hasOwnProperty(mimeType)) return null; + const possibleExtensions = ACCEPTED_MIMES[mimeType] ?? []; + if (possibleExtensions.length === 0) return null; + return possibleExtensions[0]; +} + +/** + * Download a file to the hotdir + * @param {string} url - The URL of the file to download + * @param {number} maxTimeout - The maximum timeout in milliseconds + * @returns {Promise<{success: boolean, fileLocation: string|null, reason: string|null}>} - The path to the downloaded file + */ +async function downloadURIToFile(url, maxTimeout = 10_000) { + if (!url || typeof url !== "string" || !validURL(url)) + return { success: false, reason: "Not a valid URL.", fileLocation: null }; + + try { + const abortController = new AbortController(); + const timeout = setTimeout(() => { + abortController.abort(); + console.error( + `Timeout ${maxTimeout}ms reached while downloading file for URL:`, + url.toString() + ); + }, maxTimeout); + + const res = await fetch(url, { signal: abortController.signal }) + .then((res) => { + if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`); + return res; + }) + .finally(() => clearTimeout(timeout)); + + const urlObj = new URL(url); + const sluggedPath = slugify(urlObj.pathname, { lower: true }); + let filename = `${urlObj.hostname}-${sluggedPath}`; + + const existingExt = path.extname(filename).toLowerCase(); + const { SUPPORTED_FILETYPE_CONVERTERS } = require("../constants"); + + // If the filename does not already have a supported file extension, + // try to infer one from the response Content-Type header. + // This handles URLs like https://arxiv.org/pdf/2307.10265 where the + // path has no explicit extension but the server responds with + // Content-Type: application/pdf. + if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(existingExt)) { + const { parseContentType } = require("../../processLink/helpers"); + const contentType = parseContentType(res.headers.get("Content-Type")); + const inferredExt = mimeToExtension(contentType); + if (inferredExt) { + console.log( + `[Collector] URL path has no recognized extension. Inferred ${inferredExt} from Content-Type: ${contentType}` + ); + filename += inferredExt; + } + } + + const localFilePath = path.join(WATCH_DIRECTORY, filename); + const writeStream = fs.createWriteStream(localFilePath); + await pipeline(res.body, writeStream); + + console.log(`[SUCCESS]: File ${localFilePath} downloaded to hotdir.`); + return { success: true, fileLocation: localFilePath, reason: null }; + } catch (error) { + console.error(`Error writing to hotdir: ${error} for URL: ${url}`); + return { success: false, reason: error.message, fileLocation: null }; + } +} + +module.exports = { + downloadURIToFile, + mimeToExtension, +}; diff --git a/collector/utils/extensions/Confluence/ConfluenceLoader/index.js b/collector/utils/extensions/Confluence/ConfluenceLoader/index.js new file mode 100644 index 0000000..687cf2a --- /dev/null +++ b/collector/utils/extensions/Confluence/ConfluenceLoader/index.js @@ -0,0 +1,154 @@ +/* + * This is a custom implementation of the Confluence langchain loader. There was an issue where + * code blocks were not being extracted. This is a temporary fix until this issue is resolved.*/ + +const { htmlToText } = require("html-to-text"); + +class ConfluencePagesLoader { + constructor({ + baseUrl, + spaceKey, + username, + accessToken, + limit = 25, + expand = "body.storage,version", + personalAccessToken, + cloud = true, + bypassSSL = false, + }) { + this.baseUrl = baseUrl; + this.spaceKey = spaceKey; + this.username = username; + this.accessToken = accessToken; + this.limit = limit; + this.expand = expand; + this.personalAccessToken = personalAccessToken; + this.cloud = cloud; + this.bypassSSL = bypassSSL; + this.log("Initialized Confluence Loader"); + if (this.bypassSSL) + this.log("!!SSL bypass is enabled!! Use at your own risk!!"); + } + + log(message, ...args) { + console.log(`\x1b[36m[Confluence Loader]\x1b[0m ${message}`, ...args); + } + + get authorizationHeader() { + if (this.personalAccessToken) { + return `Bearer ${this.personalAccessToken}`; + } else if (this.username && this.accessToken) { + const authToken = Buffer.from( + `${this.username}:${this.accessToken}` + ).toString("base64"); + return `Basic ${authToken}`; + } + return undefined; + } + + async load(options) { + try { + const pages = await this.fetchAllPagesInSpace( + options?.start, + options?.limit + ); + return pages.map((page) => this.createDocumentFromPage(page)); + } catch (error) { + this.log("Error:", error); + return []; + } + } + + async fetchConfluenceData(url) { + try { + const initialHeaders = { + "Content-Type": "application/json", + Accept: "application/json", + }; + const authHeader = this.authorizationHeader; + if (authHeader) initialHeaders.Authorization = authHeader; + + // If SSL bypass is enabled, set the NODE_TLS_REJECT_UNAUTHORIZED environment variable + if (this.bypassSSL) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; + const response = await fetch(url, { headers: initialHeaders }); + if (!response.ok) { + throw new Error( + `Failed to fetch ${url} from Confluence: ${response.status}` + ); + } + return await response.json(); + } catch (error) { + this.log("Error:", error); + throw new Error(error.message); + } finally { + if (this.bypassSSL) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1"; + } + } + + // https://developer.atlassian.com/cloud/confluence/rest/v2/intro/#auth + async fetchAllPagesInSpace(start = 0, limit = this.limit) { + const url = `${this.baseUrl}${ + this.cloud ? "/wiki" : "" + }/rest/api/content?spaceKey=${ + this.spaceKey + }&limit=${limit}&start=${start}&expand=${this.expand}`; + const data = await this.fetchConfluenceData(url); + if (data.size === 0) { + return []; + } + const nextPageStart = start + data.size; + const nextPageResults = await this.fetchAllPagesInSpace( + nextPageStart, + limit + ); + return data.results.concat(nextPageResults); + } + + createDocumentFromPage(page) { + // Function to extract code blocks + const extractCodeBlocks = (content) => { + const codeBlockRegex = + /]*>[\s\S]*?<\/ac:plain-text-body>[\s\S]*?<\/ac:structured-macro>/g; + const languageRegex = + /(.*?)<\/ac:parameter>/; + + return content.replace(codeBlockRegex, (match) => { + const language = match.match(languageRegex)?.[1] || ""; + const code = + match.match( + /<\/ac:plain-text-body>/ + )?.[1] || ""; + return `\n\`\`\`${language}\n${code.trim()}\n\`\`\`\n`; + }); + }; + + const contentWithCodeBlocks = extractCodeBlocks(page.body.storage.value); + const plainTextContent = htmlToText(contentWithCodeBlocks, { + wordwrap: false, + preserveNewlines: true, + }); + const textWithPreservedStructure = plainTextContent.replace( + /\n{3,}/g, + "\n\n" + ); + const pageUrl = `${this.baseUrl}${this.cloud ? "/wiki" : ""}/spaces/${ + this.spaceKey + }/pages/${page.id}`; + + return { + pageContent: textWithPreservedStructure, + metadata: { + id: page.id, + status: page.status, + title: page.title, + type: page.type, + url: pageUrl, + version: page.version?.number, + updated_by: page.version?.by?.displayName, + updated_at: page.version?.when, + }, + }; + } +} + +module.exports = { ConfluencePagesLoader }; diff --git a/collector/utils/extensions/Confluence/index.js b/collector/utils/extensions/Confluence/index.js new file mode 100644 index 0000000..b4a0b24 --- /dev/null +++ b/collector/utils/extensions/Confluence/index.js @@ -0,0 +1,292 @@ +const fs = require("fs"); +const path = require("path"); +const { default: slugify } = require("slugify"); +const { v4 } = require("uuid"); +const { writeToServerDocuments, sanitizeFileName } = require("../../files"); +const { tokenizeString } = require("../../tokenizer"); +const { ConfluencePagesLoader } = require("./ConfluenceLoader"); + +/** + * Load Confluence documents from a spaceID and Confluence credentials + * @param {object} args - forwarded request body params + * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker + * @returns + */ +async function loadConfluence( + { + baseUrl = null, + spaceKey = null, + username = null, + accessToken = null, + cloud = true, + personalAccessToken = null, + bypassSSL = false, + }, + response +) { + if (!personalAccessToken && (!username || !accessToken)) { + return { + success: false, + reason: + "You need either a personal access token (PAT), or a username and access token to use the Confluence connector.", + }; + } + + if (!baseUrl || !validBaseUrl(baseUrl)) { + return { + success: false, + reason: "Provided base URL is not a valid URL.", + }; + } + + if (!spaceKey) { + return { + success: false, + reason: "You need to provide a Confluence space key.", + }; + } + + const normalizedBaseUrl = resolveConfluenceBaseUrl(baseUrl, cloud); + const { hostname } = new URL(normalizedBaseUrl); + console.log(`-- Working Confluence ${normalizedBaseUrl} --`); + const loader = new ConfluencePagesLoader({ + baseUrl: normalizedBaseUrl, + spaceKey, + username, + accessToken, + cloud, + personalAccessToken, + bypassSSL, + }); + + const { docs, error } = await loader + .load() + .then((docs) => { + return { docs, error: null }; + }) + .catch((e) => { + return { + docs: [], + error: e.message?.split("Error:")?.[1] || e.message, + }; + }); + + if (!docs.length || !!error) { + return { + success: false, + reason: error ?? "No pages found for that Confluence space.", + }; + } + const outFolder = slugify( + `confluence-${hostname}-${v4().slice(0, 4)}` + ).toLowerCase(); + + const outFolderPath = + process.env.NODE_ENV === "development" + ? path.resolve( + __dirname, + `../../../../server/storage/documents/${outFolder}` + ) + : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`); + + if (!fs.existsSync(outFolderPath)) + fs.mkdirSync(outFolderPath, { recursive: true }); + + docs.forEach((doc) => { + if (!doc.pageContent) return; + + const data = { + id: v4(), + url: doc.metadata.url + ".page", + title: doc.metadata.title || doc.metadata.source, + docAuthor: normalizedBaseUrl, + description: doc.metadata.title, + docSource: `${normalizedBaseUrl} Confluence`, + chunkSource: generateChunkSource( + { + doc, + baseUrl: normalizedBaseUrl, + spaceKey, + accessToken, + username, + cloud, + bypassSSL, + }, + response.locals.encryptionWorker + ), + published: new Date().toLocaleString(), + wordCount: doc.pageContent.split(" ").length, + pageContent: doc.pageContent, + token_count_estimate: tokenizeString(doc.pageContent), + }; + + console.log( + `[Confluence Loader]: Saving ${doc.metadata.title} to ${outFolder}` + ); + + const fileName = sanitizeFileName( + `${slugify(doc.metadata.title)}-${data.id}` + ); + writeToServerDocuments({ + data, + filename: fileName, + destinationOverride: outFolderPath, + }); + }); + + return { + success: true, + reason: null, + data: { + spaceKey, + destination: outFolder, + }, + }; +} + +/** + * Gets the page content from a specific Confluence page, not all pages in a workspace. + * @returns + */ +async function fetchConfluencePage({ + pageUrl, + baseUrl, + spaceKey, + username, + accessToken, + cloud = true, + bypassSSL = false, +}) { + if (!pageUrl || !baseUrl || !spaceKey || !username || !accessToken) { + return { + success: false, + content: null, + reason: + "You need either a username and access token, or a personal access token (PAT), to use the Confluence connector.", + }; + } + + if (!validBaseUrl(baseUrl)) { + return { + success: false, + content: null, + reason: "Provided base URL is not a valid URL.", + }; + } + + if (!spaceKey) { + return { + success: false, + content: null, + reason: "You need to provide a Confluence space key.", + }; + } + + console.log(`-- Working Confluence Page ${pageUrl} --`); + const normalizedBaseUrl = resolveConfluenceBaseUrl(baseUrl, cloud); + const loader = new ConfluencePagesLoader({ + baseUrl: normalizedBaseUrl, + spaceKey, + username, + accessToken, + cloud, + bypassSSL, + }); + + const { docs, error } = await loader + .load() + .then((docs) => { + return { docs, error: null }; + }) + .catch((e) => { + return { + docs: [], + error: e.message?.split("Error:")?.[1] || e.message, + }; + }); + + if (!docs.length || !!error) { + return { + success: false, + reason: error ?? "No pages found for that Confluence space.", + content: null, + }; + } + + const targetDocument = docs.find( + (doc) => doc.pageContent && doc.metadata.url === pageUrl + ); + if (!targetDocument) { + return { + success: false, + reason: "Target page could not be found in Confluence space.", + content: null, + }; + } + + return { + success: true, + reason: null, + content: targetDocument.pageContent, + }; +} + +/** + * Validates if the provided baseUrl is a valid URL at all. + * @param {string} baseUrl + * @returns {boolean} + */ +function validBaseUrl(baseUrl) { + try { + new URL(baseUrl); + return true; + } catch { + return false; + } +} + +/** + * Resolves the Confluence base URL, preserving context paths for self-hosted deployments. + * @param {string} baseUrl + * @param {boolean} cloud + * @returns {string} + */ +function resolveConfluenceBaseUrl(baseUrl, cloud = true) { + const url = new URL(baseUrl); + // Cloud URLs use just the origin; self-hosted may have a context path like /confluence + if (cloud) return url.origin; + + const contextPath = url.pathname.replace(/\/+$/, ""); + return `${url.origin}${contextPath}`; +} + +/** + * Generate the full chunkSource for a specific Confluence page so that we can resync it later. + * This data is encrypted into a single `payload` query param so we can replay credentials later + * since this was encrypted with the systems persistent password and salt. + * @param {object} chunkSourceInformation + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + * @returns {string} + */ +function generateChunkSource( + { doc, baseUrl, spaceKey, accessToken, username, cloud, bypassSSL }, + encryptionWorker +) { + const payload = { + baseUrl, + spaceKey, + token: accessToken, + username, + cloud, + bypassSSL, + }; + return `confluence://${doc.metadata.url}?payload=${encryptionWorker.encrypt( + JSON.stringify(payload) + )}`; +} + +module.exports = { + loadConfluence, + fetchConfluencePage, + resolveConfluenceBaseUrl, +}; diff --git a/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js b/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js new file mode 100644 index 0000000..24b1f02 --- /dev/null +++ b/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js @@ -0,0 +1,338 @@ +/** + * Copyright 2024 + * + * Authors: + * - Eugen Mayer (KontextWork) + */ + +const { htmlToText } = require("html-to-text"); +const { tokenizeString } = require("../../../tokenizer"); +const { + sanitizeFileName, + writeToServerDocuments, + documentsFolder, + normalizePath, + isWithin, +} = require("../../../files"); +const { default: slugify } = require("slugify"); +const path = require("path"); +const fs = require("fs"); +const { processSingleFile } = require("../../../../processSingleFile"); +const { + WATCH_DIRECTORY, + SUPPORTED_FILETYPE_CONVERTERS, +} = require("../../../constants"); + +class Page { + /** + * + * @param {number }id + * @param {string }title + * @param {string} created + * @param {string} type + * @param {string} processedBody + * @param {string} url + * @param {number} spaceId + */ + constructor({ id, title, created, type, processedBody, url, spaceId }) { + this.id = id; + this.title = title; + this.url = url; + this.created = created; + this.type = type; + this.processedBody = processedBody; + this.spaceId = spaceId; + } +} + +class DrupalWiki { + /** + * + * @param baseUrl + * @param spaceId + * @param accessToken + */ + constructor({ baseUrl, accessToken }) { + this.baseUrl = baseUrl; + this.accessToken = accessToken; + this.storagePath = this.#prepareStoragePath(baseUrl); + } + + /** + * Load all pages for the given space, fetching storing each page one by one + * to minimize the memory usage + * + * @param {number} spaceId + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + * @returns {Promise} + */ + async loadAndStoreAllPagesForSpace(spaceId, encryptionWorker) { + const pageIndex = await this.#getPageIndexForSpace(spaceId); + for (const pageId of pageIndex) { + try { + const page = await this.loadPage(pageId); + + // Pages with an empty body will lead to embedding issues / exceptions + if (page.processedBody.trim() !== "") { + this.#storePage(page, encryptionWorker); + await this.#downloadAndProcessAttachments(page.id); + } else { + console.log(`Skipping page (${page.id}) since it has no content`); + } + } catch (e) { + console.error( + `Could not process DrupalWiki page ${pageId} (skipping and continuing): ` + ); + console.error(e); + } + } + } + + /** + * @param {number} pageId + * @returns {Promise} + */ + async loadPage(pageId) { + return this.#fetchPage(pageId); + } + + /** + * Fetches the page ids for the configured space + * @param {number} spaceId + * @returns{Promise} array of pageIds + */ + async #getPageIndexForSpace(spaceId) { + // errors on fetching the pageIndex is fatal, no error handling + let hasNext = true; + let pageIds = []; + let pageNr = 0; + do { + let { isLast, pageIdsForPage } = await this.#getPagesForSpacePaginated( + spaceId, + pageNr + ); + hasNext = !isLast; + pageNr++; + if (pageIdsForPage.length) { + pageIds = pageIds.concat(pageIdsForPage); + } + } while (hasNext); + + return pageIds; + } + + /** + * + * @param {number} pageNr + * @param {number} spaceId + * @returns {Promise<{isLast,pageIds}>} + */ + async #getPagesForSpacePaginated(spaceId, pageNr) { + /* + * { + * content: Page[], + * last: boolean, + * pageable: { + * pageNumber: number + * } + * } + */ + const data = await this._doFetch( + `${this.baseUrl}/api/rest/scope/api/page?size=100&space=${spaceId}&page=${pageNr}` + ); + + const pageIds = data.content.map((page) => { + return Number(page.id); + }); + + return { + isLast: data.last, + pageIdsForPage: pageIds, + }; + } + + /** + * @param pageId + * @returns {Promise} + */ + async #fetchPage(pageId) { + const data = await this._doFetch( + `${this.baseUrl}/api/rest/scope/api/page/${pageId}` + ); + const url = `${this.baseUrl}/node/${data.id}`; + return new Page({ + id: data.id, + title: data.title, + created: data.lastModified, + type: data.type, + processedBody: this.#processPageBody({ + body: data.body, + title: data.title, + lastModified: data.lastModified, + url: url, + }), + url: url, + }); + } + + /** + * @param {Page} page + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + */ + #storePage(page, encryptionWorker) { + const { hostname } = new URL(this.baseUrl); + + // This UUID will ensure that re-importing the same page without any changes will not + // show up (deduplication). + const targetUUID = `${hostname}.${page.spaceId}.${page.id}.${page.created}`; + const wordCount = page.processedBody.split(" ").length; + const data = { + id: targetUUID, + url: `drupalwiki://${page.url}`, + title: page.title, + docAuthor: this.baseUrl, + description: page.title, + docSource: `${this.baseUrl} DrupalWiki`, + chunkSource: this.#generateChunkSource(page.id, encryptionWorker), + published: new Date().toLocaleString(), + wordCount: wordCount, + pageContent: page.processedBody, + token_count_estimate: tokenizeString(page.processedBody), + }; + + const fileName = sanitizeFileName(`${slugify(page.title)}-${data.id}`); + console.log( + `[DrupalWiki Loader]: Saving page '${page.title}' (${page.id}) to '${this.storagePath}/${fileName}'` + ); + writeToServerDocuments({ + data, + filename: fileName, + destinationOverride: this.storagePath, + }); + } + + /** + * Generate the full chunkSource for a specific Confluence page so that we can resync it later. + * This data is encrypted into a single `payload` query param so we can replay credentials later + * since this was encrypted with the systems persistent password and salt. + * @param {number} pageId + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + * @returns {string} + */ + #generateChunkSource(pageId, encryptionWorker) { + const payload = { + baseUrl: this.baseUrl, + pageId: pageId, + accessToken: this.accessToken, + }; + return `drupalwiki://${ + this.baseUrl + }/node/${pageId}?payload=${encryptionWorker.encrypt( + JSON.stringify(payload) + )}`; + } + + async _doFetch(url) { + const response = await fetch(url, { + headers: this.#getHeaders(), + }); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status}`); + } + return response.json(); + } + + #getHeaders() { + return { + "Content-Type": "application/json", + Accept: "application/json", + Authorization: `Bearer ${this.accessToken}`, + }; + } + + #prepareStoragePath(baseUrl) { + const { hostname } = new URL(baseUrl); + const subFolder = slugify(`drupalwiki-${hostname}`).toLowerCase(); + const outFolder = path.resolve(documentsFolder, subFolder); + if (!fs.existsSync(outFolder)) fs.mkdirSync(outFolder, { recursive: true }); + return outFolder; + } + + /** + * @param {string} body + * @param {string} url + * @param {string} title + * @param {string} lastModified + * @returns {string} + * @private + */ + #processPageBody({ body, title }) { + const textContent = body.trim() !== "" ? body : title; + + const plainTextContent = htmlToText(textContent, { + wordwrap: false, + preserveNewlines: true, + selectors: [ + { + selector: "table", + format: "dataTable", + options: { + colSpacing: 3, + rowSpacing: 1, + uppercaseHeaderCells: true, + maxColumnWidth: Infinity, + }, + }, + ], + }); + + const plainBody = plainTextContent.replace(/\n{3,}/g, "\n\n"); + return plainBody; + } + + async #downloadAndProcessAttachments(pageId) { + try { + const data = await this._doFetch( + `${this.baseUrl}/api/rest/scope/api/attachment?pageId=${pageId}&size=2000` + ); + + const extensionsList = Object.keys(SUPPORTED_FILETYPE_CONVERTERS); + for (const attachment of data.content || data) { + const { fileName, id: attachId } = attachment; + const lowerName = fileName.toLowerCase(); + if (!extensionsList.some((ext) => lowerName.endsWith(ext))) { + continue; + } + + const downloadUrl = `${this.baseUrl}/api/rest/scope/api/attachment/${attachId}/download`; + const attachmentResponse = await fetch(downloadUrl, { + headers: this.#getHeaders(), + }); + if (!attachmentResponse.ok) { + console.log(`Skipping attachment: ${fileName} - Download failed`); + continue; + } + + const buffer = await attachmentResponse.arrayBuffer(); + const localFilePath = normalizePath( + sanitizeFileName(path.resolve(WATCH_DIRECTORY, fileName)) + ); + if (!isWithin(path.resolve(WATCH_DIRECTORY), localFilePath)) { + console.error( + `[DrupalWiki Loader]: File name ${localFilePath} is not within the storage path ${path.resolve( + WATCH_DIRECTORY + )}` + ); + continue; + } + + require("fs").writeFileSync(localFilePath, Buffer.from(buffer)); + await processSingleFile(localFilePath); + } + } catch (err) { + console.error(`Fetching/processing attachments failed:`, err); + } + } +} + +module.exports = { DrupalWiki }; diff --git a/collector/utils/extensions/DrupalWiki/index.js b/collector/utils/extensions/DrupalWiki/index.js new file mode 100644 index 0000000..414ce89 --- /dev/null +++ b/collector/utils/extensions/DrupalWiki/index.js @@ -0,0 +1,102 @@ +/** + * Copyright 2024 + * + * Authors: + * - Eugen Mayer (KontextWork) + */ + +const { DrupalWiki } = require("./DrupalWiki"); +const { validBaseUrl } = require("../../../utils/http"); + +async function loadAndStoreSpaces( + { baseUrl = null, spaceIds = null, accessToken = null }, + response +) { + if (!baseUrl) { + return { + success: false, + reason: + "Please provide your baseUrl like https://mywiki.drupal-wiki.net.", + }; + } else if (!validBaseUrl(baseUrl)) { + return { + success: false, + reason: "Provided base URL is not a valid URL.", + }; + } + + if (!spaceIds) { + return { + success: false, + reason: + "Please provide a list of spaceIds like 21,56,67 you want to extract", + }; + } + + if (!accessToken) { + return { + success: false, + reason: "Please provide a REST API-Token.", + }; + } + + console.log(`-- Working Drupal Wiki ${baseUrl} for spaceIds: ${spaceIds} --`); + const drupalWiki = new DrupalWiki({ baseUrl, accessToken }); + + const encryptionWorker = response.locals.encryptionWorker; + const spaceIdsArr = spaceIds.split(",").map((idStr) => { + return Number(idStr.trim()); + }); + + for (const spaceId of spaceIdsArr) { + try { + await drupalWiki.loadAndStoreAllPagesForSpace(spaceId, encryptionWorker); + console.log(`--- Finished space ${spaceId} ---`); + } catch (e) { + console.error(e); + return { + success: false, + reason: e.message, + data: {}, + }; + } + } + console.log(`-- Finished all spaces--`); + + return { + success: true, + reason: null, + data: { + spaceIds, + destination: drupalWiki.storagePath, + }, + }; +} + +/** + * Gets the page content from a specific Confluence page, not all pages in a workspace. + * @returns + */ +async function loadPage({ baseUrl, pageId, accessToken }) { + console.log(`-- Working Drupal Wiki Page ${pageId} of ${baseUrl} --`); + const drupalWiki = new DrupalWiki({ baseUrl, accessToken }); + try { + const page = await drupalWiki.loadPage(pageId); + return { + success: true, + reason: null, + content: page.processedBody, + }; + } catch { + return { + success: false, + reason: `Failed (re)-fetching DrupalWiki page ${pageId} form ${baseUrl}}`, + content: null, + }; + } +} + +module.exports = { + loadAndStoreSpaces, + loadPage, +}; diff --git a/collector/utils/extensions/ObsidianVault/index.js b/collector/utils/extensions/ObsidianVault/index.js new file mode 100644 index 0000000..52d4bbb --- /dev/null +++ b/collector/utils/extensions/ObsidianVault/index.js @@ -0,0 +1,95 @@ +const { v4 } = require("uuid"); +const { default: slugify } = require("slugify"); +const path = require("path"); +const fs = require("fs"); +const { + writeToServerDocuments, + sanitizeFileName, + documentsFolder, +} = require("../../files"); + +function parseObsidianVaultPath(files = []) { + const possiblePaths = new Set(); + files.forEach( + (file) => file?.path && possiblePaths.add(file.path.split("/")[0]) + ); + + switch (possiblePaths.size) { + case 0: + return null; + case 1: + // The user specified a vault properly - so all files are in the same folder. + return possiblePaths.values().next().value; + default: + return null; + } +} + +async function loadObsidianVault({ files = [] }) { + if (!files || files?.length === 0) + return { success: false, error: "No files provided" }; + const vaultName = parseObsidianVaultPath(files); + const folderUUId = v4().slice(0, 4); + const outFolder = vaultName + ? slugify(`obsidian-vault-${vaultName}-${folderUUId}`).toLowerCase() + : slugify(`obsidian-${folderUUId}`).toLowerCase(); + const outFolderPath = path.resolve(documentsFolder, outFolder); + if (!fs.existsSync(outFolderPath)) + fs.mkdirSync(outFolderPath, { recursive: true }); + + console.log( + `Processing ${files.length} files from Obsidian Vault ${ + vaultName ? `"${vaultName}"` : "" + }` + ); + const results = []; + for (const file of files) { + try { + const fullPageContent = file?.content; + // If the file has no content or is just whitespace, skip it. + if (!fullPageContent || fullPageContent.trim() === "") continue; + + const data = { + id: v4(), + url: `obsidian://${file.path}`, + title: file.name, + docAuthor: "Obsidian Vault", + description: file.name, + docSource: "Obsidian Vault", + chunkSource: `obsidian://${file.path}`, + published: new Date().toLocaleString(), + wordCount: fullPageContent.split(" ").length, + pageContent: fullPageContent, + token_count_estimate: fullPageContent.length / 4, // rough estimate + }; + + const targetFileName = sanitizeFileName( + `${slugify(file.name)}-${data.id}` + ); + writeToServerDocuments({ + data, + filename: targetFileName, + destinationOverride: outFolderPath, + }); + results.push({ file: file.path, status: "success" }); + } catch (e) { + console.error(`Failed to process ${file.path}:`, e); + results.push({ file: file.path, status: "failed", reason: e.message }); + } + } + + return { + success: true, + data: { + processed: results.filter((r) => r.status === "success").length, + failed: results.filter((r) => r.status === "failed").length, + total: files.length, + results, + destination: path.basename(outFolderPath), + }, + }; +} + +module.exports = { + loadObsidianVault, +}; diff --git a/collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js b/collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js new file mode 100644 index 0000000..ab8c9f5 --- /dev/null +++ b/collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js @@ -0,0 +1,157 @@ +const { htmlToText } = require("html-to-text"); +const pdf = require("pdf-parse"); + +class PaperlessNgxLoader { + constructor({ baseUrl, apiToken }) { + this.baseUrl = new URL(baseUrl).origin; + this.apiToken = apiToken; + this.baseHeaders = { + Authorization: `Token ${this.apiToken}`, + }; + } + + async load() { + try { + const documents = await this.fetchAllDocuments(); + return documents.map((doc) => this.createDocumentFromPage(doc)); + } catch (error) { + console.error("Error:", error); + throw error; + } + } + + /** + * Fetches all documents from Paperless-ngx + * @returns {Promise<{{[key: string]: any, content: string}[]}>} The documents with their content + */ + async fetchAllDocuments() { + try { + const documents = []; + let nextUrl = `${this.baseUrl}/api/documents/`; + let page = 1; + + while (nextUrl) { + console.log(`Fetching documents page ${page} from Paperless-ngx`); + try { + const data = await fetch(nextUrl, { + headers: { + "Content-Type": "application/json", + ...this.baseHeaders, + }, + }).then((res) => { + if (!res.ok) + throw new Error( + `Failed to fetch documents from Paperless-ngx: ${res.status}` + ); + return res.json(); + }); + + const validResults = data.results.filter((doc) => doc?.id); + if (!validResults.length) break; + + documents.push(...validResults); + + if (data.next === nextUrl) break; + nextUrl = data.next || null; + page++; + } catch (error) { + console.error( + `Error fetching page ${page} from Paperless-ngx:`, + error + ); + break; + } + } + + console.log( + `Fetched ${documents.length} documents from Paperless-ngx (Pages: ${ + page - 1 + })` + ); + + const documentsWithContent = await Promise.all( + documents.map(async (doc) => { + const content = await this.fetchDocumentContent(doc.id); + return { ...doc, content }; + }) + ); + + return documentsWithContent.filter((doc) => !!doc.content); + } catch (error) { + throw new Error( + `Failed to fetch documents from Paperless-ngx: ${error.message}` + ); + } + } + + /** + * Fetches the content of a document from Paperless-ngx + * @param {string} documentId - The ID of the document to fetch + * @returns {Promise} The content of the document + */ + async fetchDocumentContent(documentId) { + try { + const response = await fetch( + `${this.baseUrl}/api/documents/${documentId}/download/`, + { + headers: this.baseHeaders, + } + ); + + if (!response.ok) + throw new Error(`Failed to fetch document content: ${response.status}`); + + const contentType = response.headers.get("content-type"); + switch (contentType) { + case "text/plain": + return await response.text(); + case "application/pdf": + const buffer = await response.arrayBuffer(); + return await this.parsePdfContent(buffer); + default: + return await response.text(); + } + } catch (error) { + console.error( + `Failed to fetch content for document ${documentId}:`, + error + ); + return ""; + } + } + + async parsePdfContent(buffer) { + try { + const data = await pdf(Buffer.from(buffer)); + return data.text; + } catch (error) { + console.error("Failed to parse PDF content:", error); + return ""; + } + } + + createDocumentFromPage(doc) { + const content = doc.content || ""; + const plainTextContent = htmlToText(content, { + wordwrap: false, + preserveNewlines: true, + }); + + return { + pageContent: plainTextContent, + metadata: { + id: doc.id, + title: doc.original_file_name, + created: doc.created, + modified: doc.modified, + added: doc.added, + tags: doc.tags, + correspondent: doc.correspondent, + documentType: doc.document_type, + url: `${this.baseUrl}/documents/${doc.id}`, + }, + }; + } +} + +module.exports = PaperlessNgxLoader; diff --git a/collector/utils/extensions/PaperlessNgx/index.js b/collector/utils/extensions/PaperlessNgx/index.js new file mode 100644 index 0000000..df8a6be --- /dev/null +++ b/collector/utils/extensions/PaperlessNgx/index.js @@ -0,0 +1,128 @@ +const fs = require("fs"); +const path = require("path"); +const { default: slugify } = require("slugify"); +const { v4 } = require("uuid"); +const { + writeToServerDocuments, + sanitizeFileName, + documentsFolder, +} = require("../../files"); +const { tokenizeString } = require("../../tokenizer"); +const { validBaseUrl } = require("../../http"); +const PaperlessNgxLoader = require("./PaperlessNgxLoader"); + +/** + * Load documents from a Paperless-ngx instance + * @param {object} args - forwarded request body params + * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker + * @returns + */ +async function loadPaperlessNgx({ baseUrl = null, apiToken = null }, response) { + if (!baseUrl || !validBaseUrl(baseUrl)) { + return { + success: false, + reason: "Provided base URL is not a valid URL.", + }; + } + + if (!apiToken) { + return { + success: false, + reason: + "You need to provide an API token to use the Paperless-ngx connector.", + }; + } + + const { origin, hostname } = new URL(baseUrl); + console.log(`-- Working Paperless-ngx ${origin} --`); + const loader = new PaperlessNgxLoader({ + baseUrl: origin, + apiToken, + }); + + const { docs, error } = await loader + .load() + .then((docs) => ({ docs, error: null })) + .catch((e) => ({ + docs: [], + error: e.message?.split("Error:")?.[1] || e.message, + })); + + if (!docs.length || !!error) { + return { + success: false, + reason: + error ?? "No parseable documents found in that Paperless-ngx instance.", + data: null, + }; + } + + const outFolder = slugify( + `paperless-${hostname}-${v4().slice(0, 4)}` + ).toLowerCase(); + const outFolderPath = path.resolve(documentsFolder, outFolder); + if (!fs.existsSync(outFolderPath)) + fs.mkdirSync(outFolderPath, { recursive: true }); + + docs.forEach((doc) => { + if (!doc.pageContent) return; + + const data = { + id: v4(), + url: doc.metadata.url, + title: doc.metadata.title, + docAuthor: doc.metadata.correspondent || "Unknown", + description: `A document from the Paperless-ngx instance at ${origin}`, + docSource: `paperless-ngx`, + chunkSource: generateChunkSource( + { doc, baseUrl: origin, apiToken }, + response.locals.encryptionWorker + ), + published: doc.metadata.created, + wordCount: doc.pageContent.split(" ").length, + pageContent: doc.pageContent, + token_count_estimate: tokenizeString(doc.pageContent), + }; + + console.log( + `[Paperless-ngx Loader]: Saving ${doc.metadata.title} to ${outFolder}` + ); + const fileName = sanitizeFileName( + `${slugify(doc.metadata.title)}-${data.id}` + ); + writeToServerDocuments({ + data, + filename: fileName, + destinationOverride: outFolderPath, + }); + }); + + return { + success: true, + reason: null, + data: { + files: docs.length, + destination: outFolder, + }, + }; +} + +/** + * Generate the full chunkSource for a specific Paperless-ngx document so that we can resync it later. + * @param {object} chunkSourceInformation + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + * @returns {string} + */ +function generateChunkSource({ doc, baseUrl, apiToken }, encryptionWorker) { + const payload = { + baseUrl, + token: apiToken, + }; + return `paperless-ngx://${doc.metadata.id}?payload=${encryptionWorker.encrypt( + JSON.stringify(payload) + )}`; +} + +module.exports = { + loadPaperlessNgx, +}; diff --git a/collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js b/collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js new file mode 100644 index 0000000..5880751 --- /dev/null +++ b/collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js @@ -0,0 +1,265 @@ +/** + * @typedef {Object} RepoLoaderArgs + * @property {string} repo - The GitHub repository URL. + * @property {string} [branch] - The branch to load from (optional). + * @property {string} [accessToken] - GitHub access token for authentication (optional). + * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional). + */ + +/** + * @class + * @classdesc Loads and manages GitHub repository content. + */ +class GitHubRepoLoader { + /** + * Creates an instance of RepoLoader. + * @param {RepoLoaderArgs} [args] - The configuration options. + * @returns {GitHubRepoLoader} + */ + constructor(args = {}) { + this.ready = false; + this.repo = this.#processRepoUrl(args?.repo); + this.branch = args?.branch; + this.accessToken = args?.accessToken || null; + this.ignorePaths = args?.ignorePaths || []; + + this.author = null; + this.project = null; + this.branches = []; + } + + /** + * Processes a repository URL to ensure it is in the correct format + * - remove the .git suffix if present + * - ensure the url is valid + * @param {string} repoUrl - The repository URL to process. + * @returns {string|null} The processed repository URL, or null if the URL is invalid. + */ + #processRepoUrl(repoUrl) { + if (!repoUrl) return repoUrl; + try { + const url = new URL(repoUrl); + if (url.pathname.endsWith(".git")) + url.pathname = url.pathname.slice(0, -4); + return url.toString(); + } catch (e) { + console.error( + `[GitHub Loader]: Error processing repository URL ${this.repo}: ${e.message}` + ); + return repoUrl; + } + } + + /** + * Validates the GitHub URL format. + * - ensure the url is valid + * - ensure the hostname is github.com + * - ensure the pathname is in the format of github.com/{author}/{project} + * - sets the author and project properties of class instance + * @returns {boolean} True if the URL is valid, false otherwise. + */ + #validGithubUrl() { + try { + const url = new URL(this.repo); + + // Not a github url at all. + if (url.hostname !== "github.com") { + console.log( + `[GitHub Loader]: Invalid GitHub URL provided! Hostname must be 'github.com'. Got ${url.hostname}` + ); + return false; + } + + // Assume the url is in the format of github.com/{author}/{project} + // Remove the first slash from the pathname so we can split it properly. + const [author, project, ..._rest] = url.pathname.slice(1).split("/"); + if (!author || !project) { + console.log( + `[GitHub Loader]: Invalid GitHub URL provided! URL must be in the format of 'github.com/{author}/{project}'. Got ${url.pathname}` + ); + return false; + } + + this.author = author; + this.project = project; + return true; + } catch (e) { + console.log( + `[GitHub Loader]: Invalid GitHub URL provided! Error: ${e.message}` + ); + return false; + } + } + + // Ensure the branch provided actually exists + // and if it does not or has not been set auto-assign to primary branch. + async #validBranch() { + await this.getRepoBranches(); + if (!!this.branch && this.branches.includes(this.branch)) return; + + console.log( + "[GitHub Loader]: Branch not set! Auto-assigning to a default branch." + ); + this.branch = this.branches.includes("main") ? "main" : "master"; + console.log(`[GitHub Loader]: Branch auto-assigned to ${this.branch}.`); + return; + } + + async #validateAccessToken() { + if (!this.accessToken) return; + const valid = await fetch("https://api.github.com/octocat", { + method: "GET", + headers: { + Authorization: `Bearer ${this.accessToken}`, + "X-GitHub-Api-Version": "2022-11-28", + }, + }) + .then((res) => { + if (!res.ok) throw new Error(res.statusText); + return res.ok; + }) + .catch((e) => { + console.error( + "Invalid GitHub Access Token provided! Access token will not be used", + e.message + ); + return false; + }); + + if (!valid) this.accessToken = null; + return; + } + + /** + * Initializes the RepoLoader instance. + * @returns {Promise} The initialized RepoLoader instance. + */ + async init() { + if (!this.#validGithubUrl()) return; + await this.#validBranch(); + await this.#validateAccessToken(); + this.ready = true; + return this; + } + + /** + * Recursively loads the repository content. + * @returns {Promise>} An array of loaded documents. + * @throws {Error} If the RepoLoader is not in a ready state. + */ + async recursiveLoader() { + if (!this.ready) throw new Error("[GitHub Loader]: not in ready state!"); + const { + GithubRepoLoader: LCGithubLoader, + } = require("@langchain/community/document_loaders/web/github"); + + if (this.accessToken) + console.log( + `[GitHub Loader]: Access token set! Recursive loading enabled!` + ); + + const loader = new LCGithubLoader(this.repo, { + branch: this.branch, + recursive: !!this.accessToken, // Recursive will hit rate limits. + maxConcurrency: 5, + unknown: "warn", + accessToken: this.accessToken, + ignorePaths: this.ignorePaths, + verbose: true, + }); + + const docs = await loader.load(); + return docs; + } + + // Sort branches to always show either main or master at the top of the result. + #branchPrefSort(branches = []) { + const preferredSort = ["main", "master"]; + return branches.reduce((acc, branch) => { + if (preferredSort.includes(branch)) return [branch, ...acc]; + return [...acc, branch]; + }, []); + } + + /** + * Retrieves all branches for the repository. + * @returns {Promise} An array of branch names. + */ + async getRepoBranches() { + if (!this.#validGithubUrl() || !this.author || !this.project) return []; + await this.#validateAccessToken(); // Ensure API access token is valid for pre-flight + + let page = 0; + let polling = true; + const branches = []; + + while (polling) { + console.log(`Fetching page ${page} of branches for ${this.project}`); + await fetch( + `https://api.github.com/repos/${this.author}/${this.project}/branches?per_page=100&page=${page}`, + { + method: "GET", + headers: { + ...(this.accessToken + ? { Authorization: `Bearer ${this.accessToken}` } + : {}), + "X-GitHub-Api-Version": "2022-11-28", + }, + } + ) + .then((res) => { + if (res.ok) return res.json(); + throw new Error(`Invalid request to Github API: ${res.statusText}`); + }) + .then((branchObjects) => { + polling = branchObjects.length > 0; + branches.push(branchObjects.map((branch) => branch.name)); + page++; + }) + .catch((err) => { + polling = false; + console.log(`RepoLoader.branches`, err); + }); + } + + this.branches = [...new Set(branches.flat())]; + return this.#branchPrefSort(this.branches); + } + + /** + * Fetches the content of a single file from the repository. + * @param {string} sourceFilePath - The path to the file in the repository. + * @returns {Promise} The content of the file, or null if fetching fails. + */ + async fetchSingleFile(sourceFilePath) { + try { + return fetch( + `https://api.github.com/repos/${this.author}/${this.project}/contents/${sourceFilePath}?ref=${this.branch}`, + { + method: "GET", + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + ...(!!this.accessToken + ? { Authorization: `Bearer ${this.accessToken}` } + : {}), + }, + } + ) + .then((res) => { + if (res.ok) return res.json(); + throw new Error(`Failed to fetch from Github API: ${res.statusText}`); + }) + .then((json) => { + if (json.hasOwnProperty("status") || !json.hasOwnProperty("content")) + throw new Error(json?.message || "missing content"); + return atob(json.content); + }); + } catch (e) { + console.error(`RepoLoader.fetchSingleFile`, e); + return null; + } + } +} + +module.exports = GitHubRepoLoader; diff --git a/collector/utils/extensions/RepoLoader/GithubRepo/index.js b/collector/utils/extensions/RepoLoader/GithubRepo/index.js new file mode 100644 index 0000000..fae6ef4 --- /dev/null +++ b/collector/utils/extensions/RepoLoader/GithubRepo/index.js @@ -0,0 +1,159 @@ +const RepoLoader = require("./RepoLoader"); +const fs = require("fs"); +const path = require("path"); +const { default: slugify } = require("slugify"); +const { v4 } = require("uuid"); +const { writeToServerDocuments } = require("../../../files"); +const { tokenizeString } = require("../../../tokenizer"); + +/** + * Load in a GitHub Repo recursively or just the top level if no PAT is provided + * @param {object} args - forwarded request body params + * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker + * @returns + */ +async function loadGithubRepo(args, response) { + const repo = new RepoLoader(args); + await repo.init(); + + if (!repo.ready) + return { + success: false, + reason: "Could not prepare GitHub repo for loading! Check URL", + }; + + console.log( + `-- Working GitHub ${repo.author}/${repo.project}:${repo.branch} --` + ); + const docs = await repo.recursiveLoader(); + if (!docs.length) { + return { + success: false, + reason: "No files were found for those settings.", + }; + } + + console.log(`[GitHub Loader]: Found ${docs.length} source files. Saving...`); + const outFolder = slugify( + `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}` + ).toLowerCase(); + + const outFolderPath = + process.env.NODE_ENV === "development" + ? path.resolve( + __dirname, + `../../../../../server/storage/documents/${outFolder}` + ) + : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`); + + if (!fs.existsSync(outFolderPath)) + fs.mkdirSync(outFolderPath, { recursive: true }); + + for (const doc of docs) { + if (!doc.pageContent) continue; + const data = { + id: v4(), + url: "github://" + doc.metadata.source, + title: doc.metadata.source, + docAuthor: repo.author, + description: "No description found.", + docSource: doc.metadata.source, + chunkSource: generateChunkSource( + repo, + doc, + response.locals.encryptionWorker + ), + published: new Date().toLocaleString(), + wordCount: doc.pageContent.split(" ").length, + pageContent: doc.pageContent, + token_count_estimate: tokenizeString(doc.pageContent), + }; + console.log( + `[GitHub Loader]: Saving ${doc.metadata.source} to ${outFolder}` + ); + writeToServerDocuments({ + data, + filename: `${slugify(doc.metadata.source)}-${data.id}`, + destinationOverride: outFolderPath, + }); + } + + return { + success: true, + reason: null, + data: { + author: repo.author, + repo: repo.project, + branch: repo.branch, + files: docs.length, + destination: outFolder, + }, + }; +} + +/** + * Gets the page content from a specific source file in a give GitHub Repo, not all items in a repo. + * @returns + */ +async function fetchGithubFile({ + repoUrl, + branch, + accessToken = null, + sourceFilePath, +}) { + const repo = new RepoLoader({ + repo: repoUrl, + branch, + accessToken, + }); + await repo.init(); + + if (!repo.ready) + return { + success: false, + content: null, + reason: "Could not prepare GitHub repo for loading! Check URL or PAT.", + }; + + console.log( + `-- Working GitHub ${repo.author}/${repo.project}:${repo.branch} file:${sourceFilePath} --` + ); + const fileContent = await repo.fetchSingleFile(sourceFilePath); + if (!fileContent) { + return { + success: false, + reason: "Target file returned a null content response.", + content: null, + }; + } + + return { + success: true, + reason: null, + content: fileContent, + }; +} + +/** + * Generate the full chunkSource for a specific file so that we can resync it later. + * This data is encrypted into a single `payload` query param so we can replay credentials later + * since this was encrypted with the systems persistent password and salt. + * @param {RepoLoader} repo + * @param {import("@langchain/core/documents").Document} doc + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + * @returns {string} + */ +function generateChunkSource(repo, doc, encryptionWorker) { + const payload = { + owner: repo.author, + project: repo.project, + branch: repo.branch, + path: doc.metadata.source, + pat: !!repo.accessToken ? repo.accessToken : null, + }; + return `github://${repo.repo}?payload=${encryptionWorker.encrypt( + JSON.stringify(payload) + )}`; +} + +module.exports = { loadGithubRepo, fetchGithubFile }; diff --git a/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js new file mode 100644 index 0000000..7a0310d --- /dev/null +++ b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js @@ -0,0 +1,447 @@ +const ignore = require("ignore"); +const MAX_RETRIES = 3; + +/** + * @typedef {Object} RepoLoaderArgs + * @property {string} repo - The GitLab repository URL. + * @property {string} [branch] - The branch to load from (optional). + * @property {string} [accessToken] - GitLab access token for authentication (optional). + * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional). + * @property {boolean} [fetchIssues] - Should issues be fetched (optional). + * @property {boolean} [fetchWikis] - Should wiki be fetched (optional). + */ + +/** + * @typedef {Object} FileTreeObject + * @property {string} id - The file object ID. + * @property {string} name - name of file. + * @property {('blob'|'tree')} type - type of file object. + * @property {string} path - path + name of file. + * @property {string} mode - Linux permission code. + */ + +/** + * @class + * @classdesc Loads and manages GitLab repository content. + */ +class GitLabRepoLoader { + /** + * Creates an instance of RepoLoader. + * @param {RepoLoaderArgs} [args] - The configuration options. + * @returns {GitLabRepoLoader} + */ + constructor(args = {}) { + this.ready = false; + this.repo = args?.repo; + this.branch = args?.branch; + this.accessToken = args?.accessToken || null; + this.ignorePaths = args?.ignorePaths || []; + this.ignoreFilter = ignore().add(this.ignorePaths); + this.withIssues = args?.fetchIssues || false; + this.withWikis = args?.fetchWikis || false; + + this.projectId = null; + this.apiBase = "https://gitlab.com"; + this.author = null; + this.project = null; + this.branches = []; + } + + #wait(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + #validGitlabUrl() { + const validPatterns = [ + //eslint-disable-next-line + /https:\/\/gitlab\.com\/(?[^\/]+)\/(?.*)/, + // This should even match the regular hosted URL, but we may want to know + // if this was a hosted GitLab (above) or a self-hosted (below) instance + // since the API interface could be different. + //eslint-disable-next-line + /(http|https):\/\/[^\/]+\/(?[^\/]+)\/(?.*)/, + ]; + + const match = validPatterns + .find((pattern) => this.repo.match(pattern)?.groups) + ?.exec(this.repo); + if (!match?.groups) return false; + + const { author, project } = match.groups; + this.projectId = encodeURIComponent(`${author}/${project}`); + this.apiBase = new URL(this.repo).origin; + this.author = author; + this.project = project; + return true; + } + + async #validBranch() { + await this.getRepoBranches(); + if (!!this.branch && this.branches.includes(this.branch)) return; + + console.log( + "[Gitlab Loader]: Branch not set! Auto-assigning to a default branch." + ); + this.branch = this.branches.includes("main") ? "main" : "master"; + console.log(`[Gitlab Loader]: Branch auto-assigned to ${this.branch}.`); + return; + } + + async #validateAccessToken() { + if (!this.accessToken) return; + try { + await fetch(`${this.apiBase}/api/v4/user`, { + method: "GET", + headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}, + }).then((res) => res.ok); + } catch (e) { + console.error( + "Invalid Gitlab Access Token provided! Access token will not be used", + e.message + ); + this.accessToken = null; + } + } + + /** + * Initializes the RepoLoader instance. + * @returns {Promise} The initialized RepoLoader instance. + */ + async init() { + if (!this.#validGitlabUrl()) return; + await this.#validBranch(); + await this.#validateAccessToken(); + this.ready = true; + return this; + } + + /** + * Recursively loads the repository content. + * @returns {Promise>} An array of loaded documents. + * @throws {Error} If the RepoLoader is not in a ready state. + */ + async recursiveLoader() { + if (!this.ready) throw new Error("[Gitlab Loader]: not in ready state!"); + + if (this.accessToken) + console.log( + `[Gitlab Loader]: Access token set! Recursive loading enabled for ${this.repo}!` + ); + + const docs = []; + + console.log(`[Gitlab Loader]: Fetching files.`); + + const files = await this.fetchFilesRecursive(); + + console.log(`[Gitlab Loader]: Fetched ${files.length} files.`); + + for (const file of files) { + if (this.ignoreFilter.ignores(file.path)) continue; + + docs.push({ + pageContent: file.content, + metadata: { + source: file.path, + url: `${this.repo}/-/blob/${this.branch}/${file.path}`, + }, + }); + } + + if (this.withIssues) { + console.log(`[Gitlab Loader]: Fetching issues.`); + const issues = await this.fetchIssues(); + console.log( + `[Gitlab Loader]: Fetched ${issues.length} issues with discussions.` + ); + docs.push( + ...issues.map((issue) => ({ + issue, + metadata: { + source: `issue-${this.repo}-${issue.iid}`, + url: issue.web_url, + }, + })) + ); + } + + if (this.withWikis) { + console.log(`[Gitlab Loader]: Fetching wiki.`); + const wiki = await this.fetchWiki(); + console.log(`[Gitlab Loader]: Fetched ${wiki.length} wiki pages.`); + docs.push( + ...wiki.map((wiki) => ({ + wiki, + metadata: { + source: `wiki-${this.repo}-${wiki.slug}`, + url: `${this.repo}/-/wikis/${wiki.slug}`, + }, + })) + ); + } + + return docs; + } + + #branchPrefSort(branches = []) { + const preferredSort = ["main", "master"]; + return branches.reduce((acc, branch) => { + if (preferredSort.includes(branch)) return [branch, ...acc]; + return [...acc, branch]; + }, []); + } + + /** + * Retrieves all branches for the repository. + * @returns {Promise} An array of branch names. + */ + async getRepoBranches() { + if (!this.#validGitlabUrl() || !this.projectId) return []; + await this.#validateAccessToken(); + this.branches = []; + + const branchesRequestData = { + endpoint: `/api/v4/projects/${this.projectId}/repository/branches`, + }; + + let branchesPage = []; + while ((branchesPage = await this.fetchNextPage(branchesRequestData))) { + if (!Array.isArray(branchesPage) || !branchesPage?.length) break; + this.branches.push(...branchesPage.map((branch) => branch.name)); + } + return this.#branchPrefSort(this.branches); + } + + /** + * Returns list of all file objects from tree API for GitLab + * @returns {Promise} + */ + async fetchFilesRecursive() { + const files = []; + const filesRequestData = { + endpoint: `/api/v4/projects/${this.projectId}/repository/tree`, + queryParams: { + ref: this.branch, + recursive: true, + }, + }; + + let filesPage = null; + let pagePromises = []; + while ((filesPage = await this.fetchNextPage(filesRequestData))) { + if (!Array.isArray(filesPage) || !filesPage?.length) break; + // Fetch all the files that are not ignored in parallel. + pagePromises = filesPage + .filter((file) => { + if (file.type !== "blob") return false; + return !this.ignoreFilter.ignores(file.path); + }) + .map(async (file) => { + const content = await this.fetchSingleFileContents(file.path); + if (!content) return null; + return { + path: file.path, + content, + }; + }); + + const pageFiles = await Promise.all(pagePromises); + + files.push(...pageFiles.filter((item) => item !== null)); + console.log(`Fetched ${files.length} files.`); + } + console.log(`Total files fetched: ${files.length}`); + return files; + } + + /** + * Fetches all issues from the repository. + * @returns {Promise} An array of issue objects. + */ + async fetchIssues() { + const issues = []; + const issuesRequestData = { + endpoint: `/api/v4/projects/${this.projectId}/issues`, + }; + + let issuesPage = null; + let pagePromises = []; + while ((issuesPage = await this.fetchNextPage(issuesRequestData))) { + if (!Array.isArray(issuesPage) || !issuesPage?.length) break; + // Fetch all the issues in parallel. + pagePromises = issuesPage.map(async (issue) => { + const discussionsRequestData = { + endpoint: `/api/v4/projects/${this.projectId}/issues/${issue.iid}/discussions`, + }; + let discussionPage = null; + const discussions = []; + + while ( + (discussionPage = await this.fetchNextPage(discussionsRequestData)) + ) { + if (!Array.isArray(discussionPage) || !discussionPage?.length) break; + discussions.push( + ...discussionPage.map(({ notes }) => + notes.map( + ({ body, author, created_at }) => + `${author.username} at ${created_at}: +${body}` + ) + ) + ); + } + const result = { + ...issue, + discussions, + }; + return result; + }); + + const pageIssues = await Promise.all(pagePromises); + + issues.push(...pageIssues); + console.log(`Fetched ${issues.length} issues.`); + } + console.log(`Total issues fetched: ${issues.length}`); + return issues; + } + + /** + * Fetches all wiki pages from the repository. + * @returns {Promise} An array of wiki page objects. + */ + async fetchWiki() { + const wikiRequestData = { + endpoint: `/api/v4/projects/${this.projectId}/wikis`, + queryParams: { + with_content: "1", + }, + }; + + const wikiPages = await this.fetchNextPage(wikiRequestData); + if (!Array.isArray(wikiPages)) return []; + console.log(`Total wiki pages fetched: ${wikiPages.length}`); + return wikiPages; + } + + /** + * Fetches the content of a single file from the repository. + * @param {string} sourceFilePath - The path to the file in the repository. + * @returns {Promise} The content of the file, or null if fetching fails. + */ + async fetchSingleFileContents(sourceFilePath, retries = 0) { + try { + const url = `${this.apiBase}/api/v4/projects/${ + this.projectId + }/repository/files/${encodeURIComponent(sourceFilePath)}/raw?ref=${ + this.branch + }`; + const response = await fetch(url, { + method: "GET", + headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}, + }); + + if (response.status === 429) { + if (retries >= MAX_RETRIES) { + console.warn( + `[Gitlab Loader]: Rate limit persists for ${sourceFilePath} after ${retries} retries. Skipping.` + ); + return null; + } + const retryAfter = Number(response.headers.get("retry-after")) || 60; + console.warn( + `[Gitlab Loader]: Rate limit hit fetching ${sourceFilePath}. Waiting ${retryAfter}s...` + ); + await this.#wait(retryAfter * 1000); + return this.fetchSingleFileContents(sourceFilePath, retries + 1); + } + + if (!response.ok) + throw new Error(`Failed to fetch single file ${sourceFilePath}`); + + return await response.text(); + } catch (e) { + console.error(`RepoLoader.fetchSingleFileContents`, e); + return null; + } + } + + /** + * Fetches the next page of data from the API. + * @param {Object} requestData - The request data. + * @returns {Promise|null>} The next page of data, or null if no more pages. + */ + async fetchNextPage(requestData, retries = 0) { + try { + if (requestData.page === -1) return null; + if (!requestData.page) requestData.page = 1; + + const { endpoint, perPage = 100, queryParams = {} } = requestData; + const params = new URLSearchParams({ + ...queryParams, + per_page: perPage, + page: requestData.page, + }); + const url = `${this.apiBase}${endpoint}?${params.toString()}`; + + const response = await fetch(url, { + method: "GET", + headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}, + }); + + if (response.status === 429) { + if (retries >= MAX_RETRIES) { + console.warn( + `[Gitlab Loader]: Rate limit persists for ${endpoint} after ${retries} retries. Skipping.` + ); + return null; + } + const retryAfter = Number(response.headers.get("retry-after")) || 60; + console.warn( + `[Gitlab Loader]: Rate limit hit for ${endpoint}. Waiting ${retryAfter}s before retrying...` + ); + await this.#wait(retryAfter * 1000); + return this.fetchNextPage(requestData, retries + 1); + } + + if (response.status === 401) { + console.warn( + `[Gitlab Loader]: Unauthorized request for ${endpoint}. Skipping.` + ); + return null; + } + + if (!response.ok) { + console.warn( + `[Gitlab Loader]: Unexpected status ${response.status} for ${endpoint}. Skipping.` + ); + return null; + } + + const data = await response.json(); + if (!Array.isArray(data)) { + console.warn(`Unexpected response format for ${endpoint}:`, data); + return []; + } + + // GitLab omits x-total-pages for large repos, so use x-next-page + // as the sole pagination signal — it's empty on the last page. + const nextPage = response.headers.get("x-next-page"); + const totalPages = response.headers.get("x-total-pages"); + console.log( + `Gitlab RepoLoader: fetched ${endpoint} page ${requestData.page}${ + totalPages ? `/${totalPages}` : "" + } with ${data.length} records.` + ); + + requestData.page = nextPage?.trim() ? Number(nextPage) : -1; + + return data; + } catch (e) { + console.error(`RepoLoader.fetchNextPage`, e); + return null; + } + } +} + +module.exports = GitLabRepoLoader; diff --git a/collector/utils/extensions/RepoLoader/GitlabRepo/index.js b/collector/utils/extensions/RepoLoader/GitlabRepo/index.js new file mode 100644 index 0000000..5c312f0 --- /dev/null +++ b/collector/utils/extensions/RepoLoader/GitlabRepo/index.js @@ -0,0 +1,261 @@ +const RepoLoader = require("./RepoLoader"); +const fs = require("fs"); +const path = require("path"); +const { default: slugify } = require("slugify"); +const { v4 } = require("uuid"); +const { sanitizeFileName, writeToServerDocuments } = require("../../../files"); +const { tokenizeString } = require("../../../tokenizer"); + +/** + * Load in a Gitlab Repo recursively or just the top level if no PAT is provided + * @param {object} args - forwarded request body params + * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker + * @returns + */ +async function loadGitlabRepo(args, response) { + const repo = new RepoLoader(args); + await repo.init(); + + if (!repo.ready) + return { + success: false, + reason: "Could not prepare Gitlab repo for loading! Check URL", + }; + + console.log( + `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} --` + ); + const docs = await repo.recursiveLoader(); + if (!docs.length) { + return { + success: false, + reason: "No files were found for those settings.", + }; + } + + console.log(`[GitLab Loader]: Found ${docs.length} source files. Saving...`); + const outFolder = slugify( + `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}` + ).toLowerCase(); + + const outFolderPath = + process.env.NODE_ENV === "development" + ? path.resolve( + __dirname, + `../../../../../server/storage/documents/${outFolder}` + ) + : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`); + + if (!fs.existsSync(outFolderPath)) + fs.mkdirSync(outFolderPath, { recursive: true }); + + for (const doc of docs) { + if (!doc.metadata || (!doc.pageContent && !doc.issue && !doc.wiki)) + continue; + let pageContent = null; + + const data = { + id: v4(), + url: "gitlab://" + doc.metadata.source, + docSource: doc.metadata.source, + chunkSource: generateChunkSource( + repo, + doc, + response.locals.encryptionWorker + ), + published: new Date().toLocaleString(), + }; + + if (doc.pageContent) { + pageContent = doc.pageContent; + + data.title = doc.metadata.source; + data.docAuthor = repo.author; + data.description = "No description found."; + } else if (doc.issue) { + pageContent = issueToMarkdown(doc.issue); + + data.title = `Issue ${doc.issue.iid}: ${doc.issue.title}`; + data.docAuthor = doc.issue.author.username; + data.description = doc.issue.description; + } else if (doc.wiki) { + pageContent = doc.wiki.content; + data.title = doc.wiki.title; + data.docAuthor = repo.author; + data.description = + doc.wiki.format === "markdown" + ? "GitLab Wiki Page (Markdown)" + : "GitLab Wiki Page"; + } else { + continue; + } + + data.wordCount = pageContent.split(" ").length; + data.token_count_estimate = tokenizeString(pageContent); + data.pageContent = pageContent; + + console.log( + `[GitLab Loader]: Saving ${doc.metadata.source} to ${outFolder}` + ); + + writeToServerDocuments({ + data, + filename: sanitizeFileName(`${slugify(doc.metadata.source)}-${data.id}`), + destinationOverride: outFolderPath, + }); + } + + return { + success: true, + reason: null, + data: { + author: repo.author, + repo: repo.project, + projectId: repo.projectId, + branch: repo.branch, + files: docs.length, + destination: outFolder, + }, + }; +} + +async function fetchGitlabFile({ + repoUrl, + branch, + accessToken = null, + sourceFilePath, +}) { + const repo = new RepoLoader({ + repo: repoUrl, + branch, + accessToken, + }); + await repo.init(); + + if (!repo.ready) + return { + success: false, + content: null, + reason: "Could not prepare GitLab repo for loading! Check URL or PAT.", + }; + console.log( + `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} file:${sourceFilePath} --` + ); + const fileContent = await repo.fetchSingleFile(sourceFilePath); + if (!fileContent) { + return { + success: false, + reason: "Target file returned a null content response.", + content: null, + }; + } + + return { + success: true, + reason: null, + content: fileContent, + }; +} + +function generateChunkSource(repo, doc, encryptionWorker) { + const payload = { + projectId: decodeURIComponent(repo.projectId), + branch: repo.branch, + path: doc.metadata.source, + pat: !!repo.accessToken ? repo.accessToken : null, + }; + return `gitlab://${repo.repo}?payload=${encryptionWorker.encrypt( + JSON.stringify(payload) + )}`; +} + +function issueToMarkdown(issue) { + const metadata = {}; + + const userFields = ["author", "assignees", "closed_by"]; + const userToUsername = ({ username }) => username; + for (const userField of userFields) { + if (issue[userField]) { + if (Array.isArray(issue[userField])) { + metadata[userField] = issue[userField].map(userToUsername); + } else { + metadata[userField] = userToUsername(issue[userField]); + } + } + } + + const singleValueFields = [ + "web_url", + "state", + "created_at", + "updated_at", + "closed_at", + "due_date", + "type", + "merge_request_count", + "upvotes", + "downvotes", + "labels", + "has_tasks", + "task_status", + "confidential", + "severity", + ]; + + for (const singleValueField of singleValueFields) { + metadata[singleValueField] = issue[singleValueField]; + } + + if (issue.milestone) { + metadata.milestone = `${issue.milestone.title} (${issue.milestone.id})`; + } + + if (issue.time_stats) { + const timeFields = ["time_estimate", "total_time_spent"]; + for (const timeField of timeFields) { + const fieldName = `human_${timeField}`; + if (issue?.time_stats[fieldName]) { + metadata[timeField] = issue.time_stats[fieldName]; + } + } + } + + const metadataString = Object.entries(metadata) + .map(([name, value]) => { + if (!value || value?.length < 1) { + return null; + } + let result = `- ${name.replace("_", " ")}:`; + + if (!Array.isArray(value)) { + result += ` ${value}`; + } else { + result += "\n" + value.map((s) => ` - ${s}`).join("\n"); + } + + return result; + }) + .filter((item) => item != null) + .join("\n"); + + let markdown = `# ${issue.title} (${issue.iid}) + +${issue.description} + +## Metadata + +${metadataString}`; + + if (issue.discussions.length > 0) { + markdown += ` + +## Activity + +${issue.discussions.join("\n\n")} +`; + } + + return markdown; +} + +module.exports = { loadGitlabRepo, fetchGitlabFile }; diff --git a/collector/utils/extensions/RepoLoader/index.js b/collector/utils/extensions/RepoLoader/index.js new file mode 100644 index 0000000..28e8780 --- /dev/null +++ b/collector/utils/extensions/RepoLoader/index.js @@ -0,0 +1,41 @@ +/** + * Dynamically load the correct repository loader from a specific platform + * by default will return GitHub. + * @param {('github'|'gitlab')} platform + * @returns {import("./GithubRepo/RepoLoader")|import("./GitlabRepo/RepoLoader")} the repo loader class for provider + */ +function resolveRepoLoader(platform = "github") { + switch (platform) { + case "github": + console.log(`Loading GitHub RepoLoader...`); + return require("./GithubRepo/RepoLoader"); + case "gitlab": + console.log(`Loading GitLab RepoLoader...`); + return require("./GitlabRepo/RepoLoader"); + default: + console.log(`Loading GitHub RepoLoader...`); + return require("./GithubRepo/RepoLoader"); + } +} + +/** + * Dynamically load the correct repository loader function from a specific platform + * by default will return Github. + * @param {('github'|'gitlab')} platform + * @returns {import("./GithubRepo")['fetchGithubFile'] | import("./GitlabRepo")['fetchGitlabFile']} the repo loader class for provider + */ +function resolveRepoLoaderFunction(platform = "github") { + switch (platform) { + case "github": + console.log(`Loading GitHub loader function...`); + return require("./GithubRepo").loadGithubRepo; + case "gitlab": + console.log(`Loading GitLab loader function...`); + return require("./GitlabRepo").loadGitlabRepo; + default: + console.log(`Loading GitHub loader function...`); + return require("./GithubRepo").loadGithubRepo; + } +} + +module.exports = { resolveRepoLoader, resolveRepoLoaderFunction }; diff --git a/collector/utils/extensions/WebsiteDepth/index.js b/collector/utils/extensions/WebsiteDepth/index.js new file mode 100644 index 0000000..2ab2a5e --- /dev/null +++ b/collector/utils/extensions/WebsiteDepth/index.js @@ -0,0 +1,211 @@ +const { v4 } = require("uuid"); +const { + PuppeteerWebBaseLoader, +} = require("langchain/document_loaders/web/puppeteer"); +const { default: slugify } = require("slugify"); +const { parse } = require("node-html-parser"); +const { writeToServerDocuments, documentsFolder } = require("../../files"); +const { tokenizeString } = require("../../tokenizer"); +const path = require("path"); +const fs = require("fs"); +const RuntimeSettings = require("../../runtimeSettings"); + +async function discoverLinks(startUrl, maxDepth = 1, maxLinks = 20) { + const baseUrl = new URL(startUrl); + const discoveredLinks = new Set([startUrl]); + let queue = [[startUrl, 0]]; // [url, currentDepth] + const scrapedUrls = new Set(); + + for (let currentDepth = 0; currentDepth < maxDepth; currentDepth++) { + const levelSize = queue.length; + const nextQueue = []; + + for (let i = 0; i < levelSize && discoveredLinks.size < maxLinks; i++) { + const [currentUrl, urlDepth] = queue[i]; + + if (!scrapedUrls.has(currentUrl)) { + scrapedUrls.add(currentUrl); + const newLinks = await getPageLinks(currentUrl, baseUrl); + + for (const link of newLinks) { + if (!discoveredLinks.has(link) && discoveredLinks.size < maxLinks) { + discoveredLinks.add(link); + if (urlDepth + 1 < maxDepth) { + nextQueue.push([link, urlDepth + 1]); + } + } + } + } + } + + queue = nextQueue; + if (queue.length === 0 || discoveredLinks.size >= maxLinks) break; + } + + return Array.from(discoveredLinks); +} + +async function getPageLinks(url, baseUrl) { + try { + const runtimeSettings = new RuntimeSettings(); + /** @type {import('puppeteer').PuppeteerLaunchOptions} */ + let launchConfig = { headless: "new" }; + + /* On MacOS 15.1, the headless=new option causes the browser to crash immediately. + * It is not clear why this is the case, but it is reproducible. Since AnythinglLM + * in production runs in a container, we can disable headless mode to workaround the issue for development purposes. + * + * This may show a popup window when scraping a page in development mode. + * This is expected behavior if seen in development mode on MacOS 15+ + */ + if ( + process.platform === "darwin" && + process.env.NODE_ENV === "development" + ) { + console.log( + "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing." + ); + launchConfig.headless = "false"; + } + + const loader = new PuppeteerWebBaseLoader(url, { + launchOptions: { + headless: launchConfig.headless, + ignoreHTTPSErrors: true, + args: runtimeSettings.get("browserLaunchArgs"), + }, + gotoOptions: { waitUntil: "networkidle2" }, + }); + const docs = await loader.load(); + const html = docs[0].pageContent; + const links = extractLinks(html, baseUrl); + return links; + } catch (error) { + console.error(`Failed to get page links from ${url}.`, error); + return []; + } +} + +function extractLinks(html, baseUrl) { + const root = parse(html); + const links = root.querySelectorAll("a"); + const extractedLinks = new Set(); + + for (const link of links) { + const href = link.getAttribute("href"); + if (href) { + const absoluteUrl = new URL(href, baseUrl.href).href; + if ( + absoluteUrl.startsWith( + baseUrl.origin + baseUrl.pathname.split("/").slice(0, -1).join("/") + ) + ) { + extractedLinks.add(absoluteUrl); + } + } + } + + return Array.from(extractedLinks); +} + +async function bulkScrapePages(links, outFolderPath) { + const runtimeSettings = new RuntimeSettings(); + /** @type {import('puppeteer').PuppeteerLaunchOptions} */ + let launchConfig = { headless: "new" }; + + /* On MacOS 15.1, the headless=new option causes the browser to crash immediately. + * It is not clear why this is the case, but it is reproducible. Since AnythinglLM + * in production runs in a container, we can disable headless mode to workaround the issue for development purposes. + * + * This may show a popup window when scraping a page in development mode. + * This is expected behavior if seen in development mode on MacOS 15+ + */ + if (process.platform === "darwin" && process.env.NODE_ENV === "development") { + console.log( + "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing." + ); + launchConfig.headless = "false"; + } + + const scrapedData = []; + + for (let i = 0; i < links.length; i++) { + const link = links[i]; + console.log(`Scraping ${i + 1}/${links.length}: ${link}`); + + try { + const loader = new PuppeteerWebBaseLoader(link, { + launchOptions: { + headless: launchConfig.headless, + ignoreHTTPSErrors: true, + args: runtimeSettings.get("browserLaunchArgs"), + }, + gotoOptions: { waitUntil: "networkidle2" }, + async evaluate(page, browser) { + const result = await page.evaluate(() => document.body.innerText); + await browser.close(); + return result; + }, + }); + const docs = await loader.load(); + const content = docs[0].pageContent; + + if (!content.length) { + console.warn(`Empty content for ${link}. Skipping.`); + continue; + } + + const url = new URL(link); + const decodedPathname = decodeURIComponent(url.pathname); + const filename = `${url.hostname}${decodedPathname.replace(/\//g, "_")}`; + + const data = { + id: v4(), + url: "file://" + slugify(filename) + ".html", + title: slugify(filename) + ".html", + docAuthor: "no author found", + description: "No description found.", + docSource: "URL link uploaded by the user.", + chunkSource: `link://${link}`, + published: new Date().toLocaleString(), + wordCount: content.split(" ").length, + pageContent: content, + token_count_estimate: tokenizeString(content), + }; + + writeToServerDocuments({ + data, + filename: data.title, + destinationOverride: outFolderPath, + }); + scrapedData.push(data); + + console.log(`Successfully scraped ${link}.`); + } catch (error) { + console.error(`Failed to scrape ${link}.`, error); + } + } + + return scrapedData; +} + +async function websiteScraper(startUrl, depth = 1, maxLinks = 20) { + const websiteName = new URL(startUrl).hostname; + const outFolder = slugify( + `${slugify(websiteName)}-${v4().slice(0, 4)}` + ).toLowerCase(); + const outFolderPath = path.resolve(documentsFolder, outFolder); + console.log("Discovering links..."); + const linksToScrape = await discoverLinks(startUrl, depth, maxLinks); + console.log(`Found ${linksToScrape.length} links to scrape.`); + + if (!fs.existsSync(outFolderPath)) + fs.mkdirSync(outFolderPath, { recursive: true }); + console.log("Starting bulk scraping..."); + const scrapedData = await bulkScrapePages(linksToScrape, outFolderPath); + console.log(`Scraped ${scrapedData.length} pages.`); + + return scrapedData; +} + +module.exports = websiteScraper; diff --git a/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/index.js b/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/index.js new file mode 100644 index 0000000..14a4639 --- /dev/null +++ b/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/index.js @@ -0,0 +1,99 @@ +const { validYoutubeVideoUrl } = require("../../../url"); + +/* + * This is just a custom implementation of the Langchain JS YouTubeLoader class + * as the dependency for YoutubeTranscript is quite fickle and its a rat race to keep it up + * and instead of waiting for patches we can just bring this simple script in-house and at least + * be able to patch it since its so flaky. When we have more connectors we can kill this because + * it will be a pain to maintain over time. + */ +class YoutubeLoader { + #videoId; + #language; + #addVideoInfo; + + constructor({ videoId = null, language = null, addVideoInfo = false } = {}) { + if (!videoId) throw new Error("Invalid video id!"); + this.#videoId = videoId; + this.#language = language; + this.#addVideoInfo = addVideoInfo; + } + + /** + * Extracts the videoId from a YouTube video URL. + * @param url The URL of the YouTube video. + * @returns The videoId of the YouTube video. + */ + static getVideoID(url) { + const videoId = validYoutubeVideoUrl(url, true); + if (videoId) return videoId; + throw new Error("Failed to get youtube video id from the url"); + } + + /** + * Creates a new instance of the YoutubeLoader class from a YouTube video + * URL. + * @param url The URL of the YouTube video. + * @param config Optional configuration options for the YoutubeLoader instance, excluding the videoId. + * @returns A new instance of the YoutubeLoader class. + */ + static createFromUrl(url, config = {}) { + const videoId = YoutubeLoader.getVideoID(url); + return new YoutubeLoader({ ...config, videoId }); + } + + /** + * Loads the transcript and video metadata from the specified YouTube + * video. It uses the youtube-transcript library to fetch the transcript + * and the youtubei.js library to fetch the video metadata. + * @returns Langchain like doc that is 1 element with PageContent and + */ + async load() { + let transcript; + const metadata = { + source: this.#videoId, + }; + try { + const fetchTranscript = await import("youtube-transcript-plus").then( + (module) => module.fetchTranscript + ); + const transcriptSegments = await fetchTranscript(this.#videoId, { + lang: this.#language, + }); + if (!transcriptSegments || transcriptSegments.length === 0) + throw new Error("Transcription not found"); + transcript = this.#convertTranscriptSegmentsToText(transcriptSegments); + if (this.#addVideoInfo) { + const { Innertube } = require("youtubei.js"); + const youtube = await Innertube.create(); + const info = (await youtube.getBasicInfo(this.#videoId)).basic_info; + metadata.description = info.short_description; + metadata.title = info.title; + metadata.view_count = info.view_count; + metadata.author = info.author; + } + } catch (e) { + throw new Error( + `Failed to get YouTube video transcription: ${e?.message}` + ); + } + return [ + { + pageContent: transcript, + metadata, + }, + ]; + } + + #convertTranscriptSegmentsToText(transcriptSegments) { + return transcriptSegments + .map((segment) => + typeof segment === "string" ? segment : segment.text || "" + ) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + } +} + +module.exports.YoutubeLoader = YoutubeLoader; diff --git a/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js b/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js new file mode 100644 index 0000000..5a7cb87 --- /dev/null +++ b/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js @@ -0,0 +1,246 @@ +const { validYoutubeVideoUrl } = require("../../../url"); + +class YoutubeTranscriptError extends Error { + constructor(message) { + super(`[YoutubeTranscript] ${message}`); + } +} + +/** + * Handles fetching and parsing YouTube video transcripts + */ +class YoutubeTranscript { + /** + * Encodes a string as a protobuf field + * @param {number} fieldNumber - The protobuf field number + * @param {string} str - The string to encode + * @returns {Buffer} Encoded protobuf field + */ + static #encodeProtobufString(fieldNumber, str) { + const utf8Bytes = Buffer.from(str, "utf8"); + const tag = (fieldNumber << 3) | 2; // wire type 2 for string + const lengthBytes = this.#encodeVarint(utf8Bytes.length); + + return Buffer.concat([ + Buffer.from([tag]), + Buffer.from(lengthBytes), + utf8Bytes, + ]); + } + + /** + * Encodes a number as a protobuf varint + * @param {number} value - The number to encode + * @returns {number[]} Encoded varint bytes + */ + static #encodeVarint(value) { + const bytes = []; + while (value >= 0x80) { + bytes.push((value & 0x7f) | 0x80); + value >>>= 7; + } + bytes.push(value); + return bytes; + } + + /** + * Creates a base64 encoded protobuf message + * @param {Object} param - The parameters to encode + * @param {string} param.param1 - First parameter + * @param {string} param.param2 - Second parameter + * @returns {string} Base64 encoded protobuf + */ + static #getBase64Protobuf({ param1, param2 }) { + const field1 = this.#encodeProtobufString(1, param1); + const field2 = this.#encodeProtobufString(2, param2); + return Buffer.concat([field1, field2]).toString("base64"); + } + + /** + * Extracts transcript text from YouTube API response + * @param {Object} responseData - The YouTube API response + * @returns {string} Combined transcript text + */ + static #extractTranscriptFromResponse(responseData) { + const transcriptRenderer = + responseData.actions?.[0]?.updateEngagementPanelAction?.content + ?.transcriptRenderer; + if (!transcriptRenderer) { + throw new Error("No transcript data found in response"); + } + + const segments = + transcriptRenderer.content?.transcriptSearchPanelRenderer?.body + ?.transcriptSegmentListRenderer?.initialSegments; + if (!segments) { + throw new Error("Transcript segments not found in response"); + } + + return segments + .map((segment) => { + const runs = segment.transcriptSegmentRenderer?.snippet?.runs; + return runs ? runs.map((run) => run.text).join("") : ""; + }) + .filter((text) => text) + .join(" ") + .trim() + .replace(/\s+/g, " "); + } + + /** + * Calculates a preference score for a caption track to determine the best match + * @param {Object} track - The caption track object from YouTube + * @param {string} track.languageCode - ISO language code (e.g., 'zh-HK', 'en', 'es') + * @param {string} track.kind - Track type ('asr' for auto-generated, "" for human-transcribed) + * @param {string[]} preferredLanguages - Array of language codes in preference order (e.g., ['zh-HK', 'en']) + * @returns {number} Preference score (lower is better) + */ + static #calculatePreferenceScore(track, preferredLanguages) { + // Language preference: index in preferredLanguages array (0 = most preferred) + const languagePreference = preferredLanguages.indexOf(track.languageCode); + const languageScore = languagePreference === -1 ? 9999 : languagePreference; + + // Kind bonus: prefer human-transcribed (undefined) over auto-generated ('asr') + const kindBonus = track.kind === "asr" ? 0.5 : 0; + + return languageScore + kindBonus; + } + + /** + * Finds the most suitable caption track based on preferred languages + * @param {string} videoBody - The raw HTML response from YouTube + * @param {string[]} preferredLanguages - Array of language codes in preference order + * @returns {Object|null} The selected caption track or null if none found + */ + static #findPreferredCaptionTrack(videoBody, preferredLanguages) { + const captionsConfigJson = videoBody.match( + /"captions":(.*?),"videoDetails":/s + ); + + const captionsConfig = captionsConfigJson?.[1] + ? JSON.parse(captionsConfigJson[1]) + : null; + + const captionTracks = captionsConfig + ? captionsConfig.playerCaptionsTracklistRenderer.captionTracks + : null; + + if (!captionTracks || captionTracks.length === 0) { + return null; + } + + const sortedTracks = [...captionTracks].sort((a, b) => { + const scoreA = this.#calculatePreferenceScore(a, preferredLanguages); + const scoreB = this.#calculatePreferenceScore(b, preferredLanguages); + return scoreA - scoreB; + }); + + return sortedTracks[0]; + } + + /** + * Fetches video page content and finds the preferred caption track + * @param {string} videoId - YouTube video ID + * @param {string[]} preferredLanguages - Array of preferred language codes + * @returns {Promise

Mount the storage locally and run AnythingLLM in Docker
+ Linux/MacOs +	+ +```shell +export STORAGE_LOCATION=$HOME/anythingllm && \ +mkdir -p $STORAGE_LOCATION && \ +touch "$STORAGE_LOCATION/.env" && \ +docker run -d --rm -p 3001:3001 \ +--cap-add SYS_ADMIN \ +-v ${STORAGE_LOCATION}:/app/server/storage \ +-v ${STORAGE_LOCATION}/.env:/app/server/.env \ +-e STORAGE_DIR="/app/server/storage" \ +mintplexlabs/anythingllm +``` + +
+ Windows +	+ +```powershell +# Run this in powershell terminal +$env:STORAGE_LOCATION="$HOME\Documents\anythingllm"; ` +If(!(Test-Path $env:STORAGE_LOCATION)) {New-Item $env:STORAGE_LOCATION -ItemType Directory}; ` +If(!(Test-Path "$env:STORAGE_LOCATION\.env")) {New-Item "$env:STORAGE_LOCATION\.env" -ItemType File}; ` +docker run -d --rm -p 3001:3001 ` +--cap-add SYS_ADMIN ` +-v "$env:STORAGE_LOCATION`:/app/server/storage" ` +-v "$env:STORAGE_LOCATION\.env:/app/server/.env" ` +-e STORAGE_DIR="/app/server/storage" ` +mintplexlabs/anythingllm; +``` + +
Docker Compose	+ + +```yaml +version: '3.8' +services: + anythingllm: + image: mintplexlabs/anythingllm + container_name: anythingllm + ports: + - "3001:3001" + cap_add: + - SYS_ADMIN + environment: + # Adjust for your environment + - STORAGE_DIR=/app/server/storage + - JWT_SECRET="make this a large list of random numbers and letters 20+" + - LLM_PROVIDER=ollama + - OLLAMA_BASE_PATH=http://127.0.0.1:11434 + - OLLAMA_MODEL_PREF=llama2 + - OLLAMA_MODEL_TOKEN_LIMIT=4096 + - EMBEDDING_ENGINE=ollama + - EMBEDDING_BASE_PATH=http://127.0.0.1:11434 + - EMBEDDING_MODEL_PREF=nomic-embed-text:latest + - EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192 + - VECTOR_DB=lancedb + - WHISPER_PROVIDER=local + - TTS_PROVIDER=native + - PASSWORDMINCHAR=8 + # Add any other keys here for services or settings + # you can find in the docker/.env.example file + volumes: + - anythingllm_storage:/app/server/storage + restart: always + +volumes: + anythingllm_storage: + driver: local + driver_opts: + type: none + o: bind + device: /path/on/local/disk +``` + +

PLEASE READ THIS

+ WARNING - This action is irreversible +

+ {t("community_hub.publish.agent_flow.success_title")} +

+ {t("community_hub.publish.agent_flow.modal_title")} +

+ Edit Preset +

Title

${title}