chore: import upstream snapshot with attribution

test/python/testpipeline/testdata/testfiletohtml.py

@@ -0,0 +1,24 @@
+"""
+FileToHTML module tests
+"""
+
+import os
+import unittest
+
+from unittest.mock import patch
+
+from txtai.pipeline.data.filetohtml import Tika
+
+
+class TestFileToHTML(unittest.TestCase):
+    """
+    FileToHTML tests.
+    """
+
+    @patch.dict(os.environ, {"TIKA_JAVA": "1112444abc"})
+    def testTika(self):
+        """
+        Test the Tika.available returns False when Java is not available
+        """
+
+        self.assertFalse(Tika.available())

test/python/testpipeline/testdata/testtabular.py

@@ -0,0 +1,123 @@
+"""
+Tabular module tests
+"""
+
+import unittest
+
+from txtai.pipeline import Tabular
+
+# pylint: disable=C0411
+from utils import Utils
+
+
+class TestTabular(unittest.TestCase):
+    """
+    Tabular tests.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """
+        Create single tabular instance
+        """
+
+        cls.tabular = Tabular("id", ["text"])
+
+    def testContent(self):
+        """
+        Test parsing additional content
+        """
+
+        tabular = Tabular("id", ["text"], True)
+
+        row = {"id": 0, "text": "This is a test", "flag": 1}
+
+        # When content is enabled, both (uid, text, tags) and (uid, data, tags) rows are generated
+        # given that data doesn't necessarily include the text to index
+        rows = tabular([row])
+        uid, data, _ = rows[1]
+
+        # Data should contain the entire input row
+        self.assertEqual(uid, 0)
+        self.assertEqual(data, row)
+
+        # Only select flag field
+        tabular.content = ["flag"]
+        rows = tabular([row])
+        uid, data, _ = rows[1]
+
+        # Data should only contain a single field, flag
+        self.assertEqual(uid, 0)
+        self.assertTrue(list(data.keys()) == ["flag"])
+        self.assertEqual(data["flag"], 1)
+
+    def testCSV(self):
+        """
+        Test parsing a CSV file
+        """
+
+        rows = self.tabular([Utils.PATH + "/tabular.csv"])
+        uid, text, _ = rows[0][0]
+
+        self.assertEqual(uid, 0)
+        self.assertEqual(text, "The first sentence")
+
+    def testDict(self):
+        """
+        Test parsing a dict
+        """
+
+        rows = self.tabular([{"id": 0, "text": "This is a test"}])
+        uid, text, _ = rows[0]
+
+        self.assertEqual(uid, 0)
+        self.assertEqual(text, "This is a test")
+
+    def testInvalid(self):
+        """
+        Test invalid file paths
+        """
+
+        with self.assertRaises(ValueError):
+            self.tabular([Utils.PATH + "/article.pdf"])
+
+        with self.assertRaises(ValueError):
+            self.tabular(["https://invalid.path"])
+
+    def testList(self):
+        """
+        Test parsing a list
+        """
+
+        rows = self.tabular([[{"id": 0, "text": "This is a test"}]])
+        uid, text, _ = rows[0][0]
+
+        self.assertEqual(uid, 0)
+        self.assertEqual(text, "This is a test")
+
+    def testMissingColumns(self):
+        """
+        Test rows with uneven or missing columns
+        """
+
+        tabular = Tabular("id", ["text"], True)
+
+        rows = tabular([{"id": 0, "text": "This is a test", "metadata": "meta"}, {"id": 1, "text": "This is a test"}])
+
+        # When content is enabled both (id, text, tag) and (id, data, tag) tuples are generated given that
+        # data doesn't necessarily include the text to index
+        _, data, _ = rows[3]
+
+        self.assertIsNone(data["metadata"])
+
+    def testNoColumns(self):
+        """
+        Test creating text without specifying columns
+        """
+
+        tabular = Tabular("id")
+        rows = tabular([{"id": 0, "text": "This is a test", "summary": "Describes text in more detail"}])
+        uid, text, _ = rows[0]
+
+        self.assertEqual(uid, 0)
+        self.assertEqual(text, "This is a test. Describes text in more detail")

test/python/testpipeline/testdata/testtextractor.py

@@ -0,0 +1,253 @@
+"""
+Textractor module tests
+"""
+
+import platform
+import unittest
+
+from txtai.pipeline import Textractor
+
+# pylint: disable=C0411
+from utils import Utils
+
+
+class TestTextractor(unittest.TestCase):
+    """
+    Textractor tests.
+    """
+
+    def testClean(self):
+        """
+        Test text cleaning method
+        """
+
+        # Default text cleaning
+        textractor = Textractor()
+        self.assertEqual(textractor(" a  b  c "), "a b c")
+
+        # Require text to be minlength
+        textractor = Textractor(minlength=10)
+        self.assertEqual(textractor(" a  b  c "), None)
+
+        # Disable text cleaning
+        textractor = Textractor(cleantext=False, minlength=10)
+        self.assertEqual(textractor(" a  b  c "), " a  b  c ")
+
+    def testChonkie(self):
+        """
+        Test a chonkie chunker
+        """
+
+        # Test chonkie chunking
+        textractor = Textractor(chunker="sentence", chunk_size=5, chunk_overlap=0)
+        self.assertEqual(textractor("This is a test. And another test."), ["This is a test.", "And another test."])
+
+        # Test bad chunker throws an exception
+        with self.assertRaises(AttributeError):
+            textractor = Textractor(chunker="badchunker")
+
+    def testDefault(self):
+        """
+        Test default text extraction
+        """
+
+        # Text input
+        textractor = Textractor(backend=None)
+        text = textractor(Utils.PATH + "/tabular.csv")
+        self.assertEqual(len(text), 125)
+
+        # Markdown input
+        textractor = Textractor(sections=True)
+        sections = textractor("# Heading 1\nText1\n\n# Heading 2\nText2\n")
+
+        # Check number of sections is as expected
+        self.assertEqual(len(sections), 2)
+
+    @unittest.skipIf(platform.system() == "Darwin", "Docling skipped on macOS to avoid MPS issues")
+    def testDocling(self):
+        """
+        Test docling backend
+        """
+
+        textractor = Textractor(backend="docling")
+
+        # Extract text and check for Markdown formatting
+        text = textractor(Utils.PATH + "/article.pdf")
+        self.assertTrue("## Introducing txtai" in text)
+
+    def testLines(self):
+        """
+        Test extraction to lines
+        """
+
+        textractor = Textractor(lines=True)
+
+        # Extract text as lines
+        lines = textractor(Utils.PATH + "/article.pdf")
+
+        # Check number of lines is as expected
+        self.assertEqual(len(lines), 35)
+
+    def testLiteParse(self):
+        """
+        Test liteparse backend
+        """
+
+        textractor = Textractor(backend="liteparse")
+
+        # Extract text and check for Markdown formatting
+        text = textractor(Utils.PATH + "/article.pdf")
+        self.assertTrue("# Introducing txtai" in text)
+
+    def testHTML(self):
+        """
+        Test HTML to Markdown
+        """
+
+        # Headings
+        self.assertMarkdown("<h1>This is a test</h1>", "# This is a test")
+        self.assertMarkdown("<h6>This is a test</h6>", "###### This is a test")
+
+        # Blockquotes
+        self.assertMarkdown("<blockquote>This is a test</blockquote>", "> This is a test")
+
+        # Lists
+        self.assertMarkdown("<ul><li>Test1</li><li>Test2</li></ul>", "- Test1\n- Test2")
+        self.assertMarkdown("<ol><li>Test1</li><li>Test2</li></ol>", "1. Test1\n2. Test2")
+
+        # Code
+        self.assertMarkdown("<code>This is a test</code>", "```\nThis is a test\n```")
+        self.assertMarkdown("<pre>This is a test</pre>", "```\nThis is a test\n```")
+
+        # Tables
+        self.assertMarkdown(
+            "<table><tr><th>Header1</th><th>Header2</th></tr><tr><td>Test1</td><td>Test2</td></tr></table>",
+            "|Header1|Header2|\n|---|---|\n|Test1|Test2|",
+        )
+
+        # Ignore list
+        self.assertMarkdown("<aside>This is a test</aside>", "")
+
+        # Text formatting
+        self.assertMarkdown("<p>This is a test</p>", "This is a test")
+        self.assertMarkdown("<p>This is a <b>test</b</p>", "This is a **test**")
+        self.assertMarkdown("<p>This is a <strong>test</strong></p>", "This is a **test**")
+        self.assertMarkdown("<p>This is a <i>test</i></p>", "This is a *test*")
+        self.assertMarkdown("<p>This is a <em>test</em></p>", "This is a *test*")
+        self.assertMarkdown("<p>This is a <a href='link'>test</a>", "This is a [test](link)")
+
+        # Collapse to outer tag
+        self.assertMarkdown("<p>This is a <strong><em>test</em></strong></p>", "This is a **test**")
+        self.assertMarkdown("<p>This is a <em><strong>test</strong></em></p>", "This is a *test*")
+
+    def testParagraphs(self):
+        """
+        Test extraction to paragraphs
+        """
+
+        textractor = Textractor(paragraphs=True)
+
+        # Extract text as paragraphs
+        paragraphs = textractor(Utils.PATH + "/article.pdf")
+
+        # Check number of paragraphs is as expected
+        self.assertEqual(len(paragraphs), 11)
+
+    def testSections(self):
+        """
+        Test extraction to sections
+        """
+
+        textractor = Textractor(sections=True)
+
+        # Extract as sections
+        sections = textractor(Utils.PATH + "/document.pdf")
+
+        # Check number of sections is as expected
+        self.assertEqual(len(sections), 3)
+
+    def testSentences(self):
+        """
+        Test extraction to sentences
+        """
+
+        textractor = Textractor(sentences=True)
+
+        # Extract text as sentences
+        sentences = textractor(Utils.PATH + "/article.pdf")
+
+        # Check number of sentences is as expected
+        self.assertEqual(len(sentences), 17)
+
+    def testSingle(self):
+        """
+        Test a single extraction with no tokenization of the results
+        """
+
+        textractor = Textractor()
+
+        # Extract text as a single block
+        text = textractor(Utils.PATH + "/article.pdf")
+
+        # Check length of text is as expected
+        self.assertEqual(len(text), 2471)
+
+    def testTable(self):
+        """
+        Test table extraction
+        """
+
+        textractor = Textractor()
+
+        # Extract text as a single block
+        for name in ["document.docx", "spreadsheet.xlsx"]:
+            text = textractor(f"{Utils.PATH}/{name}")
+
+            # Check for table header
+            self.assertTrue("|---|" in text)
+
+    def testTikaFlag(self):
+        """
+        Test legacy tika flag
+        """
+
+        textractor = Textractor(tika=True)
+        self.assertIsNotNone(textractor.html)
+
+        textractor = Textractor(tika=False)
+        self.assertIsNone(textractor.html)
+
+    def testTuples(self):
+        """
+        Test output tuples
+        """
+
+        # Default text cleaning
+        textractor = Textractor(tuples=True)
+
+        path, text = textractor(Utils.PATH + "/article.pdf")
+        self.assertEqual(path, Utils.PATH + "/article.pdf")
+        self.assertEqual(len(text), 2471)
+
+    def testURL(self):
+        """
+        Test parsing a remote URL
+        """
+
+        # Test parsing URLs for each backend
+        for backend in ["docling", "liteparse", "tika"]:
+            textractor = Textractor(backend=backend)
+            text = textractor("https://github.com/neuml/txtai")
+            self.assertTrue("txtai is an all-in-one AI framework" in text)
+
+    def assertMarkdown(self, html, expected):
+        """
+        Helper method to assert generated markdown is as expected.
+
+        Args:
+            html: input html snippet
+            expected: expected markdown text
+        """
+
+        textractor = Textractor()
+        self.assertEqual(textractor(f"<html><body>{html}</body></html>"), expected)

test/python/testpipeline/testdata/testtokenizer.py

@@ -0,0 +1,68 @@
+"""
+Tokenizer module tests
+"""
+
+import unittest
+
+from txtai.pipeline import Tokenizer
+
+
+class TestTokenizer(unittest.TestCase):
+    """
+    Tokenizer tests.
+    """
+
+    def testAlphanumTokenize(self):
+        """
+        Test alphanumeric tokenization
+        """
+
+        # Alphanumeric tokenization through backwards compatible static method
+        self.assertEqual(Tokenizer.tokenize("Y this is a test!"), ["test"])
+        self.assertEqual(Tokenizer.tokenize("abc123 ABC 123"), ["abc123", "abc"])
+
+    def testEmptyTokenize(self):
+        """
+        Test handling empty and None inputs
+        """
+
+        # Test that parser can handle empty or None strings
+        self.assertEqual(Tokenizer.tokenize(""), [])
+        self.assertEqual(Tokenizer.tokenize(None), None)
+
+    def testStandardTokenize(self):
+        """
+        Test standard tokenization
+        """
+
+        # Default standard tokenizer parameters
+        tokenizer = Tokenizer()
+
+        # Define token tests
+        tests = [
+            ("Y this is a test!", ["y", "this", "is", "a", "test"]),
+            ("abc123 ABC 123", ["abc123", "abc", "123"]),
+            ("Testing hy-phenated words", ["testing", "hy", "phenated", "words"]),
+            ("111-111-1111", ["111", "111", "1111"]),
+            ("Test.1234", ["test", "1234"]),
+        ]
+
+        # Run through tests
+        for test, result in tests:
+            # Unicode Text Segmentation per Unicode Annex #29
+            self.assertEqual(tokenizer(test), result)
+
+    def testNgramTokenize(self):
+        """
+        Test ngram tokenization
+        """
+
+        # Standard ngram tokenization
+        tokenizer = Tokenizer(lowercase=True, ngrams=3)
+        result = tokenizer("NGRAM TEST")
+        self.assertIn("ngr", result)
+
+        # Case sensitive ngram tokenization
+        tokenizer = Tokenizer(lowercase=False, ngrams=3)
+        result = tokenizer("NGRAM TEST")
+        self.assertIn("NGR", result)

test/python/testpipeline/testdata/testurlretrieve.py

@@ -0,0 +1,115 @@
+"""
+URLRetrieve module tests
+"""
+
+import contextlib
+import unittest
+
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from threading import Thread
+from urllib.request import build_opener
+
+from txtai.pipeline import URLRetrieve
+from txtai.pipeline.data.urlretrieve import SafeRedirectHandler
+
+
+class RequestHandler(BaseHTTPRequestHandler):
+    """
+    Test HTTP handler.
+    """
+
+    def do_GET(self):
+        """
+        GET request handler.
+        """
+
+        if self.path == "/valid":
+            redirect = "https://github.com/neuml/txtai"
+        elif self.path == "/invalid":
+            redirect = "http://127.0.0.1"
+        else:
+            redirect = None
+
+        if redirect:
+            self.send_response(301)
+            self.send_header("Location", redirect)
+            self.end_headers()
+        else:
+            response = "test".encode("utf-8")
+
+            self.send_response(200)
+            self.send_header("content-type", "text/plain")
+            self.send_header("content-length", len(response))
+            self.end_headers()
+
+            self.wfile.write(response)
+            self.wfile.flush()
+
+
+class TestURLRetrieve(unittest.TestCase):
+    """
+    URLRetrieve tests.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """
+        Create mock http server
+        """
+
+        cls.httpd = HTTPServer(("127.0.0.1", 8006), RequestHandler)
+
+        server = Thread(target=cls.httpd.serve_forever, daemon=True)
+        server.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        """
+        Shutdown mock http server.
+        """
+
+        cls.httpd.shutdown()
+
+    def testRedirect(self):
+        """
+        Test redirects
+        """
+
+        urlretrieve = URLRetrieve(safeopen=True)
+
+        # Test redirect handler
+        opener = build_opener(SafeRedirectHandler(urlretrieve))
+
+        # Test valid direct
+        with contextlib.closing(opener.open("http://127.0.0.1:8006/valid")) as connection:
+            self.assertTrue("txtai is an all-in-one AI framework" in str(connection.read()))
+
+        # Test invalid redirect
+        with self.assertRaises(IOError):
+            contextlib.closing(opener.open("http://127.0.0.1:8006/invalid"))
+
+    def testRetrieve(self):
+        """
+        Test retrieval
+        """
+
+        urlretrieve = URLRetrieve()
+        data = urlretrieve("http://127.0.0.1:8006/data")
+        self.assertEqual(data, b"test")
+
+    def testSafeopen(self):
+        """
+        Test safeopen checks
+        """
+
+        urlretrieve = URLRetrieve(safeopen=True)
+
+        # Verify that local ip addresses fail
+        with self.assertRaises(IOError):
+            urlretrieve("http://127.0.0.1")
+
+        with self.assertRaises(IOError):
+            urlretrieve("https://127.0.0.1")
+
+        with self.assertRaises(IOError):
+            urlretrieve("https://[::1]")