Propose PDFRouterParser and Loader

Author: pprados

libs/community/langchain_community/document_loaders/parsers/__init__.py

@@ -29,6 +29,7 @@
     from langchain_community.document_loaders.parsers.pdf import (
         PDFMinerParser,
         PDFPlumberParser,
+        PDFRouterParser,
         PyMuPDFParser,
         PyPDFium2Parser,
         PyPDFParser,
@@ -51,6 +52,7 @@
     "PDFPlumberParser": "langchain_community.document_loaders.parsers.pdf",
     "PyMuPDFParser": "langchain_community.document_loaders.parsers.pdf",
     "PyPDFParser": "langchain_community.document_loaders.parsers.pdf",
+    "PDFRouterParser": "langchain_community.document_loaders.parsers.pdf",
     "PyPDFium2Parser": "langchain_community.document_loaders.parsers.pdf",
     "RapidOCRBlobParser": "langchain_community.document_loaders.parsers.images",
     "TesseractBlobParser": "langchain_community.document_loaders.parsers.images",
@@ -76,6 +78,7 @@ def __getattr__(name: str) -> Any:
     "OpenAIWhisperParser",
     "PDFMinerParser",
     "PDFPlumberParser",
+    "PDFRouterParser",
     "PyMuPDFParser",
     "PyPDFParser",
     "PyPDFium2Parser",

libs/community/langchain_community/document_loaders/parsers/pdf.py

@@ -2,10 +2,10 @@
 
 from __future__ import annotations
 
-import re
 import html
 import io
 import logging
+import re
 import threading
 import warnings
 from datetime import datetime
@@ -1670,6 +1670,7 @@ def lazy_parse(self, blob: Blob) -> Iterator[Document]:  # type: ignore[valid-ty
 
             yield from docs
 
+
 class PDFRouterParser(BaseBlobParser):
     """
     Load PDFs using different parsers based on the metadata of the PDF
@@ -1702,7 +1703,7 @@ class PDFRouterParser(BaseBlobParser):
 
     def __init__(
         self,
-        routes: list[
+        routes: Sequence[
             tuple[
                 str,
                 dict[str, Union[re.Pattern, str]],
@@ -1736,7 +1737,8 @@ def lazy_parse(self, blob: Blob) -> Iterator[Document]:  # type: ignore[valid-ty
             import pypdf  # noqa:F401
         except ImportError:
             raise ImportError(
-                "pypdf package not found, please install it with `pip install pypdf.six`"
+                "pypdf package not found, please install it with "
+                "`pip install pypdf.six`"
             )
         from pypdf import PdfReader
 

libs/community/langchain_community/document_loaders/pdf.py

@@ -22,7 +22,6 @@
 from urllib.parse import urlparse
 
 import requests
-
 from langchain_core.document_loaders import BaseBlobParser
 from langchain_core.documents import Document
 from langchain_core.utils import get_from_dict_or_env
@@ -37,9 +36,10 @@
     DocumentIntelligenceParser,
     PDFMinerParser,
     PDFPlumberParser,
+    PDFRouterParser,
     PyMuPDFParser,
     PyPDFium2Parser,
-    PyPDFParser, PDFRouterParser,
+    PyPDFParser,
 )
 from langchain_community.document_loaders.unstructured import UnstructuredFileLoader
 
@@ -1426,6 +1426,7 @@ def lazy_load(self) -> Iterator[Document]:
 # Legacy: only for backwards compatibility. Use PyPDFLoader instead
 PagedPDFSplitter = PyPDFLoader
 
+
 class PDFRouterLoader(BasePDFLoader):
     """
     Load PDFs using different parsers based on the metadata of the PDF
@@ -1462,7 +1463,7 @@ def __init__(
         routes: list[
             tuple[
                 str,
-                dict[str, Union[re.Pattern | str]],
+                dict[str, Union[re.Pattern, str]],
                 BaseBlobParser,
             ]
         ],
@@ -1472,14 +1473,11 @@ def __init__(
         super().__init__(file_path)
         self.parser = PDFRouterParser(routes, password=password)
 
-
     def lazy_load(
         self,
     ) -> Iterator[Document]:
         if self.web_path:
-            blob = Blob.from_data(
-                open(self.file_path, "rb").read(), path=self.web_path
-            )  # type: ignore[attr-defined]
+            blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path)  # type: ignore[attr-defined]
         else:
             blob = Blob.from_path(self.file_path)  # type: ignore[attr-defined]
         yield from self.parser.lazy_parse(blob)

libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py

@@ -2,7 +2,7 @@
 
 import re
 from pathlib import Path
-from typing import TYPE_CHECKING, Iterator
+from typing import TYPE_CHECKING, Iterator, Literal
 
 import pytest
 
@@ -11,10 +11,12 @@
 from langchain_community.document_loaders.blob_loaders import Blob
 from langchain_community.document_loaders.parsers import (
     BaseImageBlobParser,
-    PDFPlumberParser, PyMuPDFParser, PyPDFium2Parser,
+    PDFMinerParser,
+    PDFPlumberParser,
+    PDFRouterParser,
+    PyMuPDFParser,
+    PyPDFium2Parser,
 )
-from langchain_community.document_loaders.parsers.pdf import PDFRouterParser, \
-    PDFMinerParser
 
 if TYPE_CHECKING:
     from PIL.Image import Image
@@ -315,8 +317,9 @@ def _analyze_image(self, img: Image) -> str:
     )
     _std_assert_with_parser(parser)
 
+
 def test_parser_router_parse() -> None:
-    mode = "single"
+    mode: Literal["single"] = "single"
     routes = [
         (
             "Microsoft",
@@ -332,9 +335,14 @@ def test_parser_router_parse() -> None:
         ),
         (
             "Xdvipdfmx",
-            {"producer": "xdvipdfmx.*", "page1": "Hello"},
-            PDFMinerParser(mode=mode),
+            {"producer": re.compile(r"xdvipdfmx.*"), "page1": "Hello"},
+            # PDFMinerParser(mode=mode),
+            PyPDFium2Parser(mode=mode),
+        ),
+        (
+            "default",
+            dict(),
+            PyPDFium2Parser(mode=mode),
         ),
-        ("default", {}, PyPDFium2Parser(mode=mode)),
     ]
     _assert_with_parser(PDFRouterParser(routes=routes), splits_by_page=False)

libs/community/tests/unit_tests/document_loaders/parsers/test_public_api.py

@@ -14,6 +14,7 @@ def test_parsers_public_api_correct() -> None:
         "OpenAIWhisperParser",
         "PyPDFParser",
         "PDFMinerParser",
+        "PDFRouterParser",
         "PyMuPDFParser",
         "PyPDFium2Parser",
         "PDFPlumberParser",