From c076d26e958060e423a872b18838a05231ab7ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cle=CC=81ment=20Doumouro?= Date: Thu, 19 Nov 2026 16:31:10 +0100 Subject: [PATCH 1/2] feature(extract-core,extract-python): added the `Pipeline.extract_pages_content` API --- extract-core/extract_core/__init__.py | 18 +- extract-core/extract_core/objects.py | 23 ++- extract-core/extract_core/pipeline.py | 9 + extract-core/uv.lock | 2 +- extract-python/extract_python/docling_.py | 27 ++- extract-python/extract_python/liteparse_.py | 177 +++++++++++--------- extract-python/extract_python/marker_.py | 44 +++-- extract-python/extract_python/mineru_.py | 26 ++- extract-python/extract_python/utils.py | 50 ++---- extract-python/tests/test_marker.py | 2 +- extract-python/tests/test_utils.py | 60 ++++--- extract-python/uv.lock | 4 +- 12 files changed, 260 insertions(+), 182 deletions(-) diff --git a/extract-core/extract_core/__init__.py b/extract-core/extract_core/__init__.py index 1b13a38..96b3d58 100644 --- a/extract-core/extract_core/__init__.py +++ b/extract-core/extract_core/__init__.py @@ -15,10 +15,12 @@ Device, Error, InputDoc, + InputPages, MarkdownDoc, OutputFormat, Pages, PipelineSize, + Range, Ranges, Result, Status, @@ -86,31 +88,33 @@ def pipeline_config_discriminator(v: Any) -> str: __all__ = [ - "LiteParsePipelineConfig", - "LiteParseConfig", "BaseModel", "BasePipelineConfig", + "BatchConcurrencySettings", "ConversionOutput", "Device", "DoclingPipelineConfig", + "DoclingSettings", "Error", "InputDoc", + "InputPages", + "LiteParseConfig", + "LiteParsePipelineConfig", "MarkdownDoc", "MarkerPipelineConfig", "MinerUBackend", "MinerUConfig", "MinerUPipelineConfig", "OutputFormat", - "Ranges", "Pages", "Pipeline", - "PipelineSize", "PipelineBySize", + "PipelineSize", "PipelineType", + "Range", + "Ranges", "Result", + "ResultBufferConfig", "Status", "SupportedExt", - "ResultBufferConfig", - "DoclingSettings", - "BatchConcurrencySettings", ] diff --git a/extract-core/extract_core/objects.py b/extract-core/extract_core/objects.py index 3de6cfa..e0e9aa9 100644 --- a/extract-core/extract_core/objects.py +++ b/extract-core/extract_core/objects.py @@ -21,6 +21,9 @@ logger = logging.getLogger(__name__) base_config = merge_configs(icij_config(), no_enum_values_config()) +Range = tuple[int, int] +Ranges = list[Range] + class BaseModel(_BaseModel): model_config = base_config @@ -241,6 +244,9 @@ def from_path(cls, path: str | Path, n_pages: int) -> Self: ext = SupportedExt(path.suffix.lower()) return cls(path=path, ext=ext, n_pages=n_pages) + def to_all_pages(self, doc_idx: int) -> "InputPages": + return InputPages(doc=self, doc_idx=doc_idx, page_range=None) + def to_docling(self): # noqa: ANN201 from docling_core.types.io import DocumentStream # noqa: PLC0415 @@ -251,9 +257,6 @@ def to_docling(self): # noqa: ANN201 return self.path -Ranges = list[tuple[int, int]] - - class Pages(BaseModel): total: int = 0 byte_ranges: Ranges = [] @@ -309,3 +312,17 @@ def to_response(self) -> ResponseResult: class ExtractionResponse(BaseModel): results: list[ResponseResult] + + +class InputPages(BaseModel): + # Contiguous chunk of pages from a doc + doc: InputDoc + doc_idx: int + page_range: Range | None = None # None page range -> full document + metadata: dict[str, Any] = Field(default_factory=dict) + + @property + def n_pages(self) -> int: + if self.page_range is None: + return self.doc.n_pages + return self.page_range[1] - self.page_range[0] diff --git a/extract-core/extract_core/pipeline.py b/extract-core/extract_core/pipeline.py index 8e305bc..22d89c1 100644 --- a/extract-core/extract_core/pipeline.py +++ b/extract-core/extract_core/pipeline.py @@ -4,6 +4,7 @@ from typing import Self from icij_common.registrable import RegistrableFromConfig +from objects import InputPages from extract_core import BasePipelineConfig @@ -20,6 +21,14 @@ async def extract_content( self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path ) -> AsyncIterable[Result]: ... + @abstractmethod + async def extract_pages_content( + self, + pages: Iterable[InputPages], + output_format: OutputFormat, + output_path: Path, + ) -> AsyncIterable[Result]: ... + @classmethod def _from_config(cls, config: C) -> Self: return cls(config) diff --git a/extract-core/uv.lock b/extract-core/uv.lock index c38e222..c974878 100644 --- a/extract-core/uv.lock +++ b/extract-core/uv.lock @@ -524,7 +524,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "docling-slim", specifier = "~=2.134" }, - { name = "icij-common", specifier = "~=0.8.2" }, + { name = "icij-common", specifier = ">=0.8.2,<0.10" }, { name = "pydantic", specifier = "~=2.13" }, { name = "pydantic-extra-types", extras = ["pycountry"], specifier = "~=2.11.1" }, { name = "typer", marker = "extra == 'cli'", specifier = "~=0.21.2" }, diff --git a/extract-python/extract_python/docling_.py b/extract-python/extract_python/docling_.py index 8aec0c4..a369c43 100644 --- a/extract-python/extract_python/docling_.py +++ b/extract-python/extract_python/docling_.py @@ -28,10 +28,12 @@ DoclingPipelineConfig, Error, InputDoc, + InputPages, MarkdownDoc, OutputFormat, Pipeline, PipelineType, + Range, Result, Status, ) @@ -41,7 +43,6 @@ from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP from .utils import ( - Range, ResultBuffer, batch_per_pages, chdir, @@ -77,24 +78,36 @@ async def extract_content( ) -> AsyncIterable[Result]: settings = self._config.settings logger.info("starting extraction with settings: %s", settings) - with self._scoped_settings, self._result_buffer as buffer: + with self._scoped_settings: max_page_batches = settings.perf.max_page_batches page_batch_size = settings.perf.page_batch_size - batches = batch_per_pages( + pages = batch_per_pages( docs, page_batch_size, max_page_batches=max_page_batches ) - for batch in batches: - page_range = list({pages.page_range for pages in batch}) + async for res in self.extract_pages_content( + pages, output_format, output_path + ): + yield res + + async def extract_pages_content( + self, + pages: Iterable[InputPages], + output_format: OutputFormat, + output_path: Path, + ) -> AsyncIterable[Result]: + with self._result_buffer as buffer: + for input_pages in pages: + page_range = list({pages.page_range for pages in input_pages}) if len(page_range) > 1: msg = "convert_all only accept 1 page range for all docs" raise ValueError(msg) page_range = page_range[0] page_range = _docling_range(page_range) - docling_docs = (pages.doc.to_docling() for pages in batch) + docling_docs = (pages.doc.to_docling() for pages in input_pages) outputs = self._converter.convert_all( docling_docs, raises_on_error=False, page_range=page_range ) - processed = iter(batch) + processed = iter(input_pages) sentinel = object() while True: res = await asyncio.to_thread(next, outputs, sentinel) diff --git a/extract-python/extract_python/liteparse_.py b/extract-python/extract_python/liteparse_.py index 0d3a2da..86f1676 100644 --- a/extract-python/extract_python/liteparse_.py +++ b/extract-python/extract_python/liteparse_.py @@ -17,20 +17,21 @@ ConversionOutput, Error, InputDoc, + InputPages, LiteParsePipelineConfig, OutputFormat, Pipeline, PipelineType, + Range, Result, Status, SupportedExt, ) +from icij_common.pydantic_utils import safe_copy from pydantic import TypeAdapter from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP from .utils import ( - ProcessedPages, - Range, ResultBuffer, UsageTracker, _run_with_concurrency, @@ -40,6 +41,8 @@ write_pages, ) +LITEPARSE_RANGE = "liteparse_range" + logger = logging.getLogger(__name__) if TYPE_CHECKING: import pypdfium2 as pdfium @@ -55,8 +58,8 @@ _LITEPARSE_CONVERSION_ERRORS = tuple() - -OrientedBatch = tuple[int | None, tuple[ProcessedPages]] +Orientation = int | None +OrientedBatch = tuple[tuple[Range, Orientation] | None, tuple[InputPages, ...]] LiteParseOutputFormat = Literal["text", "json", "markdown"] @@ -76,7 +79,7 @@ class LiteParsePageResult: dict[int, tuple["ParsedPage", list["ExtractedImage"], list["PageError"]]] | None ) = None - def filter(self, page: ProcessedPages) -> Self: + def filter(self, page: InputPages) -> Self: if self._by_ix is None: self._by_ix = self._results_by_index() pages, images, errors = [], [], [] @@ -119,20 +122,44 @@ async def extract_content( self._tessdata_path = Path(tessdata_path) if self._config.detect_page_orientation: logger.info("reading docs page orientation detection...") - batches = await self._batch_per_orientation(docs) + pages = await self._group_per_page_range_and_orientation(docs) else: - batches = chunk_docs(docs, self._config.page_batch_size) - batches = ((None, (c,)) for c in batches) + pages = chunk_docs(docs, self._config.page_batch_size) logger.info("parsing pages...") - async for res in self._parse(batches, output_format, output_path): + async for res in self.extract_pages_content(pages, output_format, output_path): yield res - async def _batch_per_orientation( + async def extract_pages_content( + self, + pages: Iterable[InputPages], + output_format: OutputFormat, + output_path: Path, + ) -> AsyncIterable[Result]: + match output_format: + case OutputFormat.MARKDOWN: + liteparse_o_fmt, postprocess_fn = "markdown", aggregate_pages_md + case _: + fmt = ( + f"unsupported output format for {LiteParsePipeline.__name__}" + f" {output_format}" + ) + raise NotImplementedError(fmt) + result_queue = asyncio.Queue() + # Buffer partial conversion to filesystem as large docs are chunked into pieces + with self._result_buffer() as buffer: + publisher = self._parse_publisher(pages, liteparse_o_fmt, result_queue) + consumer = self._parse_consumer( + result_queue, buffer, postprocess_fn, output_path + ) + async for res in publish_and_consume(publisher, consumer=consumer): + yield res + + async def _group_per_page_range_and_orientation( self, docs: Iterable[InputDoc] - ) -> list[OrientedBatch]: + ) -> list[InputPages]: logger.info("computing page complexities...") complexities = await self._compute_complexities(docs) - batches = [b async for b in self._batch_with_orientation(complexities)] + batches = [b async for b in self._groups_from_complexities(complexities)] return batches async def _compute_complexities( @@ -150,21 +177,26 @@ async def _compute_complexities( ] return batches - async def _batch_with_orientation( + async def _groups_from_complexities( self, complexities: list[tuple[InputDoc, list["PageComplexityStats"]]] - ) -> AsyncIterable[OrientedBatch]: + ) -> AsyncIterable[InputPages]: batch_size = self._config.page_batch_size - batches_per_orientation = defaultdict(list) - not_pdfs = [doc for doc, comps in complexities if comps is None] + groups = defaultdict(list) + not_pdfs = (doc for doc, comps in complexities if comps is None) for doc in not_pdfs: for batch in chunk_docs([doc], batch_size): - rng = _liteparse_range(batch.page_range) - batches_per_orientation[(rng, None)].append(batch) + rng, orientation = None, None + if batch.page_range is not None: + rng = _liteparse_range(batch.page_range) + update = {"liteparse_range": rng, "orientation": orientation} + update = {"metadata": update} + batch = tuple(safe_copy(b, update) for b in batch) + groups[(rng, orientation)].extend(batch) complexities = [ (doc, comps) for doc, comps in complexities if comps is not None ] logger.info("reading complex pages orientation...") - offset = len(not_pdfs) + offset = sum(len(v) for v in groups.values()) with self._executor() as executor: for doc, comps in complexities: pages_by_orientation = await _doc_pages_by_orientation( @@ -174,42 +206,18 @@ async def _batch_with_orientation( for batch in batched(pages, batch_size, strict=False): batch = _pages_from_complexities(doc, offset, *batch) # noqa: PLW2901 rng = _liteparse_range_from_pages(*batch) - batches_per_orientation[(rng, orientation)].append(batch) + update = {"liteparse_range": rng, "orientation": orientation} + update = {"metadata": update} + batch = tuple(safe_copy(b, update) for b in batch) + groups[(rng, orientation)].extend(batch) offset += 1 - # We yield pages per orientation only here to make sure to yield - # (orientation, range) in the most regular way and avoid reload - for (_, orientation), batches in batches_per_orientation.items(): - for batch in batches: - yield orientation, batch - - async def _parse( - self, - batches: Iterable[OrientedBatch], - output_format: OutputFormat, - output_path: Path, - ) -> AsyncIterable[Result]: - match output_format: - case OutputFormat.MARKDOWN: - liteparse_o_fmt, postprocess_fn = "markdown", aggregate_pages_md - case _: - fmt = ( - f"unsupported output format for {LiteParsePipeline.__name__}" - f" {output_format}" - ) - raise NotImplementedError(fmt) - result_queue = asyncio.Queue() - # Buffer partial conversion to filesystem as large docs are chunked into pieces - with self._result_buffer() as buffer: - publisher = self._parse_publisher(batches, liteparse_o_fmt, result_queue) - consumer = self._parse_consumer( - result_queue, buffer, postprocess_fn, output_path - ) - async for res in publish_and_consume(publisher, consumer=consumer): - yield res + for group in groups.values(): + for input_pages in sorted(group, key=lambda p: p.n_pages): + yield input_pages async def _parse_publisher( self, - batches: Iterable[OrientedBatch], + pages: Iterable[InputPages], output_format: LiteParseOutputFormat, queue: asyncio.Queue, ) -> None: @@ -225,24 +233,29 @@ async def _parse_publisher( publish_tasks = [] with _close_parsers(parsers): async with asyncio.TaskGroup() as tg: - for orientation, batch in batches: + for input_pages in pages: + page_range = input_pages.metadata.get(LITEPARSE_RANGE) + orientation = input_pages.metadata.get("orientation") # We avoid recreating the parser at each iteration since it creates - # a worker pool, instead process in (orientation, page range) order - # and cache the parser - page_range = _liteparse_range_from_pages(*batch) - n_pages = sum(p.n_pages for p in batch) - key = (page_range, orientation) - if key != current_key: + # a worker pool + parser_key = ( + (page_range, orientation, None) + if orientation is None + else (page_range, orientation, input_pages.n_pages) + ) + if parser_key != current_key: if parser is not None: close_t = tg.create_task(_await_and_close(tracker, parser)) close_parsers_tasks.append(close_t) - logger.debug("loading parser for %s", key) - parser = self._make_parser(output_format, key, n_pages=n_pages) + logger.debug("loading parser for %s", parser_key) + parser = self._make_parser(output_format, *parser_key) parsers.append(parser) parser.__enter__().warm_up() - current_key = key + current_key = parser_key await sem.acquire(), await tracker.acquire(parser) - task = self._convert_and_queue(parser, batch, queue, sem, tracker) + task = self._convert_and_queue( + parser, input_pages, queue, sem, tracker + ) publish_tasks.append(tg.create_task(task)) await asyncio.gather(*publish_tasks) queue.put_nowait(None) @@ -256,12 +269,11 @@ async def _parse_consumer( output_path: Path, ) -> AsyncIterable[Result]: while (queued := await queue.get()) is not None: - pages, res = queued - for p in pages: - filtered = res.filter(p) - buffer.add(p, filtered) - doc_ix = pages[0].doc_idx - doc = pages[0].doc + input_pages, res = queued + filtered = res.filter(input_pages) + buffer.add(input_pages, filtered) + doc_ix = input_pages.doc_idx + doc = input_pages.doc if buffer.is_complete(doc_ix): res = result_aggregator(doc, buffer.pop_complete(doc_ix), output_path) yield res @@ -269,12 +281,12 @@ async def _parse_consumer( async def _convert_and_queue( self, parser: "LiteParse", - pages: tuple[ProcessedPages], + pages: InputPages, queue: asyncio.Queue, sem: asyncio.Semaphore, tracker: UsageTracker, ) -> None: - res = await parser.aparse(pages[0].doc.path) + res = await parser.aparse(pages.doc.path) sem.release(), await tracker.release(parser) res = LiteParsePageResult( pages=res.pages, errors=res.page_errors, images=res.images @@ -284,8 +296,9 @@ async def _convert_and_queue( def _make_parser( self, output_format: LiteParseOutputFormat, - key: tuple[str, int | None], - n_pages: int, + target_pages: str, + orientation: int | None, + n_pages: int | None, ) -> "LiteParse": from liteparse.parser import LiteParse # noqa: PLC0415 @@ -293,7 +306,6 @@ def _make_parser( parse_timeout = self._config.per_page_timeout if parse_timeout is not None: parse_timeout = parse_timeout * page_batch_size - target_pages, orientation = key if orientation is not None: orientation = [(p + 1, orientation) for p in range(n_pages)] pool_size = max(1, self._config.n_cpus - 1) @@ -340,8 +352,8 @@ def _result_buffer(self) -> ResultBuffer: async def _compute_batch_complexities( - pages: ProcessedPages, -) -> tuple[ProcessedPages, list["PageComplexityStats"]]: + pages: InputPages, +) -> tuple[InputPages, list["PageComplexityStats"]]: from liteparse.parser import LiteParse # noqa: PLC0415 target_pages = _liteparse_range_from_pages(pages) @@ -428,15 +440,18 @@ def _liteparse_range(rng: Range) -> str: return f"{rng[0] + 1}-{rng[1]}" -def _liteparse_range_from_pages(*pages: ProcessedPages) -> str: - pages = sorted(pages, key=lambda p: p.page_range[0]) - ranges = (_liteparse_range(p.page_range) for p in pages) +def _liteparse_range_from_pages(*pages: InputPages) -> str: + pages = sorted(pages, key=lambda p: p.page_range or -1) + ranges = ( + p.page_range if p.page_range is not None else (0, p.doc.n_pages) for p in pages + ) + ranges = (_liteparse_range(r) for r in ranges) return ",".join(ranges) def _pages_from_complexities( doc: InputDoc, doc_ix: int, *complexities: "PageComplexityStats" -) -> tuple[ProcessedPages]: +) -> tuple[InputPages]: complexities = sorted(complexities, key=lambda p: p.page_number) pages = [] start, end = None, None @@ -445,11 +460,11 @@ def _pages_from_complexities( start = p.page_number - 1 end = start + 1 if p.page_number != end: - processed = ProcessedPages(doc=doc, doc_idx=doc_ix, page_range=(start, end)) + processed = InputPages(doc=doc, doc_idx=doc_ix, page_range=(start, end)) pages.append(processed) start = p.page_number - 1 end = p.page_number - processed = ProcessedPages(doc=doc, doc_idx=doc_ix, page_range=(start, end)) + processed = InputPages(doc=doc, doc_idx=doc_ix, page_range=(start, end)) pages.append(processed) return pages diff --git a/extract-python/extract_python/marker_.py b/extract-python/extract_python/marker_.py index 73a05df..2c97f1a 100644 --- a/extract-python/extract_python/marker_.py +++ b/extract-python/extract_python/marker_.py @@ -7,10 +7,12 @@ from extract_core import ( InputDoc, + InputPages, MarkdownDoc, OutputFormat, Pipeline, PipelineType, + Range, Result, Status, ) @@ -30,6 +32,16 @@ class MarkerPipeline(Pipeline): async def extract_content( self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path + ) -> AsyncIterable[Result]: + pages = (doc.to_all_pages(doc_ix) for doc_ix, doc in enumerate(docs)) + async for res in self.extract_pages_content(pages, output_format, output_path): + yield res + + async def extract_pages_content( + self, + pages: Iterable[InputPages], + output_format: OutputFormat, + output_path: Path, ) -> AsyncIterable[Result]: from marker.config.parser import ConfigParser # noqa: PLC0415 from marker.converters.pdf import PdfConverter # noqa: PLC0415 @@ -37,16 +49,28 @@ async def extract_content( config = deepcopy(self._config.config) config["output_format"] = output_format.to_marker() - config_parser = ConfigParser(config) - renderer = config_parser.get_renderer() - converter = PdfConverter( - config=config_parser.generate_config_dict(), - artifact_dict=create_model_dict(device=self._device), - processor_list=config_parser.get_processors(), - renderer=renderer, - ) - for doc in docs: - yield await _process_doc(doc, converter, output_format, output_path) + current_rng = object() + for input_pages in pages: + if input_pages.page_range != current_rng: + current_rng = input_pages.page_range + if current_rng is not None: + config["page_range"] = _to_marker_page_range(current_rng) + config_parser = ConfigParser(config) + renderer = config_parser.get_renderer() + converter = PdfConverter( + config=config_parser.generate_config_dict(), + artifact_dict=create_model_dict(device=self._device), + processor_list=config_parser.get_processors(), + renderer=renderer, + ) + res = await _process_doc( + input_pages.doc, converter, output_format, output_path + ) + yield res + + +def _to_marker_page_range(rng: Range) -> str: + return f"{rng[0]}-{rng[1]}" @report_recoverable_errors(_MARKER_CONVERSION_ERRORS) diff --git a/extract-python/extract_python/mineru_.py b/extract-python/extract_python/mineru_.py index dfcd34f..8079226 100644 --- a/extract-python/extract_python/mineru_.py +++ b/extract-python/extract_python/mineru_.py @@ -9,6 +9,7 @@ from extract_core import ( ConversionOutput, InputDoc, + InputPages, MinerUPipelineConfig, OutputFormat, Pipeline, @@ -16,6 +17,7 @@ Result, Status, ) +from extract_core.objects import Range from mineru import MiddleJson, MinerUParser, ParseResult from mineru.config import VlmConfig @@ -35,7 +37,16 @@ def __init__(self, config: MinerUPipelineConfig): async def extract_content( self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path ) -> AsyncIterable[Result]: + pages = (doc.to_all_pages(doc_ix) for doc_ix, doc in enumerate(docs)) + async for res in self.extract_pages_content(pages, output_format, output_path): + yield res + async def extract_pages_content( + self, + pages: Iterable[InputPages], + output_format: OutputFormat, + output_path: Path, + ) -> AsyncIterable[Result]: with reset_env(): os.environ["MINERU_DEVICE_MODE"] = self._device # TODO: exclude files which are not pdf and return an error @@ -50,13 +61,24 @@ async def extract_content( image_analysis=config.image_analysis, vlm_config=vlm_config, ) - for doc in docs: - res = await parser.parse_async(doc.path) + for input_pages in pages: + page_rng = "" + if input_pages.page_range is not None: + page_rng = _to_mineru_page_range(input_pages.page_range) + doc = input_pages.doc + path = doc.path + res = await parser.parse_async(path, page_range=page_rng) yield _process_doc( doc, res, output_format=output_format, output_path=output_path ) +def _to_mineru_page_range(rng: Range) -> str: + # page range is a Python range, MinerU indexes from 1 and upper bound + # is inclusive + return f"{rng[0] + 1}-{rng[1]}" + + def _process_doc( doc: InputDoc, result: ParseResult, diff --git a/extract-python/extract_python/utils.py b/extract-python/extract_python/utils.py index d03ae30..33938f9 100644 --- a/extract-python/extract_python/utils.py +++ b/extract-python/extract_python/utils.py @@ -17,16 +17,14 @@ ) from contextlib import asynccontextmanager, contextmanager from copy import copy -from dataclasses import dataclass from functools import wraps from itertools import tee from pathlib import Path, PurePath from tempfile import TemporaryDirectory from types import TracebackType from typing import Any, BinaryIO, Protocol, Self -from warnings import deprecated -from extract_core import Error, InputDoc, Pages, Result, Status +from extract_core import Error, InputDoc, InputPages, Pages, Result, Status from pympler import asizeof logger = logging.getLogger(__name__) @@ -104,37 +102,19 @@ def write_pages(pages: Iterable[str], page_sep: str, out: BinaryIO) -> Pages: return Pages.from_pages_bytes_sizes(pages_byte_sizes) -Range = tuple[int, int] - - -@dataclass(frozen=True) -class ProcessedPages: - doc: InputDoc - doc_idx: int - page_range: Range | None = None - - @property - def n_pages(self) -> int: - if self.page_range is None: - return self.doc.n_pages - return self.page_range[1] - self.page_range[0] - - @property - @deprecated("Use n_pages instead") - def page_length(self) -> int: - return self.n_pages - - def chunk_docs( - docs: Iterable[InputDoc], n_pages: int, *, offset: int = 0 -) -> Iterable[ProcessedPages]: + docs: Iterable[InputDoc], page_batch_size: int, *, offset: int = 0 +) -> Iterable[InputPages]: for doc_i, doc in enumerate(docs): doc_idx = offset + doc_i - for i in range(0, doc.n_pages, n_pages): + if doc.n_pages >= page_batch_size: + yield doc.to_all_pages(doc_idx) + continue + for i in range(0, doc.n_pages, page_batch_size): start = i - end = min(start + n_pages, doc.n_pages) + end = min(start + page_batch_size, doc.n_pages) rng = (start, end) - pages = ProcessedPages(doc=doc, doc_idx=doc_idx, page_range=rng) + pages = InputPages(doc=doc, doc_idx=doc_idx, page_range=rng) yield pages @@ -228,7 +208,7 @@ def _fs_buffer_path(self) -> Path: raise ValueError(msg) return self.__fs_buffer_path - def add(self, processed: ProcessedPages, result: R) -> bool: + def add(self, processed: InputPages, result: R) -> bool: size = asizeof.asizeof(result) if self._current_size + size > self._max_bytes: path = self._page_path(processed.doc_idx) @@ -284,7 +264,7 @@ def batch_per_pages( *, max_page_batches: int, chunk_size: int = 1000, -) -> Iterable[tuple[ProcessedPages]]: +) -> Iterable[tuple[InputPages]]: # convert_all only accept to process docs on the exact same page_range # # We collect by chunk to avoid collecting too many inputs, input docs are @@ -307,14 +287,14 @@ def batch_per_pages( def _bin_fill( docs: Iterable[InputDoc], max_pages: int, offset: int = 0 -) -> Generator[tuple[ProcessedPages], None, int]: +) -> Generator[tuple[InputPages], None, int]: bins = defaultdict(deque) doc_idx = offset for doc in docs: if doc.n_pages > max_pages: msg = f"expected docs to have <= {max_pages} pages" raise ValueError(msg) - pages = ProcessedPages(doc=doc, doc_idx=doc_idx) + pages = InputPages(doc=doc, doc_idx=doc_idx) doc_idx += 1 available_space = (s for s in sorted(bins.keys()) if doc.n_pages <= s) available_space = next(available_space, max_pages) @@ -334,7 +314,7 @@ def _bin_fill( def _by_page_ranges( docs: Iterable[InputDoc], max_pages: int, offset: int = 0 -) -> Generator[tuple[ProcessedPages], None, int]: +) -> Generator[tuple[InputPages], None, int]: by_range = defaultdict(list) doc_idx = offset for doc in docs: @@ -348,7 +328,7 @@ def _by_page_ranges( rng = (start, end) rng_size = end - start alone_in_batch = rng_size == max_pages - pages = ProcessedPages(doc=doc, doc_idx=doc_idx, page_range=rng) + pages = InputPages(doc=doc, doc_idx=doc_idx, page_range=rng) if alone_in_batch: yield (pages,) continue diff --git a/extract-python/tests/test_marker.py b/extract-python/tests/test_marker.py index 55a142f..f0b86cb 100644 --- a/extract-python/tests/test_marker.py +++ b/extract-python/tests/test_marker.py @@ -10,8 +10,8 @@ Pipeline, Status, ) -from extract_python import MarkerPipeline +from extract_python import MarkerPipeline from tests import TEST_DATA_DIR diff --git a/extract-python/tests/test_utils.py b/extract-python/tests/test_utils.py index b1ba1d0..37e2dbc 100644 --- a/extract-python/tests/test_utils.py +++ b/extract-python/tests/test_utils.py @@ -6,15 +6,11 @@ from unittest.mock import MagicMock import pytest -from extract_core import InputDoc, SupportedExt -from extract_python.utils import ( - ProcessedPages, - ResultBuffer, - batch_per_pages, - write_pages, -) +from extract_core import InputDoc, InputPages, SupportedExt from pympler import asizeof +from extract_python.utils import ResultBuffer, batch_per_pages, write_pages + def _read_page(doc: BytesIO, start: int, *, end: int) -> str: doc.seek(start) @@ -131,12 +127,12 @@ def test_result_buffer_in_memory() -> None: max_size_bytes=max_size_bytes, save_fn=save_fn, load_fn=load_fn ) doc_idx = 0 - first = ProcessedPages( + first = InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), doc_idx=doc_idx, page_range=(0, 1), ) - last = ProcessedPages( + last = InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), doc_idx=doc_idx, page_range=(1, 2), @@ -164,12 +160,12 @@ def test_result_buffer_offload_on_fs() -> None: max_size_bytes=max_size_bytes, save_fn=save_fn, load_fn=load_fn ) doc_idx = 0 - first = ProcessedPages( + first = InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), doc_idx=doc_idx, page_range=(0, 1), ) - last = ProcessedPages( + last = InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), doc_idx=doc_idx, page_range=(1, 2), @@ -205,7 +201,7 @@ def test_result_buffer_offload_on_fs_should_preserve_root(tmpdir: Path) -> None: def test_result_buffer_raise_for_inconsistent_state() -> None: # Given max_size_bytes = 0 - pages = ProcessedPages( + pages = InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), doc_idx=0, page_range=(0, 1), @@ -236,19 +232,19 @@ def test_batch_per_pages_should_yield_short_docs_first() -> None: # Then expected_batches = [ ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=6), doc_idx=0 ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=8), doc_idx=1, page_range=(0, 6), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=8), doc_idx=1, page_range=(6, 8), @@ -276,24 +272,24 @@ def test_batch_per_pages_should_bin_short_docs() -> None: # Then expected_batches = [ ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=4), doc_idx=1 ), - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=2), doc_idx=3 ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=5), doc_idx=2 ), - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=1), doc_idx=4, ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=3), doc_idx=0 ), ), @@ -320,80 +316,80 @@ def test_batch_per_pages_should_group_long_docs_by_page_ranges() -> None: # Then expected_batches = [ ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=12), doc_idx=0, page_range=(0, 8), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=11), doc_idx=1, page_range=(0, 8), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=12), doc_idx=2, page_range=(0, 8), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=12), doc_idx=0, page_range=(8, 12), ), - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=12), doc_idx=2, page_range=(8, 12), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=11), doc_idx=3, page_range=(0, 8), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=11), doc_idx=1, page_range=(8, 11), ), - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=11), doc_idx=3, page_range=(8, 11), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=9), doc_idx=4, page_range=(0, 8), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("5"), n_pages=10), doc_idx=5, page_range=(0, 8), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=9), doc_idx=4, page_range=(8, 9), ), ), ( - ProcessedPages( + InputPages( doc=InputDoc(ext=SupportedExt.PDF, path=Path("5"), n_pages=10), doc_idx=5, page_range=(8, 10), diff --git a/extract-python/uv.lock b/extract-python/uv.lock index 7bd284c..3d1e3d5 100644 --- a/extract-python/uv.lock +++ b/extract-python/uv.lock @@ -894,7 +894,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "docling-slim", specifier = "~=2.134" }, - { name = "icij-common", specifier = "~=0.8.2" }, + { name = "icij-common", specifier = ">=0.8.2,<0.10" }, { name = "pydantic", specifier = "~=2.13" }, { name = "pydantic-extra-types", extras = ["pycountry"], specifier = "~=2.11.1" }, { name = "typer", marker = "extra == 'cli'", specifier = "~=0.21.2" }, @@ -920,7 +920,6 @@ name = "extract-python" source = { editable = "." } dependencies = [ { name = "extract-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-14-extract-python-liteparse' and extra == 'extra-14-extract-python-mineru') or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, - { name = "icij-common", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-14-extract-python-liteparse' and extra == 'extra-14-extract-python-mineru') or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, { name = "pympler", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-14-extract-python-liteparse' and extra == 'extra-14-extract-python-mineru') or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, ] @@ -974,7 +973,6 @@ requires-dist = [ { name = "docling-slim", extras = ["feat-ocr-nemotron", "models-onnxruntime", "models-vlm-inline"], marker = "extra == 'docling-gpu'", specifier = "~=2.127" }, { name = "extract-core", editable = "../extract-core" }, { name = "html2image", marker = "extra == 'benches'", specifier = "~=2.0.7" }, - { name = "icij-common", specifier = "~=0.8.2" }, { name = "liteparse", marker = "extra == 'liteparse'", specifier = "~=2.15" }, { name = "markdown2", marker = "extra == 'benches'", specifier = ">=2.5.4" }, { name = "marker-pdf", marker = "extra == 'marker'", specifier = "~=1.10" }, From 53f53b9ce577a98d750e4578937b71030c8cd75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cle=CC=81ment=20Doumouro?= Date: Fri, 20 Nov 2026 16:26:25 +0100 Subject: [PATCH 2/2] feature(extract-python): implement the `HybridLiteParsePipeline` --- .github/workflows/tests-extract-core.yml | 18 +- .github/workflows/tests-extract-python.yml | 22 +- extract-core/extract_core/__init__.py | 40 +++- extract-core/extract_core/configs.py | 1 + extract-core/extract_core/liteparse_.py | 47 ++++- extract-core/extract_core/pipeline.py | 3 +- extract-core/pyproject.toml | 3 + extract-core/tests/cli/test_cli.py | 3 +- extract-core/tests/test_docling.py | 1 + extract-core/tests/test_objects.py | 3 +- extract-python/benches/compare.py | 3 +- extract-python/extract_python/__init__.py | 15 +- .../extract_python/hybrid_liteparse_.py | 152 ++++++++++++++ extract-python/extract_python/liteparse_.py | 8 +- extract-python/pyproject.toml | 4 + extract-python/tests/test_docling.py | 4 +- extract-python/tests/test_hybrid_liteparse.py | 197 ++++++++++++++++++ extract-python/tests/test_liteparse.py | 1 + extract-python/tests/test_mineru.py | 2 +- 19 files changed, 493 insertions(+), 34 deletions(-) create mode 100644 extract-python/extract_python/hybrid_liteparse_.py create mode 100644 extract-python/tests/test_hybrid_liteparse.py diff --git a/.github/workflows/tests-extract-core.yml b/.github/workflows/tests-extract-core.yml index e50413d..9199546 100644 --- a/.github/workflows/tests-extract-core.yml +++ b/.github/workflows/tests-extract-core.yml @@ -6,25 +6,29 @@ on: pull_request: paths: - 'extract-core/**' +env: + PYTHON_VERSION: 3.13 + ASTRAL_VERSION: 0.11.6 jobs: lint-core: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: astral-sh/ruff-action@v4.0.0 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - args: "--version" # skips test by displaying the version + version: ${{ env.ASTRAL_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + enable-cache: true + working-directory: extract-core - name: Check formatting - run: ruff format --config qa/ruff.toml --check extract-core + run: cd extract-core && uv run ruff format --check - name: Lint test - run: ruff check --config qa/ruff.toml extract-core + run: cd extract-core && uv run ruff check tests-core: runs-on: ubuntu-latest - env: - PYTHON_VERSION: 3.13 - ASTRAL_VERSION: 0.11.6 steps: - uses: actions/checkout@v6 - name: Setup Python project diff --git a/.github/workflows/tests-extract-python.yml b/.github/workflows/tests-extract-python.yml index 2ff0812..ad37adf 100644 --- a/.github/workflows/tests-extract-python.yml +++ b/.github/workflows/tests-extract-python.yml @@ -8,24 +8,29 @@ on: - 'extract-core/**' - 'extract-python/**' +env: + PYTHON_VERSION: 3.13 + ASTRAL_VERSION: 0.11.6 + jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: astral-sh/ruff-action@v4.0.0 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - args: "--version" # skips test by displaying the version + version: ${{ env.ASTRAL_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + enable-cache: true + working-directory: extract-core - name: Check formatting - run: ruff format --config qa/ruff.toml --check extract-python + run: cd extract-python && uv run ruff format --check - name: Lint test - run: ruff check --config qa/ruff.toml extract-python + run: cd extract-python && uv run ruff check tests-no-miner-u: runs-on: ubuntu-latest - env: - PYTHON_VERSION: 3.13 - ASTRAL_VERSION: 0.11.6 steps: - uses: actions/checkout@v6 - name: Setup Python project @@ -60,9 +65,6 @@ jobs: tests-miner-u: runs-on: ubuntu-latest - env: - PYTHON_VERSION: 3.13 - ASTRAL_VERSION: 0.11.6 steps: - uses: actions/checkout@v6 - name: Setup Python project diff --git a/extract-core/extract_core/__init__.py b/extract-core/extract_core/__init__.py index 96b3d58..a4ebdf4 100644 --- a/extract-core/extract_core/__init__.py +++ b/extract-core/extract_core/__init__.py @@ -1,7 +1,7 @@ -from typing import Annotated, Any +from typing import Annotated, Any, ClassVar from icij_common.pydantic_utils import make_enum_discriminator, tagged_union -from pydantic import Discriminator, Tag +from pydantic import Discriminator, Field, Tag from .configs import ( BasePipelineConfig, @@ -59,9 +59,15 @@ MinerUBackend, MinerUPipelineConfig, MinerUConfig = None, None, None try: - from .liteparse_ import LiteParseConfig, LiteParsePipelineConfig + from .liteparse_ import ( + LiteParseConfig, + LiteParseLayoutReason, + LiteParseOCRReason, + LiteParsePipelineConfig, + LiteParseRouterConfig, + ) except ModuleNotFoundError: - LiteParsePipelineConfig, LiteParseConfig = None, None + LiteParsePipelineConfig, LiteParseConfig, LiteParseRouterConfig = None, None, None _pipeline_type_discriminator = make_enum_discriminator("pipeline", PipelineType) @@ -78,6 +84,28 @@ def pipeline_config_discriminator(v: Any) -> str: return _pipeline_type_discriminator(v) +if LiteParsePipelineConfig is not None: + PipelineConfigNoHybrid = PipelineConfig = Annotated[ + tagged_union( + BasePipelineConfig.__subclasses__(), lambda t: t.pipeline.default.value + ) + | Annotated[PipelineBySize, Tag("by_size")], + Discriminator(pipeline_config_discriminator), + ] + + class HybridLiteParsePipelineConfig(BasePipelineConfig): + pipeline: ClassVar[PipelineType] = Field( + frozen=True, default=PipelineType.HYBRID_LITEPARSE + ) + router: LiteParseRouterConfig = Field(default_factory=LiteParseRouterConfig) + liteparse: LiteParsePipelineConfig = Field( + default_factory=LiteParsePipelineConfig + ) + advanced: PipelineConfigNoHybrid = Field(default_factory=DoclingPipelineConfig) + +else: + HybridLiteParsePipelineConfig = None + PipelineConfig = Annotated[ tagged_union( BasePipelineConfig.__subclasses__(), lambda t: t.pipeline.default.value @@ -99,7 +127,10 @@ def pipeline_config_discriminator(v: Any) -> str: "InputDoc", "InputPages", "LiteParseConfig", + "LiteParseLayoutReason", + "LiteParseOCRReason", "LiteParsePipelineConfig", + "LiteParseRouterConfig", "MarkdownDoc", "MarkerPipelineConfig", "MinerUBackend", @@ -108,6 +139,7 @@ def pipeline_config_discriminator(v: Any) -> str: "OutputFormat", "Pages", "Pipeline", + "PipelineConfig", "PipelineBySize", "PipelineSize", "PipelineType", diff --git a/extract-core/extract_core/configs.py b/extract-core/extract_core/configs.py index fd89b44..a2d972d 100644 --- a/extract-core/extract_core/configs.py +++ b/extract-core/extract_core/configs.py @@ -13,6 +13,7 @@ class PipelineType(StrEnum): DOCLING = "docling" LITEPARSE = "liteparse" + HYBRID_LITEPARSE = "hybrid-liteparse" MARKER = "marker" MINER_U = "mineru" diff --git a/extract-core/extract_core/liteparse_.py b/extract-core/extract_core/liteparse_.py index a3d7b72..853375c 100644 --- a/extract-core/extract_core/liteparse_.py +++ b/extract-core/extract_core/liteparse_.py @@ -1,10 +1,11 @@ import os from copy import copy from datetime import timedelta +from enum import StrEnum from pathlib import Path from typing import ClassVar, Self -from pydantic import Field +from pydantic import Field, model_validator from .configs import BasePipelineConfig, PipelineType, ResultBufferConfig from .objects import BaseModel, Device, SupportedExt @@ -82,3 +83,47 @@ def supported_exts(cls) -> set[SupportedExt]: def to_device(self) -> Self: return self + + +class LiteParseOCRReason(StrEnum): + SCANNED = "scanned" + NO_TEXT = "no-text" + SPARSE_TEXT = "sparse-text" + EMBEDDED_IMAGES = "embedded-images" + GARBLED = "garbled" + VECTOR_TEXT = "vector-text" + ANNOTATION_TEXT = "annotation-text" + + +class LiteParseLayoutReason(StrEnum): + MULTI_COLUMN = "multi-column" + TABLE_LIKELY = "table-likely" + DENSE_GRAPHICS = "dense-graphics" + + +def _all_ocr_reasons() -> set[LiteParseOCRReason]: + return set(LiteParseOCRReason) + + +class LiteParseRouterConfig(BaseModel): + needs_ocr_is_advanced: bool = True + complex_layout_is_advanced: bool = True + advanced_ocr_reasons: set[LiteParseOCRReason] | None = None + advanced_layout_reasons: set[LiteParseLayoutReason] | None = None + + @model_validator(mode="after") + def reasons_are_consistent_with_flags(self) -> Self: + if self.needs_ocr_is_advanced and self.advanced_ocr_reasons is not None: + msg = ( + "advanced_ocr_reasons can be supplied when needs_ocr_is_advanced" + "is set to True. When needs_ocr_is_advanced ocr reasons are ignored." + ) + raise ValueError(msg) + if self.complex_layout_is_advanced and self.advanced_layout_reasons is not None: + msg = ( + "complex_layout_is_advanced can be supplied when " + "advanced_layout_reasons is set to True." + " When advanced_layout_reasons ocr reasons are ignored." + ) + raise ValueError(msg) + return self diff --git a/extract-core/extract_core/pipeline.py b/extract-core/extract_core/pipeline.py index 22d89c1..7af3b8f 100644 --- a/extract-core/extract_core/pipeline.py +++ b/extract-core/extract_core/pipeline.py @@ -4,11 +4,10 @@ from typing import Self from icij_common.registrable import RegistrableFromConfig -from objects import InputPages from extract_core import BasePipelineConfig -from .objects import InputDoc, OutputFormat, Result +from .objects import InputDoc, InputPages, OutputFormat, Result class Pipeline[C: BasePipelineConfig](RegistrableFromConfig, ABC): diff --git a/extract-core/pyproject.toml b/extract-core/pyproject.toml index 37dc22a..51026c0 100644 --- a/extract-core/pyproject.toml +++ b/extract-core/pyproject.toml @@ -130,3 +130,6 @@ artifacts = [ fallback-version = "0.0.0" pattern-prefix = "extract-core-" pattern = "default-unprefixed" + +[tool.ruff] +extend = "../qa/ruff.toml" diff --git a/extract-core/tests/cli/test_cli.py b/extract-core/tests/cli/test_cli.py index 82178f0..1f93d54 100644 --- a/extract-core/tests/cli/test_cli.py +++ b/extract-core/tests/cli/test_cli.py @@ -1,7 +1,8 @@ import pytest -from extract_core.cli import cli_app from typer.testing import CliRunner +from extract_core.cli import cli_app + @pytest.fixture # noqa: F405 def typer_asyncio_patch() -> None: diff --git a/extract-core/tests/test_docling.py b/extract-core/tests/test_docling.py index 7febaae..4ebd202 100644 --- a/extract-core/tests/test_docling.py +++ b/extract-core/tests/test_docling.py @@ -1,6 +1,7 @@ from docling.datamodel.accelerator_options import AcceleratorDevice from docling.datamodel.backend_options import PdfBackendOptions from docling.datamodel.base_models import InputFormat + from extract_core import Device, DoclingFormatOption, DoclingPipelineConfig diff --git a/extract-core/tests/test_objects.py b/extract-core/tests/test_objects.py index dd09e5f..7094dcb 100644 --- a/extract-core/tests/test_objects.py +++ b/extract-core/tests/test_objects.py @@ -6,9 +6,10 @@ ThreadedPdfPipelineOptions, ) from docling.document_converter import PdfFormatOption +from pydantic import TypeAdapter + from extract_core import DoclingPipelineConfig, PipelineConfig from extract_core.objects import Status -from pydantic import TypeAdapter def test_supported_ext_should_include_all_docling_input_formats() -> None: diff --git a/extract-python/benches/compare.py b/extract-python/benches/compare.py index 5f45783..8d3b249 100644 --- a/extract-python/benches/compare.py +++ b/extract-python/benches/compare.py @@ -4,10 +4,11 @@ import markdown2 import pypdfium2 from extract_core import BaseModel, OutputFormat, Pages -from extract_python.utils import chdir from html2image import Html2Image from PIL import Image, ImageDraw +from extract_python.utils import chdir + _WHITE_BACKGROUND_CSS = "body {background: white;}" diff --git a/extract-python/extract_python/__init__.py b/extract-python/extract_python/__init__.py index d507c18..450fd0e 100644 --- a/extract-python/extract_python/__init__.py +++ b/extract-python/extract_python/__init__.py @@ -22,15 +22,28 @@ MinerUPipeline = None try: + from .hybrid_liteparse_ import ( + HybridLiteParsePipeline, + LiteParseLayoutReason, + LiteParseOCRReason, + ) from .liteparse_ import LiteParsePipeline except ModuleNotFoundError: logger.warning("liteparse is not available, might be optional...") + HybridLiteParsePipeline, LiteParseLayoutReason, LiteParseOCRReason = ( + None, + None, + None, + ) LiteParsePipeline = None __all__ = [ "DoclingPipeline", "DOCLING_DEFAULT_ARTIFACTS_PATH", + "HybridLiteParsePipeline", + "LiteParseLayoutReason", + "LiteParseOCRReason", + "LiteParsePipeline", "MarkerPipeline", "MinerUPipeline", - "LiteParsePipeline", ] diff --git a/extract-python/extract_python/hybrid_liteparse_.py b/extract-python/extract_python/hybrid_liteparse_.py new file mode 100644 index 0000000..5e4bf7d --- /dev/null +++ b/extract-python/extract_python/hybrid_liteparse_.py @@ -0,0 +1,152 @@ +import logging +from collections.abc import AsyncIterable, Iterable, Iterator +from itertools import chain, groupby +from pathlib import Path +from typing import TYPE_CHECKING, Self + +from extract_core import ( + HybridLiteParsePipelineConfig, + InputDoc, + InputPages, + LiteParseLayoutReason, + LiteParseOCRReason, + LiteParseRouterConfig, + OutputFormat, + Pipeline, + PipelineType, + Result, +) + +from .liteparse_ import LiteParsePipeline + +_IS_ADVANCED = "is_advanced" + +LITEPARSE_RANGE = "liteparse_range" + +logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from liteparse.types import PageComplexityStats + + +class LiteParseRouter: + def __init__(self, config: LiteParseRouterConfig): + self._config = config + + def is_advanced(self, complexity: "PageComplexityStats") -> bool: + config = self._config + if config.needs_ocr_is_advanced and complexity.needs_ocr: + return True + layout = complexity.layout + if config.complex_layout_is_advanced and layout.is_complex: + return True + if config.advanced_ocr_reasons and any( + LiteParseOCRReason(r) in config.advanced_ocr_reasons + for r in complexity.reasons + ): + return True + if config.advanced_layout_reasons and any( # noqa: SIM103 + LiteParseLayoutReason(r) in config.advanced_layout_reasons + for r in layout.reasons + ): + return True + return False + + +@Pipeline.register(PipelineType.HYBRID_LITEPARSE) +class HybridLiteParsePipeline(Pipeline): + def __init__( + self, router: LiteParseRouter, liteparse: LiteParsePipeline, advanced: Pipeline + ): + self._router = router + self._liteparse = liteparse + self._advanced = advanced + + @classmethod + def _from_config(cls, config: HybridLiteParsePipelineConfig) -> Self: + router = LiteParseRouter(config.router) + liteparse = LiteParsePipeline(config.liteparse) + advanced = Pipeline.from_config(config.advanced) + return cls(router, liteparse, advanced) + + async def extract_content( + self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path + ) -> AsyncIterable[Result]: + complexities = await self._liteparse.compute_complexities(docs) + simple, advanced = self._route(complexities) + async for res in self.extract_pages_content(simple, output_format, output_path): + yield res + async for res in self.extract_pages_content( + advanced, output_format, output_path + ): + yield res + + async def extract_pages_content( + self, + pages: Iterable[InputPages], + output_format: OutputFormat, + output_path: Path, + ) -> AsyncIterable[Result]: + pages = _ensure_same_parser(pages) + first = next(pages, None) + if first is None: + return + pages = chain((first,), pages) + is_advanced = first.metadata[_IS_ADVANCED] + pipeline = self._advanced if is_advanced else self._liteparse + async for res in pipeline.extract_pages_content( + pages, output_format, output_path + ): + yield res + + def _route( + self, complexities: list[tuple[InputDoc, list["PageComplexityStats"]]] + ) -> tuple[list[InputPages], list[InputPages]]: + simple, advanced = [], [] + for doc_ix, (doc, doc_complexities) in enumerate(complexities): + current_page = 0 + grouped = groupby(doc_complexities, key=self._router.is_advanced) + for is_advanced, group in grouped: + group = tuple(group) # noqa: PLW2901 + input_pages = _group_to_input_pages( + group, current_page, doc, doc_ix=doc_ix, is_advanced=is_advanced + ) + if is_advanced: + advanced.append(input_pages) + else: + simple.append(input_pages) + current_page += input_pages.n_pages + return simple, advanced + + +def _group_to_input_pages( + group: tuple["PageComplexityStats", ...], + rng_start: int, + doc: InputDoc, + *, + doc_ix: int, + is_advanced: bool, +) -> InputPages: + n_pages = len(group) + start = rng_start + end = start + n_pages + rng = (start, end) + expected_page_nums = list(range(start + 1, end + 1)) + page_nums = [p.page_number for p in group] + if page_nums != expected_page_nums: + msg = ( + f"inconsistent page numbers within group, expected {page_nums} found {rng}" + ) + raise ValueError(msg) + meta = {_IS_ADVANCED: is_advanced} + input_pages = InputPages(doc=doc, doc_idx=doc_ix, page_range=rng, metadata=meta) + return input_pages + + +def _ensure_same_parser(pages: Iterable[InputPages]) -> Iterator[InputPages]: + is_advanced = None + for p in pages: + if is_advanced is None: + is_advanced = p.metadata["is_advanced"] + elif p.metadata["is_advanced"] != is_advanced: + raise ValueError("expected all pages to have the same complexities") + yield p diff --git a/extract-python/extract_python/liteparse_.py b/extract-python/extract_python/liteparse_.py index 86f1676..8409914 100644 --- a/extract-python/extract_python/liteparse_.py +++ b/extract-python/extract_python/liteparse_.py @@ -158,11 +158,11 @@ async def _group_per_page_range_and_orientation( self, docs: Iterable[InputDoc] ) -> list[InputPages]: logger.info("computing page complexities...") - complexities = await self._compute_complexities(docs) + complexities = await self.compute_complexities(docs) batches = [b async for b in self._groups_from_complexities(complexities)] return batches - async def _compute_complexities( + async def compute_complexities( self, docs: Iterable[InputDoc] ) -> list[tuple[InputDoc, list["PageComplexityStats"]]]: docs = list(docs) @@ -190,7 +190,7 @@ async def _groups_from_complexities( rng = _liteparse_range(batch.page_range) update = {"liteparse_range": rng, "orientation": orientation} update = {"metadata": update} - batch = tuple(safe_copy(b, update) for b in batch) + batch = tuple(safe_copy(b, update) for b in batch) # noqa: PLW2901 groups[(rng, orientation)].extend(batch) complexities = [ (doc, comps) for doc, comps in complexities if comps is not None @@ -208,7 +208,7 @@ async def _groups_from_complexities( rng = _liteparse_range_from_pages(*batch) update = {"liteparse_range": rng, "orientation": orientation} update = {"metadata": update} - batch = tuple(safe_copy(b, update) for b in batch) + batch = tuple(safe_copy(b, update) for b in batch) # noqa: PLW2901 groups[(rng, orientation)].extend(batch) offset += 1 for group in groups.values(): diff --git a/extract-python/pyproject.toml b/extract-python/pyproject.toml index e696a34..9dd047a 100644 --- a/extract-python/pyproject.toml +++ b/extract-python/pyproject.toml @@ -134,7 +134,11 @@ packages = ["extract_python"] exclude = [ "tests", ] + [tool.uv-dynamic-versioning] fallback-version = "0.0.0" pattern-prefix = "extract-python-" pattern = "default-unprefixed" + +[tool.ruff] +extend = "../qa/ruff.toml" diff --git a/extract-python/tests/test_docling.py b/extract-python/tests/test_docling.py index a3653ee..6cdf8c3 100644 --- a/extract-python/tests/test_docling.py +++ b/extract-python/tests/test_docling.py @@ -20,9 +20,10 @@ ) from extract_core.default_config import default_config from extract_core.objects import Device, PipelineSize -from extract_python import DoclingPipeline from icij_common.pydantic_utils import safe_copy +from extract_python import DoclingPipeline + from . import TEST_DATA_DIR @@ -91,6 +92,7 @@ def test_should_serialize_and_deserialize_format_options() -> None: # Given from docling.document_converter import PdfFormatOption # noqa: PLC0415 from docling.pipeline.vlm_pipeline import VlmPipeline # noqa: PLC0415 + from extract_python.docling_ import SerializableFormatOptions # noqa: PLC0415 vlm_options = VlmConvertOptions.from_preset("granite_docling") diff --git a/extract-python/tests/test_hybrid_liteparse.py b/extract-python/tests/test_hybrid_liteparse.py new file mode 100644 index 0000000..fafdb7e --- /dev/null +++ b/extract-python/tests/test_hybrid_liteparse.py @@ -0,0 +1,197 @@ +from collections.abc import AsyncIterable, Iterable +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest +from extract_core import ( + ConversionOutput, + InputDoc, + InputPages, + LiteParseLayoutReason, + LiteParseOCRReason, + LiteParseRouterConfig, + OutputFormat, + Pages, + Range, + Result, + Status, +) +from liteparse.types import LayoutComplexityStats, PageComplexityStats + +from extract_python import HybridLiteParsePipeline +from extract_python.hybrid_liteparse_ import LiteParseRouter + + +def _make_complexity( + *, + needs_ocr: bool = False, + complex_layout: bool = False, + ocr_reasons: list[str] | None = None, + layout_reasons: list[str] | None = None, + page_number: int = 1, +) -> PageComplexityStats: + if ocr_reasons is None: + ocr_reasons = [] + if layout_reasons is None: + layout_reasons = [] + return PageComplexityStats( + page_number=page_number, + text_length=2, + text_coverage=0.5, + has_substantial_images=True, + image_block_count=1, + image_coverage=0.5, + largest_image_coverage=0.8, + full_page_image=True, + uncovered_vector_area=0.5, + is_garbled=True, + page_area=0.5, + needs_ocr=needs_ocr, + reasons=ocr_reasons, + layout=LayoutComplexityStats( + column_count=1, + ruled_table_count=1, + ruled_table_coverage=0.5, + text_table_run_count=1, + figure_count=1, + figure_coverage=0.5, + is_complex=complex_layout, + reasons=layout_reasons, + ), + ) + + +@pytest.mark.parametrize( + ("config", "complexity", "expected_is_advanced"), + [ + ( + LiteParseRouterConfig( + needs_ocr_is_advanced=False, complex_layout_is_advanced=False + ), + _make_complexity(), + False, + ), + ( + LiteParseRouterConfig( + needs_ocr_is_advanced=True, complex_layout_is_advanced=False + ), + _make_complexity(needs_ocr=True), + True, + ), + ( + LiteParseRouterConfig( + needs_ocr_is_advanced=False, complex_layout_is_advanced=True + ), + _make_complexity(complex_layout=True), + True, + ), + ( + LiteParseRouterConfig( + needs_ocr_is_advanced=False, + complex_layout_is_advanced=False, + advanced_ocr_reasons={LiteParseOCRReason.SCANNED}, + ), + _make_complexity(complex_layout=True, ocr_reasons=["no-text"]), + False, + ), + ( + LiteParseRouterConfig( + needs_ocr_is_advanced=False, + complex_layout_is_advanced=False, + advanced_ocr_reasons={LiteParseOCRReason.SCANNED}, + ), + _make_complexity(complex_layout=True, ocr_reasons=["scanned"]), + True, + ), + ( + LiteParseRouterConfig( + needs_ocr_is_advanced=False, + complex_layout_is_advanced=False, + advanced_layout_reasons={LiteParseLayoutReason.MULTI_COLUMN}, + ), + _make_complexity(complex_layout=True, layout_reasons=["dense-graphics"]), + False, + ), + ( + LiteParseRouterConfig( + needs_ocr_is_advanced=False, + complex_layout_is_advanced=False, + advanced_layout_reasons={LiteParseLayoutReason.DENSE_GRAPHICS}, + ), + _make_complexity(complex_layout=True, layout_reasons=["dense-graphics"]), + True, + ), + ], +) +def test_liteparse_router( + config: LiteParseRouterConfig, + complexity: PageComplexityStats, + *, + expected_is_advanced: bool, +) -> None: + # Given + router = LiteParseRouter(config) + # When + is_advanced = router.is_advanced(complexity=complexity) + assert is_advanced == expected_is_advanced + + +class _PageCapture: + def __init__(self) -> None: + self._pages = [] + + @property + def pages(self) -> list[tuple[int, Range]]: + return self._pages + + async def __call__( + self, + pages: Iterable[InputPages], + output_format: OutputFormat, # noqa: ARG002 + output_path: Path, # noqa: ARG002 + ) -> AsyncIterable[Result]: + current = None + mocked_output = ConversionOutput(path=Path(""), pages=Pages(), confidence=0.8) + for page in pages: + self._pages.append((page.doc_idx, page.page_range)) + if page.doc is not current: + current = page.doc + yield Result( + input=page.doc, status=Status.SUCCESS, output=mocked_output + ) + + +async def test_hybrid_liteparse(docs: list[InputDoc], tmpdir: Path) -> None: + # Given + output_path = Path(tmpdir) + router = LiteParseRouter( + LiteParseRouterConfig( + needs_ocr_is_advanced=True, complex_layout_is_advanced=False + ) + ) + liteparse = AsyncMock() + mocked_complexities = [ + (docs[0], [_make_complexity(needs_ocr=True)]), + ( + docs[0], + [ + _make_complexity(needs_ocr=False, page_number=1), + _make_complexity(needs_ocr=True, page_number=2), + _make_complexity(needs_ocr=False, page_number=3), + ], + ), + ] + liteparse.compute_complexities.return_value = mocked_complexities + liteparse_capture = _PageCapture() + liteparse.extract_pages_content = MagicMock(side_effect=liteparse_capture) + advanced = AsyncMock() + advanced_capture = _PageCapture() + advanced.extract_pages_content = MagicMock(side_effect=advanced_capture) + pipeline = HybridLiteParsePipeline(router, liteparse, advanced) + output_format = OutputFormat.MARKDOWN + # When + async for _ in pipeline.extract_content(docs, output_format, output_path): + pass + # Then + assert liteparse_capture.pages == [(1, (0, 1)), (1, (2, 3))] + assert advanced_capture.pages == [(0, (0, 1)), (1, (1, 2))] diff --git a/extract-python/tests/test_liteparse.py b/extract-python/tests/test_liteparse.py index 84317ad..5900612 100644 --- a/extract-python/tests/test_liteparse.py +++ b/extract-python/tests/test_liteparse.py @@ -9,6 +9,7 @@ Pipeline, Status, ) + from extract_python import DoclingPipeline from extract_python.liteparse_ import LiteParsePipeline diff --git a/extract-python/tests/test_mineru.py b/extract-python/tests/test_mineru.py index 2fb0f0a..b7a42f2 100644 --- a/extract-python/tests/test_mineru.py +++ b/extract-python/tests/test_mineru.py @@ -11,8 +11,8 @@ Pipeline, Status, ) -from extract_python import MinerUPipeline +from extract_python import MinerUPipeline from tests import TEST_DATA_DIR