Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions .github/workflows/tests-extract-core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,29 @@ on:
pull_request:
paths:
- 'extract-core/**'
env:
PYTHON_VERSION: 3.13
ASTRAL_VERSION: 0.11.6

jobs:
lint-core:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: astral-sh/ruff-action@v4.0.0
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
args: "--version" # skips test by displaying the version
version: ${{ env.ASTRAL_VERSION }}
python-version: ${{ env.PYTHON_VERSION }}
enable-cache: true
working-directory: extract-core
- name: Check formatting
run: ruff format --config qa/ruff.toml --check extract-core
run: cd extract-core && uv run ruff format --check
- name: Lint test
run: ruff check --config qa/ruff.toml extract-core
run: cd extract-core && uv run ruff check

tests-core:
runs-on: ubuntu-latest
env:
PYTHON_VERSION: 3.13
ASTRAL_VERSION: 0.11.6
steps:
- uses: actions/checkout@v6
- name: Setup Python project
Expand Down
22 changes: 12 additions & 10 deletions .github/workflows/tests-extract-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,29 @@ on:
- 'extract-core/**'
- 'extract-python/**'

env:
PYTHON_VERSION: 3.13
ASTRAL_VERSION: 0.11.6

jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: astral-sh/ruff-action@v4.0.0
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
args: "--version" # skips test by displaying the version
version: ${{ env.ASTRAL_VERSION }}
python-version: ${{ env.PYTHON_VERSION }}
enable-cache: true
working-directory: extract-core
- name: Check formatting
run: ruff format --config qa/ruff.toml --check extract-python
run: cd extract-python && uv run ruff format --check
- name: Lint test
run: ruff check --config qa/ruff.toml extract-python
run: cd extract-python && uv run ruff check

tests-no-miner-u:
runs-on: ubuntu-latest
env:
PYTHON_VERSION: 3.13
ASTRAL_VERSION: 0.11.6
steps:
- uses: actions/checkout@v6
- name: Setup Python project
Expand Down Expand Up @@ -60,9 +65,6 @@ jobs:

tests-miner-u:
runs-on: ubuntu-latest
env:
PYTHON_VERSION: 3.13
ASTRAL_VERSION: 0.11.6
steps:
- uses: actions/checkout@v6
- name: Setup Python project
Expand Down
58 changes: 47 additions & 11 deletions extract-core/extract_core/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from typing import Annotated, Any
from typing import Annotated, Any, ClassVar

from icij_common.pydantic_utils import make_enum_discriminator, tagged_union
from pydantic import Discriminator, Tag
from pydantic import Discriminator, Field, Tag

from .configs import (
BasePipelineConfig,
Expand All @@ -15,10 +15,12 @@
Device,
Error,
InputDoc,
InputPages,
MarkdownDoc,
OutputFormat,
Pages,
PipelineSize,
Range,
Ranges,
Result,
Status,
Expand Down Expand Up @@ -57,9 +59,15 @@
MinerUBackend, MinerUPipelineConfig, MinerUConfig = None, None, None

try:
from .liteparse_ import LiteParseConfig, LiteParsePipelineConfig
from .liteparse_ import (
LiteParseConfig,
LiteParseLayoutReason,
LiteParseOCRReason,
LiteParsePipelineConfig,
LiteParseRouterConfig,
)
except ModuleNotFoundError:
LiteParsePipelineConfig, LiteParseConfig = None, None
LiteParsePipelineConfig, LiteParseConfig, LiteParseRouterConfig = None, None, None


_pipeline_type_discriminator = make_enum_discriminator("pipeline", PipelineType)
Expand All @@ -76,6 +84,28 @@ def pipeline_config_discriminator(v: Any) -> str:
return _pipeline_type_discriminator(v)


if LiteParsePipelineConfig is not None:
PipelineConfigNoHybrid = PipelineConfig = Annotated[
tagged_union(
BasePipelineConfig.__subclasses__(), lambda t: t.pipeline.default.value
)
| Annotated[PipelineBySize, Tag("by_size")],
Discriminator(pipeline_config_discriminator),
]

class HybridLiteParsePipelineConfig(BasePipelineConfig):
pipeline: ClassVar[PipelineType] = Field(
frozen=True, default=PipelineType.HYBRID_LITEPARSE
)
router: LiteParseRouterConfig = Field(default_factory=LiteParseRouterConfig)
liteparse: LiteParsePipelineConfig = Field(
default_factory=LiteParsePipelineConfig
)
advanced: PipelineConfigNoHybrid = Field(default_factory=DoclingPipelineConfig)

else:
HybridLiteParsePipelineConfig = None

PipelineConfig = Annotated[
tagged_union(
BasePipelineConfig.__subclasses__(), lambda t: t.pipeline.default.value
Expand All @@ -86,31 +116,37 @@ def pipeline_config_discriminator(v: Any) -> str:


__all__ = [
"LiteParsePipelineConfig",
"LiteParseConfig",
"BaseModel",
"BasePipelineConfig",
"BatchConcurrencySettings",
"ConversionOutput",
"Device",
"DoclingPipelineConfig",
"DoclingSettings",
"Error",
"InputDoc",
"InputPages",
"LiteParseConfig",
"LiteParseLayoutReason",
"LiteParseOCRReason",
"LiteParsePipelineConfig",
"LiteParseRouterConfig",
"MarkdownDoc",
"MarkerPipelineConfig",
"MinerUBackend",
"MinerUConfig",
"MinerUPipelineConfig",
"OutputFormat",
"Ranges",
"Pages",
"Pipeline",
"PipelineSize",
"PipelineConfig",
"PipelineBySize",
"PipelineSize",
"PipelineType",
"Range",
"Ranges",
"Result",
"ResultBufferConfig",
"Status",
"SupportedExt",
"ResultBufferConfig",
"DoclingSettings",
"BatchConcurrencySettings",
]
1 change: 1 addition & 0 deletions extract-core/extract_core/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
class PipelineType(StrEnum):
DOCLING = "docling"
LITEPARSE = "liteparse"
HYBRID_LITEPARSE = "hybrid-liteparse"
MARKER = "marker"
MINER_U = "mineru"

Expand Down
47 changes: 46 additions & 1 deletion extract-core/extract_core/liteparse_.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import os
from copy import copy
from datetime import timedelta
from enum import StrEnum
from pathlib import Path
from typing import ClassVar, Self

from pydantic import Field
from pydantic import Field, model_validator

from .configs import BasePipelineConfig, PipelineType, ResultBufferConfig
from .objects import BaseModel, Device, SupportedExt
Expand Down Expand Up @@ -82,3 +83,47 @@ def supported_exts(cls) -> set[SupportedExt]:

def to_device(self) -> Self:
return self


class LiteParseOCRReason(StrEnum):
SCANNED = "scanned"
NO_TEXT = "no-text"
SPARSE_TEXT = "sparse-text"
EMBEDDED_IMAGES = "embedded-images"
GARBLED = "garbled"
VECTOR_TEXT = "vector-text"
ANNOTATION_TEXT = "annotation-text"


class LiteParseLayoutReason(StrEnum):
MULTI_COLUMN = "multi-column"
TABLE_LIKELY = "table-likely"
DENSE_GRAPHICS = "dense-graphics"


def _all_ocr_reasons() -> set[LiteParseOCRReason]:
return set(LiteParseOCRReason)


class LiteParseRouterConfig(BaseModel):
needs_ocr_is_advanced: bool = True
complex_layout_is_advanced: bool = True
advanced_ocr_reasons: set[LiteParseOCRReason] | None = None
advanced_layout_reasons: set[LiteParseLayoutReason] | None = None

@model_validator(mode="after")
def reasons_are_consistent_with_flags(self) -> Self:
if self.needs_ocr_is_advanced and self.advanced_ocr_reasons is not None:
msg = (
"advanced_ocr_reasons can be supplied when needs_ocr_is_advanced"
"is set to True. When needs_ocr_is_advanced ocr reasons are ignored."
)
raise ValueError(msg)
if self.complex_layout_is_advanced and self.advanced_layout_reasons is not None:
msg = (
"complex_layout_is_advanced can be supplied when "
"advanced_layout_reasons is set to True."
" When advanced_layout_reasons ocr reasons are ignored."
)
raise ValueError(msg)
return self
23 changes: 20 additions & 3 deletions extract-core/extract_core/objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
logger = logging.getLogger(__name__)
base_config = merge_configs(icij_config(), no_enum_values_config())

Range = tuple[int, int]
Ranges = list[Range]


class BaseModel(_BaseModel):
model_config = base_config
Expand Down Expand Up @@ -241,6 +244,9 @@ def from_path(cls, path: str | Path, n_pages: int) -> Self:
ext = SupportedExt(path.suffix.lower())
return cls(path=path, ext=ext, n_pages=n_pages)

def to_all_pages(self, doc_idx: int) -> "InputPages":
return InputPages(doc=self, doc_idx=doc_idx, page_range=None)

def to_docling(self): # noqa: ANN201
from docling_core.types.io import DocumentStream # noqa: PLC0415

Expand All @@ -251,9 +257,6 @@ def to_docling(self): # noqa: ANN201
return self.path


Ranges = list[tuple[int, int]]


class Pages(BaseModel):
total: int = 0
byte_ranges: Ranges = []
Expand Down Expand Up @@ -309,3 +312,17 @@ def to_response(self) -> ResponseResult:

class ExtractionResponse(BaseModel):
results: list[ResponseResult]


class InputPages(BaseModel):
# Contiguous chunk of pages from a doc
doc: InputDoc
doc_idx: int
page_range: Range | None = None # None page range -> full document
metadata: dict[str, Any] = Field(default_factory=dict)

@property
def n_pages(self) -> int:
if self.page_range is None:
return self.doc.n_pages
return self.page_range[1] - self.page_range[0]
10 changes: 9 additions & 1 deletion extract-core/extract_core/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from extract_core import BasePipelineConfig

from .objects import InputDoc, OutputFormat, Result
from .objects import InputDoc, InputPages, OutputFormat, Result


class Pipeline[C: BasePipelineConfig](RegistrableFromConfig, ABC):
Expand All @@ -20,6 +20,14 @@ async def extract_content(
self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path
) -> AsyncIterable[Result]: ...

@abstractmethod
async def extract_pages_content(
self,
pages: Iterable[InputPages],
output_format: OutputFormat,
output_path: Path,
) -> AsyncIterable[Result]: ...

@classmethod
def _from_config(cls, config: C) -> Self:
return cls(config)
3 changes: 3 additions & 0 deletions extract-core/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,6 @@ artifacts = [
fallback-version = "0.0.0"
pattern-prefix = "extract-core-"
pattern = "default-unprefixed"

[tool.ruff]
extend = "../qa/ruff.toml"
3 changes: 2 additions & 1 deletion extract-core/tests/cli/test_cli.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import pytest
from extract_core.cli import cli_app
from typer.testing import CliRunner

from extract_core.cli import cli_app


@pytest.fixture # noqa: F405
def typer_asyncio_patch() -> None:
Expand Down
1 change: 1 addition & 0 deletions extract-core/tests/test_docling.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from docling.datamodel.accelerator_options import AcceleratorDevice
from docling.datamodel.backend_options import PdfBackendOptions
from docling.datamodel.base_models import InputFormat

from extract_core import Device, DoclingFormatOption, DoclingPipelineConfig


Expand Down
3 changes: 2 additions & 1 deletion extract-core/tests/test_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
ThreadedPdfPipelineOptions,
)
from docling.document_converter import PdfFormatOption
from pydantic import TypeAdapter

from extract_core import DoclingPipelineConfig, PipelineConfig
from extract_core.objects import Status
from pydantic import TypeAdapter


def test_supported_ext_should_include_all_docling_input_formats() -> None:
Expand Down
2 changes: 1 addition & 1 deletion extract-core/uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion extract-python/benches/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
import markdown2
import pypdfium2
from extract_core import BaseModel, OutputFormat, Pages
from extract_python.utils import chdir
from html2image import Html2Image
from PIL import Image, ImageDraw

from extract_python.utils import chdir

_WHITE_BACKGROUND_CSS = "body {background: white;}"


Expand Down
Loading
Loading