Spaces:
Sleeping
Sprint 3: vertical slice — PaddleOCR adapter, normalization pipeline, ALTO serializer
Browse filesFirst end-to-end flow: image provider output → CanonicalDocument → ALTO XML.
Provider system skeleton:
- BaseRuntime ABC (execute, is_available)
- BaseAdapter ABC (family, version, normalize)
- ProviderProfile model (runtime_type, family, capabilities, auth, timeouts)
- CapabilityMatrix model (11 boolean capabilities)
Normalization:
- CanonicalBuilder: fluent builder pattern for constructing CanonicalDocuments
(builder.add_page → page.add_text_region → region.add_line → line.add_word)
- WordBoxJsonAdapter: converts PaddleOCR 4-point polygon format to canonical
model with full provenance (native for words, inferred for block)
- NormalizationPipeline: resolves adapter by family, runs normalization
ALTO XML serializer (lxml):
- Deterministic mapping: Page→Page, TextRegion→TextBlock, TextLine→TextLine,
Word→String with HPOS/VPOS/WIDTH/HEIGHT/CONTENT/WC attributes
- Hyphenation support (SUBS_TYPE, SUBS_CONTENT)
- SP elements between words
- Description with MeasurementUnit, fileName, Processing
- ALTO v4 namespace
42 new tests (adapter: 16, ALTO serializer: 17, builder: 7, integration: 2)
283 total tests passing. Integration test validates full pipeline:
raw PaddleOCR JSON → normalize → canonical → ALTO XML with correct structure.
https://claude.ai/code/session_01Cuzvc9Pjfo5u46eT3ta2Cg
- src/app/normalization/canonical_builder.py +332 -0
- src/app/normalization/pipeline.py +62 -0
- src/app/providers/adapters/base.py +51 -0
- src/app/providers/adapters/word_box_json.py +183 -0
- src/app/providers/capabilities.py +27 -0
- src/app/providers/profiles.py +55 -0
- src/app/providers/runtimes/base.py +38 -0
- src/app/serializers/alto_xml.py +219 -0
- tests/fixtures/paddle_ocr_sample.json +22 -0
- tests/integration/test_paddle_to_alto.py +148 -0
- tests/unit/test_adapter_word_box.py +229 -0
- tests/unit/test_alto_serializer.py +285 -0
- tests/unit/test_canonical_builder.py +102 -0
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CanonicalBuilder — builder pattern for constructing CanonicalDocuments.
|
| 2 |
+
|
| 3 |
+
Usage:
|
| 4 |
+
builder = CanonicalBuilder("doc_001", InputType.IMAGE, "page.png")
|
| 5 |
+
page = builder.add_page(page_id="p1", width=2480, height=3508)
|
| 6 |
+
region = page.add_text_region("tb1", geometry=geo, provenance=prov)
|
| 7 |
+
line = region.add_line("tl1", geometry=geo, provenance=prov)
|
| 8 |
+
line.add_word("w1", text="Hello", geometry=geo, provenance=prov, confidence=0.95)
|
| 9 |
+
doc = builder.build()
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
from typing import Any
|
| 15 |
+
|
| 16 |
+
from src.app.domain.models import (
|
| 17 |
+
AltoReadiness,
|
| 18 |
+
CanonicalDocument,
|
| 19 |
+
Geometry,
|
| 20 |
+
Hyphenation,
|
| 21 |
+
NonTextRegion,
|
| 22 |
+
Page,
|
| 23 |
+
PageXmlReadiness,
|
| 24 |
+
Provenance,
|
| 25 |
+
Source,
|
| 26 |
+
TextLine,
|
| 27 |
+
TextRegion,
|
| 28 |
+
Word,
|
| 29 |
+
)
|
| 30 |
+
from src.app.domain.models.status import (
|
| 31 |
+
BlockRole,
|
| 32 |
+
InputType,
|
| 33 |
+
NonTextKind,
|
| 34 |
+
ReadinessLevel,
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class WordBuilder:
|
| 39 |
+
"""Accumulates word data before the line is finalized."""
|
| 40 |
+
|
| 41 |
+
def __init__(
|
| 42 |
+
self,
|
| 43 |
+
word_id: str,
|
| 44 |
+
text: str,
|
| 45 |
+
geometry: Geometry,
|
| 46 |
+
provenance: Provenance,
|
| 47 |
+
*,
|
| 48 |
+
confidence: float | None = None,
|
| 49 |
+
lang: str | None = None,
|
| 50 |
+
hyphenation: Hyphenation | None = None,
|
| 51 |
+
normalized_text: str | None = None,
|
| 52 |
+
style_refs: list[str] | None = None,
|
| 53 |
+
metadata: dict[str, Any] | None = None,
|
| 54 |
+
) -> None:
|
| 55 |
+
self._data = {
|
| 56 |
+
"id": word_id,
|
| 57 |
+
"text": text,
|
| 58 |
+
"geometry": geometry,
|
| 59 |
+
"provenance": provenance,
|
| 60 |
+
"confidence": confidence,
|
| 61 |
+
"lang": lang,
|
| 62 |
+
"hyphenation": hyphenation,
|
| 63 |
+
"normalized_text": normalized_text,
|
| 64 |
+
"style_refs": style_refs or [],
|
| 65 |
+
"metadata": metadata,
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
def build(self) -> Word:
|
| 69 |
+
return Word(**self._data)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
class LineBuilder:
|
| 73 |
+
"""Accumulates words for a single line."""
|
| 74 |
+
|
| 75 |
+
def __init__(
|
| 76 |
+
self,
|
| 77 |
+
line_id: str,
|
| 78 |
+
geometry: Geometry,
|
| 79 |
+
provenance: Provenance,
|
| 80 |
+
*,
|
| 81 |
+
confidence: float | None = None,
|
| 82 |
+
lang: str | None = None,
|
| 83 |
+
metadata: dict[str, Any] | None = None,
|
| 84 |
+
) -> None:
|
| 85 |
+
self._line_id = line_id
|
| 86 |
+
self._geometry = geometry
|
| 87 |
+
self._provenance = provenance
|
| 88 |
+
self._confidence = confidence
|
| 89 |
+
self._lang = lang
|
| 90 |
+
self._metadata = metadata
|
| 91 |
+
self._words: list[WordBuilder] = []
|
| 92 |
+
|
| 93 |
+
def add_word(
|
| 94 |
+
self,
|
| 95 |
+
word_id: str,
|
| 96 |
+
text: str,
|
| 97 |
+
geometry: Geometry,
|
| 98 |
+
provenance: Provenance,
|
| 99 |
+
*,
|
| 100 |
+
confidence: float | None = None,
|
| 101 |
+
lang: str | None = None,
|
| 102 |
+
hyphenation: Hyphenation | None = None,
|
| 103 |
+
normalized_text: str | None = None,
|
| 104 |
+
style_refs: list[str] | None = None,
|
| 105 |
+
metadata: dict[str, Any] | None = None,
|
| 106 |
+
) -> WordBuilder:
|
| 107 |
+
wb = WordBuilder(
|
| 108 |
+
word_id,
|
| 109 |
+
text,
|
| 110 |
+
geometry,
|
| 111 |
+
provenance,
|
| 112 |
+
confidence=confidence,
|
| 113 |
+
lang=lang,
|
| 114 |
+
hyphenation=hyphenation,
|
| 115 |
+
normalized_text=normalized_text,
|
| 116 |
+
style_refs=style_refs,
|
| 117 |
+
metadata=metadata,
|
| 118 |
+
)
|
| 119 |
+
self._words.append(wb)
|
| 120 |
+
return wb
|
| 121 |
+
|
| 122 |
+
def build(self) -> TextLine:
|
| 123 |
+
if not self._words:
|
| 124 |
+
raise ValueError(f"Line {self._line_id} has no words")
|
| 125 |
+
return TextLine(
|
| 126 |
+
id=self._line_id,
|
| 127 |
+
geometry=self._geometry,
|
| 128 |
+
provenance=self._provenance,
|
| 129 |
+
confidence=self._confidence,
|
| 130 |
+
lang=self._lang,
|
| 131 |
+
words=[w.build() for w in self._words],
|
| 132 |
+
metadata=self._metadata,
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class RegionBuilder:
|
| 137 |
+
"""Accumulates lines for a text region (block)."""
|
| 138 |
+
|
| 139 |
+
def __init__(
|
| 140 |
+
self,
|
| 141 |
+
region_id: str,
|
| 142 |
+
geometry: Geometry,
|
| 143 |
+
provenance: Provenance,
|
| 144 |
+
*,
|
| 145 |
+
role: BlockRole | None = None,
|
| 146 |
+
confidence: float | None = None,
|
| 147 |
+
lang: str | None = None,
|
| 148 |
+
metadata: dict[str, Any] | None = None,
|
| 149 |
+
) -> None:
|
| 150 |
+
self._region_id = region_id
|
| 151 |
+
self._geometry = geometry
|
| 152 |
+
self._provenance = provenance
|
| 153 |
+
self._role = role
|
| 154 |
+
self._confidence = confidence
|
| 155 |
+
self._lang = lang
|
| 156 |
+
self._metadata = metadata
|
| 157 |
+
self._lines: list[LineBuilder] = []
|
| 158 |
+
|
| 159 |
+
def add_line(
|
| 160 |
+
self,
|
| 161 |
+
line_id: str,
|
| 162 |
+
geometry: Geometry,
|
| 163 |
+
provenance: Provenance,
|
| 164 |
+
*,
|
| 165 |
+
confidence: float | None = None,
|
| 166 |
+
lang: str | None = None,
|
| 167 |
+
metadata: dict[str, Any] | None = None,
|
| 168 |
+
) -> LineBuilder:
|
| 169 |
+
lb = LineBuilder(
|
| 170 |
+
line_id,
|
| 171 |
+
geometry,
|
| 172 |
+
provenance,
|
| 173 |
+
confidence=confidence,
|
| 174 |
+
lang=lang,
|
| 175 |
+
metadata=metadata,
|
| 176 |
+
)
|
| 177 |
+
self._lines.append(lb)
|
| 178 |
+
return lb
|
| 179 |
+
|
| 180 |
+
def build(self) -> TextRegion:
|
| 181 |
+
if not self._lines:
|
| 182 |
+
raise ValueError(f"Region {self._region_id} has no lines")
|
| 183 |
+
return TextRegion(
|
| 184 |
+
id=self._region_id,
|
| 185 |
+
role=self._role,
|
| 186 |
+
geometry=self._geometry,
|
| 187 |
+
provenance=self._provenance,
|
| 188 |
+
confidence=self._confidence,
|
| 189 |
+
lang=self._lang,
|
| 190 |
+
lines=[ln.build() for ln in self._lines],
|
| 191 |
+
metadata=self._metadata,
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
class PageBuilder:
|
| 196 |
+
"""Accumulates regions for a single page."""
|
| 197 |
+
|
| 198 |
+
def __init__(
|
| 199 |
+
self,
|
| 200 |
+
page_id: str,
|
| 201 |
+
page_index: int,
|
| 202 |
+
width: float,
|
| 203 |
+
height: float,
|
| 204 |
+
) -> None:
|
| 205 |
+
self._page_id = page_id
|
| 206 |
+
self._page_index = page_index
|
| 207 |
+
self._width = width
|
| 208 |
+
self._height = height
|
| 209 |
+
self._text_regions: list[RegionBuilder] = []
|
| 210 |
+
self._non_text_regions: list[NonTextRegion] = []
|
| 211 |
+
self._reading_order: list[str] = []
|
| 212 |
+
self._warnings: list[str] = []
|
| 213 |
+
self._metadata: dict[str, Any] | None = None
|
| 214 |
+
|
| 215 |
+
def add_text_region(
|
| 216 |
+
self,
|
| 217 |
+
region_id: str,
|
| 218 |
+
geometry: Geometry,
|
| 219 |
+
provenance: Provenance,
|
| 220 |
+
*,
|
| 221 |
+
role: BlockRole | None = None,
|
| 222 |
+
confidence: float | None = None,
|
| 223 |
+
lang: str | None = None,
|
| 224 |
+
metadata: dict[str, Any] | None = None,
|
| 225 |
+
) -> RegionBuilder:
|
| 226 |
+
rb = RegionBuilder(
|
| 227 |
+
region_id,
|
| 228 |
+
geometry,
|
| 229 |
+
provenance,
|
| 230 |
+
role=role,
|
| 231 |
+
confidence=confidence,
|
| 232 |
+
lang=lang,
|
| 233 |
+
metadata=metadata,
|
| 234 |
+
)
|
| 235 |
+
self._text_regions.append(rb)
|
| 236 |
+
self._reading_order.append(region_id)
|
| 237 |
+
return rb
|
| 238 |
+
|
| 239 |
+
def add_non_text_region(
|
| 240 |
+
self,
|
| 241 |
+
region_id: str,
|
| 242 |
+
kind: NonTextKind,
|
| 243 |
+
geometry: Geometry,
|
| 244 |
+
provenance: Provenance,
|
| 245 |
+
*,
|
| 246 |
+
confidence: float | None = None,
|
| 247 |
+
metadata: dict[str, Any] | None = None,
|
| 248 |
+
) -> None:
|
| 249 |
+
self._non_text_regions.append(
|
| 250 |
+
NonTextRegion(
|
| 251 |
+
id=region_id,
|
| 252 |
+
kind=kind,
|
| 253 |
+
geometry=geometry,
|
| 254 |
+
provenance=provenance,
|
| 255 |
+
confidence=confidence,
|
| 256 |
+
metadata=metadata,
|
| 257 |
+
)
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
def add_warning(self, warning: str) -> None:
|
| 261 |
+
self._warnings.append(warning)
|
| 262 |
+
|
| 263 |
+
def set_metadata(self, metadata: dict[str, Any]) -> None:
|
| 264 |
+
self._metadata = metadata
|
| 265 |
+
|
| 266 |
+
def build(self) -> Page:
|
| 267 |
+
return Page(
|
| 268 |
+
id=self._page_id,
|
| 269 |
+
page_index=self._page_index,
|
| 270 |
+
width=self._width,
|
| 271 |
+
height=self._height,
|
| 272 |
+
alto_readiness=AltoReadiness(
|
| 273 |
+
level=ReadinessLevel.NONE, missing=["word_text"]
|
| 274 |
+
),
|
| 275 |
+
page_readiness=PageXmlReadiness(
|
| 276 |
+
level=ReadinessLevel.NONE, missing=["word_text"]
|
| 277 |
+
),
|
| 278 |
+
reading_order=self._reading_order,
|
| 279 |
+
text_regions=[r.build() for r in self._text_regions],
|
| 280 |
+
non_text_regions=self._non_text_regions,
|
| 281 |
+
warnings=self._warnings,
|
| 282 |
+
metadata=self._metadata,
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
class CanonicalBuilder:
|
| 287 |
+
"""Top-level builder for constructing a CanonicalDocument."""
|
| 288 |
+
|
| 289 |
+
def __init__(
|
| 290 |
+
self,
|
| 291 |
+
document_id: str,
|
| 292 |
+
input_type: InputType,
|
| 293 |
+
filename: str | None = None,
|
| 294 |
+
*,
|
| 295 |
+
mime_type: str | None = None,
|
| 296 |
+
checksum: str | None = None,
|
| 297 |
+
metadata: dict[str, Any] | None = None,
|
| 298 |
+
) -> None:
|
| 299 |
+
self._document_id = document_id
|
| 300 |
+
self._source = Source(
|
| 301 |
+
input_type=input_type,
|
| 302 |
+
filename=filename,
|
| 303 |
+
mime_type=mime_type,
|
| 304 |
+
checksum=checksum,
|
| 305 |
+
)
|
| 306 |
+
self._pages: list[PageBuilder] = []
|
| 307 |
+
self._metadata = metadata
|
| 308 |
+
|
| 309 |
+
def add_page(
|
| 310 |
+
self,
|
| 311 |
+
page_id: str,
|
| 312 |
+
page_index: int,
|
| 313 |
+
width: float,
|
| 314 |
+
height: float,
|
| 315 |
+
) -> PageBuilder:
|
| 316 |
+
pb = PageBuilder(page_id, page_index, width, height)
|
| 317 |
+
self._pages.append(pb)
|
| 318 |
+
return pb
|
| 319 |
+
|
| 320 |
+
def build(self) -> CanonicalDocument:
|
| 321 |
+
"""Build and validate the CanonicalDocument.
|
| 322 |
+
|
| 323 |
+
Raises pydantic.ValidationError if the resulting document is invalid.
|
| 324 |
+
"""
|
| 325 |
+
if not self._pages:
|
| 326 |
+
raise ValueError("Document must have at least one page")
|
| 327 |
+
return CanonicalDocument(
|
| 328 |
+
document_id=self._document_id,
|
| 329 |
+
source=self._source,
|
| 330 |
+
pages=[p.build() for p in self._pages],
|
| 331 |
+
metadata=self._metadata,
|
| 332 |
+
)
|
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Normalization pipeline — orchestrates raw → canonical conversion.
|
| 2 |
+
|
| 3 |
+
The pipeline:
|
| 4 |
+
1. Resolves the adapter from the provider profile
|
| 5 |
+
2. Runs the adapter to produce a CanonicalDocument
|
| 6 |
+
3. Returns the document (enrichers and validators are separate steps)
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from src.app.domain.models import CanonicalDocument, RawProviderPayload
|
| 12 |
+
from src.app.domain.models.geometry import GeometryContext
|
| 13 |
+
from src.app.providers.adapters.base import BaseAdapter
|
| 14 |
+
from src.app.providers.adapters.word_box_json import WordBoxJsonAdapter
|
| 15 |
+
|
| 16 |
+
# Registry of available adapters by family name
|
| 17 |
+
_ADAPTERS: dict[str, type[BaseAdapter]] = {
|
| 18 |
+
"word_box_json": WordBoxJsonAdapter,
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def get_adapter(family: str) -> BaseAdapter:
|
| 23 |
+
"""Instantiate an adapter for the given provider family.
|
| 24 |
+
|
| 25 |
+
Raises KeyError if the family is not registered.
|
| 26 |
+
"""
|
| 27 |
+
adapter_cls = _ADAPTERS.get(family)
|
| 28 |
+
if adapter_cls is None:
|
| 29 |
+
raise KeyError(
|
| 30 |
+
f"No adapter registered for family '{family}'. "
|
| 31 |
+
f"Available: {list(_ADAPTERS.keys())}"
|
| 32 |
+
)
|
| 33 |
+
return adapter_cls()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def normalize(
|
| 37 |
+
raw: RawProviderPayload,
|
| 38 |
+
family: str,
|
| 39 |
+
geometry_context: GeometryContext,
|
| 40 |
+
*,
|
| 41 |
+
document_id: str,
|
| 42 |
+
source_filename: str | None = None,
|
| 43 |
+
) -> CanonicalDocument:
|
| 44 |
+
"""Run the normalization pipeline: raw payload → CanonicalDocument.
|
| 45 |
+
|
| 46 |
+
Args:
|
| 47 |
+
raw: The raw provider output.
|
| 48 |
+
family: Provider family name (determines which adapter to use).
|
| 49 |
+
geometry_context: Coordinate space of the provider output.
|
| 50 |
+
document_id: ID for the produced document.
|
| 51 |
+
source_filename: Original input filename.
|
| 52 |
+
|
| 53 |
+
Returns:
|
| 54 |
+
A validated CanonicalDocument.
|
| 55 |
+
"""
|
| 56 |
+
adapter = get_adapter(family)
|
| 57 |
+
return adapter.normalize(
|
| 58 |
+
raw,
|
| 59 |
+
geometry_context,
|
| 60 |
+
document_id=document_id,
|
| 61 |
+
source_filename=source_filename,
|
| 62 |
+
)
|
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Base adapter ABC — defines how raw provider output is normalized."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from abc import ABC, abstractmethod
|
| 6 |
+
|
| 7 |
+
from src.app.domain.models import CanonicalDocument, RawProviderPayload
|
| 8 |
+
from src.app.domain.models.geometry import GeometryContext
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class BaseAdapter(ABC):
|
| 12 |
+
"""Abstract base for provider adapters.
|
| 13 |
+
|
| 14 |
+
An adapter takes a RawProviderPayload and produces a CanonicalDocument.
|
| 15 |
+
It handles the translation from provider-specific format to the canonical
|
| 16 |
+
model, including geometry normalization and provenance tagging.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
@property
|
| 20 |
+
@abstractmethod
|
| 21 |
+
def family(self) -> str:
|
| 22 |
+
"""The provider family this adapter handles (e.g. 'word_box_json')."""
|
| 23 |
+
...
|
| 24 |
+
|
| 25 |
+
@property
|
| 26 |
+
@abstractmethod
|
| 27 |
+
def version(self) -> str:
|
| 28 |
+
"""Adapter version string (e.g. 'adapter.paddle.v1')."""
|
| 29 |
+
...
|
| 30 |
+
|
| 31 |
+
@abstractmethod
|
| 32 |
+
def normalize(
|
| 33 |
+
self,
|
| 34 |
+
raw: RawProviderPayload,
|
| 35 |
+
geometry_context: GeometryContext,
|
| 36 |
+
*,
|
| 37 |
+
document_id: str,
|
| 38 |
+
source_filename: str | None = None,
|
| 39 |
+
) -> CanonicalDocument:
|
| 40 |
+
"""Convert raw provider output to a CanonicalDocument.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
raw: The raw provider payload.
|
| 44 |
+
geometry_context: Describes the coordinate space of the provider output.
|
| 45 |
+
document_id: ID for the produced CanonicalDocument.
|
| 46 |
+
source_filename: Original filename of the input image.
|
| 47 |
+
|
| 48 |
+
Returns:
|
| 49 |
+
A validated CanonicalDocument.
|
| 50 |
+
"""
|
| 51 |
+
...
|
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""word_box_json adapter — handles PaddleOCR and similar providers.
|
| 2 |
+
|
| 3 |
+
PaddleOCR output format (standard):
|
| 4 |
+
[
|
| 5 |
+
[[[x1,y1],[x2,y2],[x3,y3],[x4,y4]], ("text", confidence)],
|
| 6 |
+
...
|
| 7 |
+
]
|
| 8 |
+
|
| 9 |
+
Each item is a detected text line/word with a 4-point polygon and
|
| 10 |
+
a (text, confidence) tuple. This adapter treats each item as a word
|
| 11 |
+
within a single inferred text block.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
from src.app.domain.models import (
|
| 17 |
+
CanonicalDocument,
|
| 18 |
+
Geometry,
|
| 19 |
+
Provenance,
|
| 20 |
+
RawProviderPayload,
|
| 21 |
+
)
|
| 22 |
+
from src.app.domain.models.geometry import GeometryContext
|
| 23 |
+
from src.app.domain.models.status import (
|
| 24 |
+
EvidenceType,
|
| 25 |
+
GeometryStatus,
|
| 26 |
+
InputType,
|
| 27 |
+
)
|
| 28 |
+
from src.app.geometry.bbox import union_all
|
| 29 |
+
from src.app.geometry.normalization import four_point_to_polygon, four_point_to_xywh
|
| 30 |
+
from src.app.normalization.canonical_builder import CanonicalBuilder
|
| 31 |
+
from src.app.providers.adapters.base import BaseAdapter
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class WordBoxJsonAdapter(BaseAdapter):
|
| 35 |
+
"""Adapter for the word_box_json family (PaddleOCR, etc.)."""
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def family(self) -> str:
|
| 39 |
+
return "word_box_json"
|
| 40 |
+
|
| 41 |
+
@property
|
| 42 |
+
def version(self) -> str:
|
| 43 |
+
return "adapter.word_box_json.v1"
|
| 44 |
+
|
| 45 |
+
def normalize(
|
| 46 |
+
self,
|
| 47 |
+
raw: RawProviderPayload,
|
| 48 |
+
geometry_context: GeometryContext,
|
| 49 |
+
*,
|
| 50 |
+
document_id: str,
|
| 51 |
+
source_filename: str | None = None,
|
| 52 |
+
) -> CanonicalDocument:
|
| 53 |
+
payload = raw.payload
|
| 54 |
+
if not isinstance(payload, list):
|
| 55 |
+
raise ValueError(
|
| 56 |
+
f"word_box_json expects a list payload, got {type(payload).__name__}"
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
builder = CanonicalBuilder(
|
| 60 |
+
document_id=document_id,
|
| 61 |
+
input_type=InputType.IMAGE,
|
| 62 |
+
filename=source_filename,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
page = builder.add_page(
|
| 66 |
+
page_id="p1",
|
| 67 |
+
page_index=0,
|
| 68 |
+
width=geometry_context.source_width,
|
| 69 |
+
height=geometry_context.source_height,
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
# Collect all word bboxes for building a block-level bbox
|
| 73 |
+
word_bboxes: list[tuple[float, float, float, float]] = []
|
| 74 |
+
word_data: list[dict] = []
|
| 75 |
+
|
| 76 |
+
for idx, item in enumerate(payload):
|
| 77 |
+
points, text_conf = self._parse_item(item, idx)
|
| 78 |
+
text, confidence = self._parse_text_conf(text_conf, idx)
|
| 79 |
+
|
| 80 |
+
bbox = four_point_to_xywh(points)
|
| 81 |
+
polygon = four_point_to_polygon(points)
|
| 82 |
+
|
| 83 |
+
# Apply resize factor if present
|
| 84 |
+
if geometry_context.resize_factor and geometry_context.resize_factor != 1.0:
|
| 85 |
+
factor = 1.0 / geometry_context.resize_factor
|
| 86 |
+
bbox = (bbox[0] * factor, bbox[1] * factor, bbox[2] * factor, bbox[3] * factor)
|
| 87 |
+
polygon = [(x * factor, y * factor) for x, y in polygon]
|
| 88 |
+
|
| 89 |
+
word_bboxes.append(bbox)
|
| 90 |
+
word_data.append({
|
| 91 |
+
"bbox": bbox,
|
| 92 |
+
"polygon": polygon,
|
| 93 |
+
"text": text,
|
| 94 |
+
"confidence": confidence,
|
| 95 |
+
"source_idx": idx,
|
| 96 |
+
})
|
| 97 |
+
|
| 98 |
+
if not word_data:
|
| 99 |
+
raise ValueError("word_box_json payload contains no items")
|
| 100 |
+
|
| 101 |
+
# Build a single inferred text block containing all items as words
|
| 102 |
+
# Each PaddleOCR item is treated as a line with one word
|
| 103 |
+
block_bbox = union_all(word_bboxes)
|
| 104 |
+
block_prov = Provenance(
|
| 105 |
+
provider=raw.provider_id,
|
| 106 |
+
adapter=self.version,
|
| 107 |
+
source_ref="$",
|
| 108 |
+
evidence_type=EvidenceType.DERIVED,
|
| 109 |
+
derived_from=[f"w{i+1}" for i in range(len(word_data))],
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
region = page.add_text_region(
|
| 113 |
+
region_id="tb1",
|
| 114 |
+
geometry=Geometry(
|
| 115 |
+
bbox=block_bbox,
|
| 116 |
+
status=GeometryStatus.INFERRED,
|
| 117 |
+
),
|
| 118 |
+
provenance=block_prov,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
for i, wd in enumerate(word_data):
|
| 122 |
+
word_id = f"w{i + 1}"
|
| 123 |
+
line_id = f"tl{i + 1}"
|
| 124 |
+
|
| 125 |
+
prov = Provenance(
|
| 126 |
+
provider=raw.provider_id,
|
| 127 |
+
adapter=self.version,
|
| 128 |
+
source_ref=f"$[{wd['source_idx']}]",
|
| 129 |
+
evidence_type=EvidenceType.PROVIDER_NATIVE,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
geo = Geometry(
|
| 133 |
+
bbox=wd["bbox"],
|
| 134 |
+
polygon=wd["polygon"],
|
| 135 |
+
status=GeometryStatus.EXACT,
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
# Each PaddleOCR detection becomes a line with one word
|
| 139 |
+
line = region.add_line(line_id, geometry=geo, provenance=prov)
|
| 140 |
+
line.add_word(
|
| 141 |
+
word_id,
|
| 142 |
+
text=wd["text"],
|
| 143 |
+
geometry=geo,
|
| 144 |
+
provenance=prov,
|
| 145 |
+
confidence=wd["confidence"],
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
return builder.build()
|
| 149 |
+
|
| 150 |
+
# -- Private helpers ---------------------------------------------------------
|
| 151 |
+
|
| 152 |
+
@staticmethod
|
| 153 |
+
def _parse_item(item: object, idx: int) -> tuple[list, object]:
|
| 154 |
+
"""Extract (points, text_conf) from a PaddleOCR result item."""
|
| 155 |
+
if not isinstance(item, (list, tuple)) or len(item) != 2:
|
| 156 |
+
raise ValueError(
|
| 157 |
+
f"Item {idx}: expected [points, (text, conf)], got {type(item).__name__}"
|
| 158 |
+
)
|
| 159 |
+
points = item[0]
|
| 160 |
+
text_conf = item[1]
|
| 161 |
+
if not isinstance(points, list) or len(points) != 4:
|
| 162 |
+
raise ValueError(
|
| 163 |
+
f"Item {idx}: expected 4 polygon points, got {len(points) if isinstance(points, list) else type(points).__name__}"
|
| 164 |
+
)
|
| 165 |
+
return points, text_conf
|
| 166 |
+
|
| 167 |
+
@staticmethod
|
| 168 |
+
def _parse_text_conf(text_conf: object, idx: int) -> tuple[str, float | None]:
|
| 169 |
+
"""Extract (text, confidence) from the second element."""
|
| 170 |
+
if isinstance(text_conf, (list, tuple)) and len(text_conf) == 2:
|
| 171 |
+
text = str(text_conf[0])
|
| 172 |
+
try:
|
| 173 |
+
confidence = float(text_conf[1])
|
| 174 |
+
confidence = max(0.0, min(1.0, confidence))
|
| 175 |
+
except (TypeError, ValueError):
|
| 176 |
+
confidence = None
|
| 177 |
+
return text, confidence
|
| 178 |
+
if isinstance(text_conf, str):
|
| 179 |
+
return text_conf, None
|
| 180 |
+
raise ValueError(
|
| 181 |
+
f"Item {idx}: expected (text, confidence) or text string, "
|
| 182 |
+
f"got {type(text_conf).__name__}"
|
| 183 |
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Capability matrix — describes what a provider can produce."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from pydantic import BaseModel, ConfigDict
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class CapabilityMatrix(BaseModel):
|
| 9 |
+
"""Explicit description of a provider's output capabilities.
|
| 10 |
+
|
| 11 |
+
Used to decide which enrichers to activate, which exports to authorize,
|
| 12 |
+
and how to inform the user about data completeness.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
model_config = ConfigDict(frozen=True)
|
| 16 |
+
|
| 17 |
+
block_geometry: bool = False
|
| 18 |
+
line_geometry: bool = False
|
| 19 |
+
word_geometry: bool = False
|
| 20 |
+
polygon_geometry: bool = False
|
| 21 |
+
baseline: bool = False
|
| 22 |
+
reading_order: bool = False
|
| 23 |
+
text_confidence: bool = False
|
| 24 |
+
language: bool = False
|
| 25 |
+
non_text_regions: bool = False
|
| 26 |
+
tables: bool = False
|
| 27 |
+
rotation_info: bool = False
|
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Provider profiles — describes a concrete configured provider instance."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from datetime import datetime, timezone
|
| 6 |
+
from enum import Enum
|
| 7 |
+
|
| 8 |
+
from pydantic import BaseModel, ConfigDict, Field
|
| 9 |
+
|
| 10 |
+
from src.app.providers.capabilities import CapabilityMatrix
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class RuntimeType(str, Enum):
|
| 14 |
+
LOCAL = "local"
|
| 15 |
+
HUB = "hub"
|
| 16 |
+
API = "api"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class ProviderFamily(str, Enum):
|
| 20 |
+
WORD_BOX_JSON = "word_box_json"
|
| 21 |
+
LINE_BOX_JSON = "line_box_json"
|
| 22 |
+
REGION_LINE_WORD_POLYGON = "region_line_word_polygon"
|
| 23 |
+
TEXT_ONLY = "text_only"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class AuthMode(str, Enum):
|
| 27 |
+
NONE = "none"
|
| 28 |
+
API_KEY = "api_key"
|
| 29 |
+
BEARER = "bearer"
|
| 30 |
+
HF_TOKEN = "hf_token"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class ProviderProfile(BaseModel):
|
| 34 |
+
"""A concrete, persisted provider configuration."""
|
| 35 |
+
|
| 36 |
+
model_config = ConfigDict(frozen=True)
|
| 37 |
+
|
| 38 |
+
provider_id: str = Field(min_length=1)
|
| 39 |
+
display_name: str = Field(min_length=1)
|
| 40 |
+
runtime_type: RuntimeType
|
| 41 |
+
model_id_or_path: str = Field(min_length=1)
|
| 42 |
+
|
| 43 |
+
endpoint: str | None = None
|
| 44 |
+
auth_mode: AuthMode = AuthMode.NONE
|
| 45 |
+
auth_secret_ref: str | None = None
|
| 46 |
+
|
| 47 |
+
family: ProviderFamily
|
| 48 |
+
capabilities: CapabilityMatrix = Field(default_factory=CapabilityMatrix)
|
| 49 |
+
|
| 50 |
+
timeout: int = Field(default=120, gt=0)
|
| 51 |
+
prompt_template: str | None = None
|
| 52 |
+
|
| 53 |
+
last_test_status: str | None = None
|
| 54 |
+
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
| 55 |
+
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Base runtime ABC — defines how a provider is executed."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from abc import ABC, abstractmethod
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from src.app.domain.models import RawProviderPayload
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class BaseRuntime(ABC):
|
| 13 |
+
"""Abstract base for provider runtimes (local, hub, api)."""
|
| 14 |
+
|
| 15 |
+
@abstractmethod
|
| 16 |
+
def execute(
|
| 17 |
+
self,
|
| 18 |
+
image_path: Path,
|
| 19 |
+
model_id: str,
|
| 20 |
+
*,
|
| 21 |
+
options: dict[str, Any] | None = None,
|
| 22 |
+
) -> RawProviderPayload:
|
| 23 |
+
"""Run the provider on an image and return the raw output.
|
| 24 |
+
|
| 25 |
+
Args:
|
| 26 |
+
image_path: Path to the input image file.
|
| 27 |
+
model_id: Model identifier (local path, hub ID, or API model name).
|
| 28 |
+
options: Additional provider-specific options.
|
| 29 |
+
|
| 30 |
+
Returns:
|
| 31 |
+
RawProviderPayload wrapping the provider's raw JSON output.
|
| 32 |
+
"""
|
| 33 |
+
...
|
| 34 |
+
|
| 35 |
+
@abstractmethod
|
| 36 |
+
def is_available(self) -> bool:
|
| 37 |
+
"""Check if this runtime is usable in the current environment."""
|
| 38 |
+
...
|
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ALTO XML serializer — deterministic conversion from CanonicalDocument to ALTO v4.
|
| 2 |
+
|
| 3 |
+
This serializer is a pure output transformation. It MUST NOT:
|
| 4 |
+
- Call any model or provider
|
| 5 |
+
- Reconstruct segmentation
|
| 6 |
+
- Correct text
|
| 7 |
+
- Invent coordinates
|
| 8 |
+
- Make export eligibility decisions
|
| 9 |
+
|
| 10 |
+
It receives a validated CanonicalDocument and produces ALTO XML bytes.
|
| 11 |
+
|
| 12 |
+
ALTO mapping:
|
| 13 |
+
Page → <Page>
|
| 14 |
+
TextRegion → <TextBlock>
|
| 15 |
+
TextLine → <TextLine>
|
| 16 |
+
Word → <String>
|
| 17 |
+
|
| 18 |
+
Coordinate mapping:
|
| 19 |
+
bbox[0] → HPOS
|
| 20 |
+
bbox[1] → VPOS
|
| 21 |
+
bbox[2] → WIDTH
|
| 22 |
+
bbox[3] → HEIGHT
|
| 23 |
+
text → CONTENT
|
| 24 |
+
confidence → WC
|
| 25 |
+
hyphenation.part=1 → SUBS_TYPE="HypPart1"
|
| 26 |
+
hyphenation.part=2 → SUBS_TYPE="HypPart2"
|
| 27 |
+
hyphenation.full_form → SUBS_CONTENT
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
from lxml import etree
|
| 33 |
+
|
| 34 |
+
from src.app.domain.models import CanonicalDocument, Page, TextLine, TextRegion, Word
|
| 35 |
+
from src.app.geometry.quantization import RoundingStrategy, quantize_bbox
|
| 36 |
+
|
| 37 |
+
ALTO_NS = "http://www.loc.gov/standards/alto/ns-v4#"
|
| 38 |
+
XSI_NS = "http://www.w3.org/2001/XMLSchema-instance"
|
| 39 |
+
SCHEMA_LOCATION = (
|
| 40 |
+
"http://www.loc.gov/standards/alto/ns-v4# "
|
| 41 |
+
"http://www.loc.gov/standards/alto/v4/alto-4-2.xsd"
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
NSMAP = {
|
| 45 |
+
None: ALTO_NS,
|
| 46 |
+
"xsi": XSI_NS,
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def serialize_alto(
|
| 51 |
+
doc: CanonicalDocument,
|
| 52 |
+
*,
|
| 53 |
+
rounding: RoundingStrategy = RoundingStrategy.ROUND,
|
| 54 |
+
pretty_print: bool = True,
|
| 55 |
+
encoding: str = "UTF-8",
|
| 56 |
+
) -> bytes:
|
| 57 |
+
"""Serialize a CanonicalDocument to ALTO v4 XML bytes.
|
| 58 |
+
|
| 59 |
+
Args:
|
| 60 |
+
doc: The validated canonical document.
|
| 61 |
+
rounding: Strategy for converting float coordinates to integers.
|
| 62 |
+
pretty_print: Whether to indent the XML output.
|
| 63 |
+
encoding: Output encoding.
|
| 64 |
+
|
| 65 |
+
Returns:
|
| 66 |
+
The ALTO XML as bytes.
|
| 67 |
+
"""
|
| 68 |
+
root = _build_alto_tree(doc, rounding)
|
| 69 |
+
return etree.tostring(
|
| 70 |
+
root,
|
| 71 |
+
pretty_print=pretty_print,
|
| 72 |
+
xml_declaration=True,
|
| 73 |
+
encoding=encoding,
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def serialize_alto_to_string(
|
| 78 |
+
doc: CanonicalDocument,
|
| 79 |
+
*,
|
| 80 |
+
rounding: RoundingStrategy = RoundingStrategy.ROUND,
|
| 81 |
+
) -> str:
|
| 82 |
+
"""Serialize to a UTF-8 string (convenience for tests)."""
|
| 83 |
+
return serialize_alto(doc, rounding=rounding).decode("utf-8")
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# -- Tree construction --------------------------------------------------------
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _build_alto_tree(
|
| 90 |
+
doc: CanonicalDocument, rounding: RoundingStrategy
|
| 91 |
+
) -> etree._Element:
|
| 92 |
+
root = etree.Element(f"{{{ALTO_NS}}}alto", nsmap=NSMAP)
|
| 93 |
+
root.set(f"{{{XSI_NS}}}schemaLocation", SCHEMA_LOCATION)
|
| 94 |
+
|
| 95 |
+
# <Description>
|
| 96 |
+
desc = etree.SubElement(root, f"{{{ALTO_NS}}}Description")
|
| 97 |
+
_add_description(desc, doc)
|
| 98 |
+
|
| 99 |
+
# <Layout>
|
| 100 |
+
layout = etree.SubElement(root, f"{{{ALTO_NS}}}Layout")
|
| 101 |
+
for page in doc.pages:
|
| 102 |
+
_add_page(layout, page, rounding)
|
| 103 |
+
|
| 104 |
+
return root
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _add_description(desc: etree._Element, doc: CanonicalDocument) -> None:
|
| 108 |
+
"""Add <Description> metadata."""
|
| 109 |
+
measurement = etree.SubElement(desc, f"{{{ALTO_NS}}}MeasurementUnit")
|
| 110 |
+
measurement.text = "pixel"
|
| 111 |
+
|
| 112 |
+
src_info = etree.SubElement(desc, f"{{{ALTO_NS}}}sourceImageInformation")
|
| 113 |
+
file_name = etree.SubElement(src_info, f"{{{ALTO_NS}}}fileName")
|
| 114 |
+
file_name.text = doc.source.filename or doc.document_id
|
| 115 |
+
|
| 116 |
+
# Processing info
|
| 117 |
+
processing = etree.SubElement(desc, f"{{{ALTO_NS}}}Processing")
|
| 118 |
+
processing.set("ID", "proc_1")
|
| 119 |
+
sw = etree.SubElement(processing, f"{{{ALTO_NS}}}processingSoftware")
|
| 120 |
+
sw_name = etree.SubElement(sw, f"{{{ALTO_NS}}}softwareName")
|
| 121 |
+
sw_name.text = "XmLLM"
|
| 122 |
+
sw_version = etree.SubElement(sw, f"{{{ALTO_NS}}}softwareVersion")
|
| 123 |
+
sw_version.text = doc.schema_version
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _add_page(
|
| 127 |
+
layout: etree._Element, page: Page, rounding: RoundingStrategy
|
| 128 |
+
) -> None:
|
| 129 |
+
"""Add a <Page> with its <PrintSpace> and blocks."""
|
| 130 |
+
page_el = etree.SubElement(layout, f"{{{ALTO_NS}}}Page")
|
| 131 |
+
page_el.set("ID", page.id)
|
| 132 |
+
page_el.set("PHYSICAL_IMG_NR", str(page.page_index + 1))
|
| 133 |
+
page_el.set("WIDTH", str(int(page.width)))
|
| 134 |
+
page_el.set("HEIGHT", str(int(page.height)))
|
| 135 |
+
|
| 136 |
+
# <PrintSpace> covers the entire page
|
| 137 |
+
ps = etree.SubElement(page_el, f"{{{ALTO_NS}}}PrintSpace")
|
| 138 |
+
ps.set("HPOS", "0")
|
| 139 |
+
ps.set("VPOS", "0")
|
| 140 |
+
ps.set("WIDTH", str(int(page.width)))
|
| 141 |
+
ps.set("HEIGHT", str(int(page.height)))
|
| 142 |
+
|
| 143 |
+
for region in page.text_regions:
|
| 144 |
+
_add_text_block(ps, region, rounding)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _add_text_block(
|
| 148 |
+
parent: etree._Element, region: TextRegion, rounding: RoundingStrategy
|
| 149 |
+
) -> None:
|
| 150 |
+
"""Add a <TextBlock> with its lines."""
|
| 151 |
+
tb = etree.SubElement(parent, f"{{{ALTO_NS}}}TextBlock")
|
| 152 |
+
tb.set("ID", region.id)
|
| 153 |
+
_set_bbox_attrs(tb, region.geometry.bbox, rounding)
|
| 154 |
+
|
| 155 |
+
if region.lang:
|
| 156 |
+
tb.set("LANG", region.lang)
|
| 157 |
+
|
| 158 |
+
for line in region.lines:
|
| 159 |
+
_add_text_line(tb, line, rounding)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _add_text_line(
|
| 163 |
+
parent: etree._Element, line: TextLine, rounding: RoundingStrategy
|
| 164 |
+
) -> None:
|
| 165 |
+
"""Add a <TextLine> with its strings."""
|
| 166 |
+
tl = etree.SubElement(parent, f"{{{ALTO_NS}}}TextLine")
|
| 167 |
+
tl.set("ID", line.id)
|
| 168 |
+
_set_bbox_attrs(tl, line.geometry.bbox, rounding)
|
| 169 |
+
|
| 170 |
+
for i, word in enumerate(line.words):
|
| 171 |
+
if i > 0:
|
| 172 |
+
_add_sp(tl)
|
| 173 |
+
_add_string(tl, word, rounding)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _add_string(
|
| 177 |
+
parent: etree._Element, word: Word, rounding: RoundingStrategy
|
| 178 |
+
) -> None:
|
| 179 |
+
"""Add a <String> element for a word."""
|
| 180 |
+
s = etree.SubElement(parent, f"{{{ALTO_NS}}}String")
|
| 181 |
+
s.set("ID", word.id)
|
| 182 |
+
_set_bbox_attrs(s, word.geometry.bbox, rounding)
|
| 183 |
+
s.set("CONTENT", word.text)
|
| 184 |
+
|
| 185 |
+
if word.confidence is not None:
|
| 186 |
+
s.set("WC", f"{word.confidence:.2f}")
|
| 187 |
+
|
| 188 |
+
if word.lang:
|
| 189 |
+
s.set("LANG", word.lang)
|
| 190 |
+
|
| 191 |
+
# Hyphenation
|
| 192 |
+
if word.hyphenation and word.hyphenation.is_hyphenated:
|
| 193 |
+
if word.hyphenation.part == 1:
|
| 194 |
+
s.set("SUBS_TYPE", "HypPart1")
|
| 195 |
+
elif word.hyphenation.part == 2:
|
| 196 |
+
s.set("SUBS_TYPE", "HypPart2")
|
| 197 |
+
if word.hyphenation.full_form:
|
| 198 |
+
s.set("SUBS_CONTENT", word.hyphenation.full_form)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _add_sp(parent: etree._Element) -> None:
|
| 202 |
+
"""Add a <SP> (space) element between words."""
|
| 203 |
+
etree.SubElement(parent, f"{{{ALTO_NS}}}SP")
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
# -- Helpers ------------------------------------------------------------------
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _set_bbox_attrs(
|
| 210 |
+
el: etree._Element,
|
| 211 |
+
bbox: tuple[float, float, float, float],
|
| 212 |
+
rounding: RoundingStrategy,
|
| 213 |
+
) -> None:
|
| 214 |
+
"""Set HPOS, VPOS, WIDTH, HEIGHT attributes from a canonical bbox."""
|
| 215 |
+
hpos, vpos, width, height = quantize_bbox(bbox, rounding)
|
| 216 |
+
el.set("HPOS", str(hpos))
|
| 217 |
+
el.set("VPOS", str(vpos))
|
| 218 |
+
el.set("WIDTH", str(width))
|
| 219 |
+
el.set("HEIGHT", str(height))
|
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
[
|
| 3 |
+
[[100, 200], [300, 200], [300, 240], [100, 240]],
|
| 4 |
+
["Bonjour", 0.96]
|
| 5 |
+
],
|
| 6 |
+
[
|
| 7 |
+
[[320, 200], [450, 200], [450, 240], [320, 240]],
|
| 8 |
+
["le", 0.99]
|
| 9 |
+
],
|
| 10 |
+
[
|
| 11 |
+
[[470, 200], [620, 200], [620, 240], [470, 240]],
|
| 12 |
+
["monde", 0.94]
|
| 13 |
+
],
|
| 14 |
+
[
|
| 15 |
+
[[100, 280], [550, 280], [550, 320], [100, 320]],
|
| 16 |
+
["Ceci est une seconde ligne", 0.91]
|
| 17 |
+
],
|
| 18 |
+
[
|
| 19 |
+
[[100, 360], [500, 360], [500, 400], [100, 400]],
|
| 20 |
+
["Troisième ligne de texte", 0.88]
|
| 21 |
+
]
|
| 22 |
+
]
|
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Integration test: PaddleOCR raw output → CanonicalDocument → ALTO XML.
|
| 2 |
+
|
| 3 |
+
This is the Sprint 3 vertical slice end-to-end test.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
from lxml import etree
|
| 12 |
+
|
| 13 |
+
from src.app.domain.models import RawProviderPayload
|
| 14 |
+
from src.app.domain.models.geometry import GeometryContext
|
| 15 |
+
from src.app.normalization.pipeline import normalize
|
| 16 |
+
from src.app.serializers.alto_xml import ALTO_NS, serialize_alto
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class TestPaddleToAlto:
|
| 20 |
+
"""Full pipeline: raw fixture → normalize → serialize → valid ALTO."""
|
| 21 |
+
|
| 22 |
+
def test_end_to_end(self, fixtures_dir: Path) -> None:
|
| 23 |
+
# 1. Load raw PaddleOCR output
|
| 24 |
+
with open(fixtures_dir / "paddle_ocr_sample.json") as f:
|
| 25 |
+
paddle_output = json.load(f)
|
| 26 |
+
|
| 27 |
+
raw = RawProviderPayload(
|
| 28 |
+
provider_id="paddleocr",
|
| 29 |
+
adapter_id="adapter.word_box_json.v1",
|
| 30 |
+
runtime_type="local",
|
| 31 |
+
payload=paddle_output,
|
| 32 |
+
image_width=2480,
|
| 33 |
+
image_height=3508,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
geo_ctx = GeometryContext(source_width=2480, source_height=3508)
|
| 37 |
+
|
| 38 |
+
# 2. Normalize to CanonicalDocument
|
| 39 |
+
doc = normalize(
|
| 40 |
+
raw,
|
| 41 |
+
family="word_box_json",
|
| 42 |
+
geometry_context=geo_ctx,
|
| 43 |
+
document_id="integration_test_001",
|
| 44 |
+
source_filename="page_001.png",
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
# Verify canonical document structure
|
| 48 |
+
assert doc.document_id == "integration_test_001"
|
| 49 |
+
assert doc.source.filename == "page_001.png"
|
| 50 |
+
assert len(doc.pages) == 1
|
| 51 |
+
|
| 52 |
+
page = doc.pages[0]
|
| 53 |
+
assert page.width == 2480
|
| 54 |
+
assert page.height == 3508
|
| 55 |
+
assert len(page.text_regions) >= 1
|
| 56 |
+
|
| 57 |
+
# Count total words
|
| 58 |
+
all_words = [
|
| 59 |
+
w
|
| 60 |
+
for r in page.text_regions
|
| 61 |
+
for l in r.lines
|
| 62 |
+
for w in l.words
|
| 63 |
+
]
|
| 64 |
+
assert len(all_words) == 5
|
| 65 |
+
|
| 66 |
+
# Verify provenance chain
|
| 67 |
+
for word in all_words:
|
| 68 |
+
assert word.provenance.provider == "paddleocr"
|
| 69 |
+
assert word.geometry.polygon is not None
|
| 70 |
+
|
| 71 |
+
# 3. Serialize to ALTO XML
|
| 72 |
+
alto_bytes = serialize_alto(doc)
|
| 73 |
+
assert alto_bytes # not empty
|
| 74 |
+
|
| 75 |
+
# 4. Parse and validate ALTO structure
|
| 76 |
+
root = etree.fromstring(alto_bytes)
|
| 77 |
+
assert root.tag == f"{{{ALTO_NS}}}alto"
|
| 78 |
+
|
| 79 |
+
# Verify page
|
| 80 |
+
alto_page = root.find(f".//{{{ALTO_NS}}}Page")
|
| 81 |
+
assert alto_page is not None
|
| 82 |
+
assert alto_page.get("WIDTH") == "2480"
|
| 83 |
+
assert alto_page.get("HEIGHT") == "3508"
|
| 84 |
+
|
| 85 |
+
# Verify strings
|
| 86 |
+
strings = root.findall(f".//{{{ALTO_NS}}}String")
|
| 87 |
+
assert len(strings) == 5
|
| 88 |
+
|
| 89 |
+
# First word content matches
|
| 90 |
+
assert strings[0].get("CONTENT") == "Bonjour"
|
| 91 |
+
|
| 92 |
+
# Coordinates are integers
|
| 93 |
+
for s in strings:
|
| 94 |
+
assert s.get("HPOS").isdigit()
|
| 95 |
+
assert s.get("VPOS").isdigit()
|
| 96 |
+
assert s.get("WIDTH").isdigit()
|
| 97 |
+
assert s.get("HEIGHT").isdigit()
|
| 98 |
+
|
| 99 |
+
# Confidence present
|
| 100 |
+
assert strings[0].get("WC") == "0.96"
|
| 101 |
+
|
| 102 |
+
# SP elements between words within a line
|
| 103 |
+
text_lines = root.findall(f".//{{{ALTO_NS}}}TextLine")
|
| 104 |
+
assert len(text_lines) == 5 # each paddle item = 1 line
|
| 105 |
+
|
| 106 |
+
def test_canonical_json_roundtrip(self, fixtures_dir: Path) -> None:
|
| 107 |
+
"""The canonical doc produced by the pipeline survives JSON serialization."""
|
| 108 |
+
with open(fixtures_dir / "paddle_ocr_sample.json") as f:
|
| 109 |
+
paddle_output = json.load(f)
|
| 110 |
+
|
| 111 |
+
raw = RawProviderPayload(
|
| 112 |
+
provider_id="paddleocr",
|
| 113 |
+
adapter_id="v1",
|
| 114 |
+
runtime_type="local",
|
| 115 |
+
payload=paddle_output,
|
| 116 |
+
image_width=2480,
|
| 117 |
+
image_height=3508,
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
geo_ctx = GeometryContext(source_width=2480, source_height=3508)
|
| 121 |
+
doc = normalize(
|
| 122 |
+
raw,
|
| 123 |
+
family="word_box_json",
|
| 124 |
+
geometry_context=geo_ctx,
|
| 125 |
+
document_id="roundtrip_test",
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
# Serialize to JSON and back
|
| 129 |
+
json_str = doc.model_dump_json()
|
| 130 |
+
from src.app.domain.models import CanonicalDocument
|
| 131 |
+
|
| 132 |
+
doc2 = CanonicalDocument.model_validate_json(json_str)
|
| 133 |
+
|
| 134 |
+
# Same structure
|
| 135 |
+
assert doc2.document_id == doc.document_id
|
| 136 |
+
assert len(doc2.pages) == 1
|
| 137 |
+
words_original = [
|
| 138 |
+
w.text for r in doc.pages[0].text_regions for l in r.lines for w in l.words
|
| 139 |
+
]
|
| 140 |
+
words_restored = [
|
| 141 |
+
w.text for r in doc2.pages[0].text_regions for l in r.lines for w in l.words
|
| 142 |
+
]
|
| 143 |
+
assert words_original == words_restored
|
| 144 |
+
|
| 145 |
+
# Same ALTO output
|
| 146 |
+
alto1 = serialize_alto(doc)
|
| 147 |
+
alto2 = serialize_alto(doc2)
|
| 148 |
+
assert alto1 == alto2
|
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the word_box_json adapter (PaddleOCR format)."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import pytest
|
| 9 |
+
|
| 10 |
+
from src.app.domain.models import (
|
| 11 |
+
EvidenceType,
|
| 12 |
+
GeometryStatus,
|
| 13 |
+
RawProviderPayload,
|
| 14 |
+
)
|
| 15 |
+
from src.app.domain.models.geometry import GeometryContext
|
| 16 |
+
from src.app.providers.adapters.word_box_json import WordBoxJsonAdapter
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@pytest.fixture
|
| 20 |
+
def adapter() -> WordBoxJsonAdapter:
|
| 21 |
+
return WordBoxJsonAdapter()
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@pytest.fixture
|
| 25 |
+
def paddle_payload(fixtures_dir: Path) -> list:
|
| 26 |
+
with open(fixtures_dir / "paddle_ocr_sample.json") as f:
|
| 27 |
+
return json.load(f)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@pytest.fixture
|
| 31 |
+
def raw(paddle_payload: list) -> RawProviderPayload:
|
| 32 |
+
return RawProviderPayload(
|
| 33 |
+
provider_id="paddleocr",
|
| 34 |
+
adapter_id="adapter.word_box_json.v1",
|
| 35 |
+
runtime_type="local",
|
| 36 |
+
payload=paddle_payload,
|
| 37 |
+
image_width=2480,
|
| 38 |
+
image_height=3508,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@pytest.fixture
|
| 43 |
+
def geo_ctx() -> GeometryContext:
|
| 44 |
+
return GeometryContext(source_width=2480, source_height=3508)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class TestWordBoxJsonAdapter:
|
| 48 |
+
def test_family(self, adapter: WordBoxJsonAdapter) -> None:
|
| 49 |
+
assert adapter.family == "word_box_json"
|
| 50 |
+
|
| 51 |
+
def test_version(self, adapter: WordBoxJsonAdapter) -> None:
|
| 52 |
+
assert "v1" in adapter.version
|
| 53 |
+
|
| 54 |
+
def test_normalize_produces_valid_document(
|
| 55 |
+
self,
|
| 56 |
+
adapter: WordBoxJsonAdapter,
|
| 57 |
+
raw: RawProviderPayload,
|
| 58 |
+
geo_ctx: GeometryContext,
|
| 59 |
+
) -> None:
|
| 60 |
+
doc = adapter.normalize(
|
| 61 |
+
raw, geo_ctx, document_id="test_doc", source_filename="test.png"
|
| 62 |
+
)
|
| 63 |
+
assert doc.document_id == "test_doc"
|
| 64 |
+
assert doc.source.filename == "test.png"
|
| 65 |
+
assert len(doc.pages) == 1
|
| 66 |
+
|
| 67 |
+
def test_page_dimensions(
|
| 68 |
+
self,
|
| 69 |
+
adapter: WordBoxJsonAdapter,
|
| 70 |
+
raw: RawProviderPayload,
|
| 71 |
+
geo_ctx: GeometryContext,
|
| 72 |
+
) -> None:
|
| 73 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 74 |
+
page = doc.pages[0]
|
| 75 |
+
assert page.width == 2480
|
| 76 |
+
assert page.height == 3508
|
| 77 |
+
|
| 78 |
+
def test_correct_word_count(
|
| 79 |
+
self,
|
| 80 |
+
adapter: WordBoxJsonAdapter,
|
| 81 |
+
raw: RawProviderPayload,
|
| 82 |
+
geo_ctx: GeometryContext,
|
| 83 |
+
) -> None:
|
| 84 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 85 |
+
page = doc.pages[0]
|
| 86 |
+
# 5 items in fixture → 5 lines, each with 1 word
|
| 87 |
+
total_words = sum(
|
| 88 |
+
len(line.words)
|
| 89 |
+
for region in page.text_regions
|
| 90 |
+
for line in region.lines
|
| 91 |
+
)
|
| 92 |
+
assert total_words == 5
|
| 93 |
+
|
| 94 |
+
def test_word_text_preserved(
|
| 95 |
+
self,
|
| 96 |
+
adapter: WordBoxJsonAdapter,
|
| 97 |
+
raw: RawProviderPayload,
|
| 98 |
+
geo_ctx: GeometryContext,
|
| 99 |
+
) -> None:
|
| 100 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 101 |
+
words = [
|
| 102 |
+
w.text
|
| 103 |
+
for r in doc.pages[0].text_regions
|
| 104 |
+
for l in r.lines
|
| 105 |
+
for w in l.words
|
| 106 |
+
]
|
| 107 |
+
assert words[0] == "Bonjour"
|
| 108 |
+
assert words[1] == "le"
|
| 109 |
+
assert words[2] == "monde"
|
| 110 |
+
|
| 111 |
+
def test_confidence_preserved(
|
| 112 |
+
self,
|
| 113 |
+
adapter: WordBoxJsonAdapter,
|
| 114 |
+
raw: RawProviderPayload,
|
| 115 |
+
geo_ctx: GeometryContext,
|
| 116 |
+
) -> None:
|
| 117 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 118 |
+
first_word = doc.pages[0].text_regions[0].lines[0].words[0]
|
| 119 |
+
assert first_word.confidence == pytest.approx(0.96)
|
| 120 |
+
|
| 121 |
+
def test_geometry_is_exact(
|
| 122 |
+
self,
|
| 123 |
+
adapter: WordBoxJsonAdapter,
|
| 124 |
+
raw: RawProviderPayload,
|
| 125 |
+
geo_ctx: GeometryContext,
|
| 126 |
+
) -> None:
|
| 127 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 128 |
+
first_word = doc.pages[0].text_regions[0].lines[0].words[0]
|
| 129 |
+
assert first_word.geometry.status == GeometryStatus.EXACT
|
| 130 |
+
|
| 131 |
+
def test_polygon_preserved(
|
| 132 |
+
self,
|
| 133 |
+
adapter: WordBoxJsonAdapter,
|
| 134 |
+
raw: RawProviderPayload,
|
| 135 |
+
geo_ctx: GeometryContext,
|
| 136 |
+
) -> None:
|
| 137 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 138 |
+
first_word = doc.pages[0].text_regions[0].lines[0].words[0]
|
| 139 |
+
assert first_word.geometry.polygon is not None
|
| 140 |
+
assert len(first_word.geometry.polygon) == 4
|
| 141 |
+
|
| 142 |
+
def test_bbox_correct(
|
| 143 |
+
self,
|
| 144 |
+
adapter: WordBoxJsonAdapter,
|
| 145 |
+
raw: RawProviderPayload,
|
| 146 |
+
geo_ctx: GeometryContext,
|
| 147 |
+
) -> None:
|
| 148 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 149 |
+
first_word = doc.pages[0].text_regions[0].lines[0].words[0]
|
| 150 |
+
# From fixture: [[100,200],[300,200],[300,240],[100,240]]
|
| 151 |
+
# → bbox = (100, 200, 200, 40)
|
| 152 |
+
x, y, w, h = first_word.geometry.bbox
|
| 153 |
+
assert x == pytest.approx(100)
|
| 154 |
+
assert y == pytest.approx(200)
|
| 155 |
+
assert w == pytest.approx(200)
|
| 156 |
+
assert h == pytest.approx(40)
|
| 157 |
+
|
| 158 |
+
def test_provenance_native(
|
| 159 |
+
self,
|
| 160 |
+
adapter: WordBoxJsonAdapter,
|
| 161 |
+
raw: RawProviderPayload,
|
| 162 |
+
geo_ctx: GeometryContext,
|
| 163 |
+
) -> None:
|
| 164 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 165 |
+
first_word = doc.pages[0].text_regions[0].lines[0].words[0]
|
| 166 |
+
assert first_word.provenance.evidence_type == EvidenceType.PROVIDER_NATIVE
|
| 167 |
+
assert first_word.provenance.provider == "paddleocr"
|
| 168 |
+
assert "$[0]" in first_word.provenance.source_ref
|
| 169 |
+
|
| 170 |
+
def test_block_is_inferred(
|
| 171 |
+
self,
|
| 172 |
+
adapter: WordBoxJsonAdapter,
|
| 173 |
+
raw: RawProviderPayload,
|
| 174 |
+
geo_ctx: GeometryContext,
|
| 175 |
+
) -> None:
|
| 176 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 177 |
+
region = doc.pages[0].text_regions[0]
|
| 178 |
+
assert region.geometry.status == GeometryStatus.INFERRED
|
| 179 |
+
assert region.provenance.evidence_type == EvidenceType.DERIVED
|
| 180 |
+
|
| 181 |
+
def test_reading_order(
|
| 182 |
+
self,
|
| 183 |
+
adapter: WordBoxJsonAdapter,
|
| 184 |
+
raw: RawProviderPayload,
|
| 185 |
+
geo_ctx: GeometryContext,
|
| 186 |
+
) -> None:
|
| 187 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 188 |
+
assert doc.pages[0].reading_order == ["tb1"]
|
| 189 |
+
|
| 190 |
+
def test_empty_payload_rejected(
|
| 191 |
+
self,
|
| 192 |
+
adapter: WordBoxJsonAdapter,
|
| 193 |
+
geo_ctx: GeometryContext,
|
| 194 |
+
) -> None:
|
| 195 |
+
raw = RawProviderPayload(
|
| 196 |
+
provider_id="test",
|
| 197 |
+
adapter_id="v1",
|
| 198 |
+
runtime_type="local",
|
| 199 |
+
payload=[],
|
| 200 |
+
)
|
| 201 |
+
with pytest.raises(ValueError, match="no items"):
|
| 202 |
+
adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 203 |
+
|
| 204 |
+
def test_dict_payload_rejected(
|
| 205 |
+
self,
|
| 206 |
+
adapter: WordBoxJsonAdapter,
|
| 207 |
+
geo_ctx: GeometryContext,
|
| 208 |
+
) -> None:
|
| 209 |
+
raw = RawProviderPayload(
|
| 210 |
+
provider_id="test",
|
| 211 |
+
adapter_id="v1",
|
| 212 |
+
runtime_type="local",
|
| 213 |
+
payload={"not": "a list"},
|
| 214 |
+
)
|
| 215 |
+
with pytest.raises(ValueError, match="list payload"):
|
| 216 |
+
adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 217 |
+
|
| 218 |
+
def test_json_roundtrip(
|
| 219 |
+
self,
|
| 220 |
+
adapter: WordBoxJsonAdapter,
|
| 221 |
+
raw: RawProviderPayload,
|
| 222 |
+
geo_ctx: GeometryContext,
|
| 223 |
+
) -> None:
|
| 224 |
+
doc = adapter.normalize(raw, geo_ctx, document_id="test_doc")
|
| 225 |
+
data = doc.model_dump(mode="json")
|
| 226 |
+
from src.app.domain.models import CanonicalDocument
|
| 227 |
+
|
| 228 |
+
doc2 = CanonicalDocument.model_validate(data)
|
| 229 |
+
assert doc2.document_id == doc.document_id
|
|
@@ -0,0 +1,285 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the ALTO XML serializer."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from lxml import etree
|
| 6 |
+
|
| 7 |
+
from src.app.domain.models import (
|
| 8 |
+
AltoReadiness,
|
| 9 |
+
CanonicalDocument,
|
| 10 |
+
EvidenceType,
|
| 11 |
+
Geometry,
|
| 12 |
+
GeometryStatus,
|
| 13 |
+
Hyphenation,
|
| 14 |
+
Page,
|
| 15 |
+
PageXmlReadiness,
|
| 16 |
+
Provenance,
|
| 17 |
+
ReadinessLevel,
|
| 18 |
+
Source,
|
| 19 |
+
TextLine,
|
| 20 |
+
TextRegion,
|
| 21 |
+
Word,
|
| 22 |
+
)
|
| 23 |
+
from src.app.domain.models.status import BlockRole, InputType
|
| 24 |
+
from src.app.serializers.alto_xml import ALTO_NS, serialize_alto, serialize_alto_to_string
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _prov(ref: str = "$.test") -> Provenance:
|
| 28 |
+
return Provenance(
|
| 29 |
+
provider="test",
|
| 30 |
+
adapter="test.v1",
|
| 31 |
+
source_ref=ref,
|
| 32 |
+
evidence_type=EvidenceType.PROVIDER_NATIVE,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _geo(x: float = 100, y: float = 200, w: float = 300, h: float = 50) -> Geometry:
|
| 37 |
+
return Geometry(bbox=(x, y, w, h), status=GeometryStatus.EXACT)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _simple_doc() -> CanonicalDocument:
|
| 41 |
+
"""A simple one-page document with one block, one line, two words."""
|
| 42 |
+
return CanonicalDocument(
|
| 43 |
+
document_id="doc_alto_test",
|
| 44 |
+
source=Source(input_type=InputType.IMAGE, filename="test.png"),
|
| 45 |
+
pages=[
|
| 46 |
+
Page(
|
| 47 |
+
id="p1",
|
| 48 |
+
page_index=0,
|
| 49 |
+
width=2480,
|
| 50 |
+
height=3508,
|
| 51 |
+
alto_readiness=AltoReadiness(level=ReadinessLevel.FULL),
|
| 52 |
+
page_readiness=PageXmlReadiness(level=ReadinessLevel.FULL),
|
| 53 |
+
reading_order=["tb1"],
|
| 54 |
+
text_regions=[
|
| 55 |
+
TextRegion(
|
| 56 |
+
id="tb1",
|
| 57 |
+
role=BlockRole.BODY,
|
| 58 |
+
geometry=_geo(100, 200, 1200, 900),
|
| 59 |
+
lang="fra",
|
| 60 |
+
provenance=_prov(),
|
| 61 |
+
lines=[
|
| 62 |
+
TextLine(
|
| 63 |
+
id="tl1",
|
| 64 |
+
geometry=_geo(110, 220, 1100, 42),
|
| 65 |
+
lang="fra",
|
| 66 |
+
provenance=_prov(),
|
| 67 |
+
words=[
|
| 68 |
+
Word(
|
| 69 |
+
id="w1",
|
| 70 |
+
text="Bonjour",
|
| 71 |
+
geometry=_geo(110, 220, 90, 40),
|
| 72 |
+
lang="fra",
|
| 73 |
+
confidence=0.96,
|
| 74 |
+
provenance=_prov(),
|
| 75 |
+
),
|
| 76 |
+
Word(
|
| 77 |
+
id="w2",
|
| 78 |
+
text="monde",
|
| 79 |
+
geometry=_geo(220, 220, 80, 40),
|
| 80 |
+
lang="fra",
|
| 81 |
+
confidence=0.94,
|
| 82 |
+
provenance=_prov(),
|
| 83 |
+
),
|
| 84 |
+
],
|
| 85 |
+
),
|
| 86 |
+
],
|
| 87 |
+
),
|
| 88 |
+
],
|
| 89 |
+
),
|
| 90 |
+
],
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class TestAltoSerialization:
|
| 95 |
+
def test_produces_valid_xml(self) -> None:
|
| 96 |
+
doc = _simple_doc()
|
| 97 |
+
xml_bytes = serialize_alto(doc)
|
| 98 |
+
# Should parse without error
|
| 99 |
+
root = etree.fromstring(xml_bytes)
|
| 100 |
+
assert root.tag == f"{{{ALTO_NS}}}alto"
|
| 101 |
+
|
| 102 |
+
def test_has_description(self) -> None:
|
| 103 |
+
doc = _simple_doc()
|
| 104 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 105 |
+
desc = root.find(f"{{{ALTO_NS}}}Description")
|
| 106 |
+
assert desc is not None
|
| 107 |
+
|
| 108 |
+
def test_has_layout(self) -> None:
|
| 109 |
+
doc = _simple_doc()
|
| 110 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 111 |
+
layout = root.find(f"{{{ALTO_NS}}}Layout")
|
| 112 |
+
assert layout is not None
|
| 113 |
+
|
| 114 |
+
def test_page_attributes(self) -> None:
|
| 115 |
+
doc = _simple_doc()
|
| 116 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 117 |
+
page = root.find(f".//{{{ALTO_NS}}}Page")
|
| 118 |
+
assert page is not None
|
| 119 |
+
assert page.get("ID") == "p1"
|
| 120 |
+
assert page.get("WIDTH") == "2480"
|
| 121 |
+
assert page.get("HEIGHT") == "3508"
|
| 122 |
+
|
| 123 |
+
def test_text_block_exists(self) -> None:
|
| 124 |
+
doc = _simple_doc()
|
| 125 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 126 |
+
tb = root.find(f".//{{{ALTO_NS}}}TextBlock")
|
| 127 |
+
assert tb is not None
|
| 128 |
+
assert tb.get("ID") == "tb1"
|
| 129 |
+
|
| 130 |
+
def test_text_block_bbox(self) -> None:
|
| 131 |
+
doc = _simple_doc()
|
| 132 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 133 |
+
tb = root.find(f".//{{{ALTO_NS}}}TextBlock")
|
| 134 |
+
assert tb.get("HPOS") == "100"
|
| 135 |
+
assert tb.get("VPOS") == "200"
|
| 136 |
+
assert tb.get("WIDTH") == "1200"
|
| 137 |
+
assert tb.get("HEIGHT") == "900"
|
| 138 |
+
|
| 139 |
+
def test_text_line_exists(self) -> None:
|
| 140 |
+
doc = _simple_doc()
|
| 141 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 142 |
+
tl = root.find(f".//{{{ALTO_NS}}}TextLine")
|
| 143 |
+
assert tl is not None
|
| 144 |
+
assert tl.get("ID") == "tl1"
|
| 145 |
+
|
| 146 |
+
def test_strings_exist(self) -> None:
|
| 147 |
+
doc = _simple_doc()
|
| 148 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 149 |
+
strings = root.findall(f".//{{{ALTO_NS}}}String")
|
| 150 |
+
assert len(strings) == 2
|
| 151 |
+
|
| 152 |
+
def test_string_content(self) -> None:
|
| 153 |
+
doc = _simple_doc()
|
| 154 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 155 |
+
strings = root.findall(f".//{{{ALTO_NS}}}String")
|
| 156 |
+
assert strings[0].get("CONTENT") == "Bonjour"
|
| 157 |
+
assert strings[1].get("CONTENT") == "monde"
|
| 158 |
+
|
| 159 |
+
def test_string_bbox(self) -> None:
|
| 160 |
+
doc = _simple_doc()
|
| 161 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 162 |
+
s = root.findall(f".//{{{ALTO_NS}}}String")[0]
|
| 163 |
+
assert s.get("HPOS") == "110"
|
| 164 |
+
assert s.get("VPOS") == "220"
|
| 165 |
+
assert s.get("WIDTH") == "90"
|
| 166 |
+
assert s.get("HEIGHT") == "40"
|
| 167 |
+
|
| 168 |
+
def test_string_confidence(self) -> None:
|
| 169 |
+
doc = _simple_doc()
|
| 170 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 171 |
+
s = root.findall(f".//{{{ALTO_NS}}}String")[0]
|
| 172 |
+
assert s.get("WC") == "0.96"
|
| 173 |
+
|
| 174 |
+
def test_string_lang(self) -> None:
|
| 175 |
+
doc = _simple_doc()
|
| 176 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 177 |
+
tb = root.find(f".//{{{ALTO_NS}}}TextBlock")
|
| 178 |
+
assert tb.get("LANG") == "fra"
|
| 179 |
+
|
| 180 |
+
def test_sp_between_words(self) -> None:
|
| 181 |
+
doc = _simple_doc()
|
| 182 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 183 |
+
tl = root.find(f".//{{{ALTO_NS}}}TextLine")
|
| 184 |
+
children = list(tl)
|
| 185 |
+
# Should be: String, SP, String
|
| 186 |
+
tags = [c.tag.split("}")[-1] for c in children]
|
| 187 |
+
assert tags == ["String", "SP", "String"]
|
| 188 |
+
|
| 189 |
+
def test_measurement_unit(self) -> None:
|
| 190 |
+
doc = _simple_doc()
|
| 191 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 192 |
+
mu = root.find(f".//{{{ALTO_NS}}}MeasurementUnit")
|
| 193 |
+
assert mu is not None
|
| 194 |
+
assert mu.text == "pixel"
|
| 195 |
+
|
| 196 |
+
def test_filename_in_description(self) -> None:
|
| 197 |
+
doc = _simple_doc()
|
| 198 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 199 |
+
fn = root.find(f".//{{{ALTO_NS}}}fileName")
|
| 200 |
+
assert fn is not None
|
| 201 |
+
assert fn.text == "test.png"
|
| 202 |
+
|
| 203 |
+
def test_string_output(self) -> None:
|
| 204 |
+
doc = _simple_doc()
|
| 205 |
+
xml_str = serialize_alto_to_string(doc)
|
| 206 |
+
assert '<?xml version=' in xml_str
|
| 207 |
+
assert "Bonjour" in xml_str
|
| 208 |
+
assert "monde" in xml_str
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class TestAltoHyphenation:
|
| 212 |
+
def test_hyphenated_words(self) -> None:
|
| 213 |
+
doc = CanonicalDocument(
|
| 214 |
+
document_id="doc_hyph",
|
| 215 |
+
source=Source(input_type=InputType.IMAGE),
|
| 216 |
+
pages=[
|
| 217 |
+
Page(
|
| 218 |
+
id="p1",
|
| 219 |
+
page_index=0,
|
| 220 |
+
width=1000,
|
| 221 |
+
height=1000,
|
| 222 |
+
alto_readiness=AltoReadiness(level=ReadinessLevel.FULL),
|
| 223 |
+
page_readiness=PageXmlReadiness(level=ReadinessLevel.FULL),
|
| 224 |
+
text_regions=[
|
| 225 |
+
TextRegion(
|
| 226 |
+
id="tb1",
|
| 227 |
+
geometry=_geo(0, 0, 1000, 500),
|
| 228 |
+
provenance=_prov(),
|
| 229 |
+
lines=[
|
| 230 |
+
TextLine(
|
| 231 |
+
id="tl1",
|
| 232 |
+
geometry=_geo(0, 0, 500, 40),
|
| 233 |
+
provenance=_prov(),
|
| 234 |
+
words=[
|
| 235 |
+
Word(
|
| 236 |
+
id="w1",
|
| 237 |
+
text="patri-",
|
| 238 |
+
geometry=_geo(0, 0, 80, 30),
|
| 239 |
+
provenance=_prov(),
|
| 240 |
+
hyphenation=Hyphenation(
|
| 241 |
+
is_hyphenated=True,
|
| 242 |
+
part=1,
|
| 243 |
+
full_form="patrimoine",
|
| 244 |
+
),
|
| 245 |
+
),
|
| 246 |
+
],
|
| 247 |
+
),
|
| 248 |
+
TextLine(
|
| 249 |
+
id="tl2",
|
| 250 |
+
geometry=_geo(0, 50, 500, 40),
|
| 251 |
+
provenance=_prov(),
|
| 252 |
+
words=[
|
| 253 |
+
Word(
|
| 254 |
+
id="w2",
|
| 255 |
+
text="moine",
|
| 256 |
+
geometry=_geo(0, 50, 70, 30),
|
| 257 |
+
provenance=_prov(),
|
| 258 |
+
hyphenation=Hyphenation(
|
| 259 |
+
is_hyphenated=True,
|
| 260 |
+
part=2,
|
| 261 |
+
full_form="patrimoine",
|
| 262 |
+
),
|
| 263 |
+
),
|
| 264 |
+
],
|
| 265 |
+
),
|
| 266 |
+
],
|
| 267 |
+
),
|
| 268 |
+
],
|
| 269 |
+
),
|
| 270 |
+
],
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
root = etree.fromstring(serialize_alto(doc))
|
| 274 |
+
strings = root.findall(f".//{{{ALTO_NS}}}String")
|
| 275 |
+
assert len(strings) == 2
|
| 276 |
+
|
| 277 |
+
s1 = strings[0]
|
| 278 |
+
assert s1.get("CONTENT") == "patri-"
|
| 279 |
+
assert s1.get("SUBS_TYPE") == "HypPart1"
|
| 280 |
+
assert s1.get("SUBS_CONTENT") == "patrimoine"
|
| 281 |
+
|
| 282 |
+
s2 = strings[1]
|
| 283 |
+
assert s2.get("CONTENT") == "moine"
|
| 284 |
+
assert s2.get("SUBS_TYPE") == "HypPart2"
|
| 285 |
+
assert s2.get("SUBS_CONTENT") == "patrimoine"
|
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the CanonicalBuilder."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import pytest
|
| 6 |
+
|
| 7 |
+
from src.app.domain.models import (
|
| 8 |
+
EvidenceType,
|
| 9 |
+
Geometry,
|
| 10 |
+
GeometryStatus,
|
| 11 |
+
Provenance,
|
| 12 |
+
)
|
| 13 |
+
from src.app.domain.models.status import BlockRole, InputType
|
| 14 |
+
from src.app.normalization.canonical_builder import CanonicalBuilder
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _geo() -> Geometry:
|
| 18 |
+
return Geometry(bbox=(100, 200, 300, 50), status=GeometryStatus.EXACT)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _prov(ref: str = "$.test") -> Provenance:
|
| 22 |
+
return Provenance(
|
| 23 |
+
provider="test",
|
| 24 |
+
adapter="test.v1",
|
| 25 |
+
source_ref=ref,
|
| 26 |
+
evidence_type=EvidenceType.PROVIDER_NATIVE,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class TestCanonicalBuilder:
|
| 31 |
+
def test_minimal_document(self) -> None:
|
| 32 |
+
builder = CanonicalBuilder("doc1", InputType.IMAGE, "test.png")
|
| 33 |
+
page = builder.add_page("p1", 0, 2480, 3508)
|
| 34 |
+
region = page.add_text_region("tb1", _geo(), _prov(), role=BlockRole.BODY)
|
| 35 |
+
line = region.add_line("tl1", _geo(), _prov())
|
| 36 |
+
line.add_word("w1", "Hello", _geo(), _prov(), confidence=0.95)
|
| 37 |
+
|
| 38 |
+
doc = builder.build()
|
| 39 |
+
assert doc.document_id == "doc1"
|
| 40 |
+
assert len(doc.pages) == 1
|
| 41 |
+
assert len(doc.pages[0].text_regions) == 1
|
| 42 |
+
assert doc.pages[0].text_regions[0].lines[0].words[0].text == "Hello"
|
| 43 |
+
|
| 44 |
+
def test_reading_order_auto_built(self) -> None:
|
| 45 |
+
builder = CanonicalBuilder("doc1", InputType.IMAGE)
|
| 46 |
+
page = builder.add_page("p1", 0, 100, 100)
|
| 47 |
+
page.add_text_region("tb1", _geo(), _prov()).\
|
| 48 |
+
add_line("tl1", _geo(), _prov()).\
|
| 49 |
+
add_word("w1", "A", _geo(), _prov())
|
| 50 |
+
page.add_text_region("tb2", _geo(), _prov()).\
|
| 51 |
+
add_line("tl2", _geo(), _prov()).\
|
| 52 |
+
add_word("w2", "B", _geo(), _prov())
|
| 53 |
+
|
| 54 |
+
doc = builder.build()
|
| 55 |
+
assert doc.pages[0].reading_order == ["tb1", "tb2"]
|
| 56 |
+
|
| 57 |
+
def test_no_pages_raises(self) -> None:
|
| 58 |
+
builder = CanonicalBuilder("doc1", InputType.IMAGE)
|
| 59 |
+
with pytest.raises(ValueError, match="at least one page"):
|
| 60 |
+
builder.build()
|
| 61 |
+
|
| 62 |
+
def test_empty_region_raises(self) -> None:
|
| 63 |
+
builder = CanonicalBuilder("doc1", InputType.IMAGE)
|
| 64 |
+
page = builder.add_page("p1", 0, 100, 100)
|
| 65 |
+
page.add_text_region("tb1", _geo(), _prov())
|
| 66 |
+
with pytest.raises(ValueError, match="no lines"):
|
| 67 |
+
builder.build()
|
| 68 |
+
|
| 69 |
+
def test_empty_line_raises(self) -> None:
|
| 70 |
+
builder = CanonicalBuilder("doc1", InputType.IMAGE)
|
| 71 |
+
page = builder.add_page("p1", 0, 100, 100)
|
| 72 |
+
region = page.add_text_region("tb1", _geo(), _prov())
|
| 73 |
+
region.add_line("tl1", _geo(), _prov())
|
| 74 |
+
with pytest.raises(ValueError, match="no words"):
|
| 75 |
+
builder.build()
|
| 76 |
+
|
| 77 |
+
def test_multiple_words_per_line(self) -> None:
|
| 78 |
+
builder = CanonicalBuilder("doc1", InputType.IMAGE)
|
| 79 |
+
page = builder.add_page("p1", 0, 1000, 1000)
|
| 80 |
+
region = page.add_text_region("tb1", _geo(), _prov())
|
| 81 |
+
line = region.add_line("tl1", _geo(), _prov())
|
| 82 |
+
line.add_word("w1", "Hello", _geo(), _prov())
|
| 83 |
+
line.add_word("w2", "world", _geo(), _prov())
|
| 84 |
+
|
| 85 |
+
doc = builder.build()
|
| 86 |
+
assert len(doc.pages[0].text_regions[0].lines[0].words) == 2
|
| 87 |
+
assert doc.pages[0].text_regions[0].lines[0].text == "Hello world"
|
| 88 |
+
|
| 89 |
+
def test_with_metadata(self) -> None:
|
| 90 |
+
builder = CanonicalBuilder(
|
| 91 |
+
"doc1", InputType.IMAGE, metadata={"project": "test"}
|
| 92 |
+
)
|
| 93 |
+
page = builder.add_page("p1", 0, 100, 100)
|
| 94 |
+
page.set_metadata({"page_quality": "good"})
|
| 95 |
+
region = page.add_text_region("tb1", _geo(), _prov())
|
| 96 |
+
line = region.add_line("tl1", _geo(), _prov())
|
| 97 |
+
line.add_word("w1", "ok", _geo(), _prov(), metadata={"custom": True})
|
| 98 |
+
|
| 99 |
+
doc = builder.build()
|
| 100 |
+
assert doc.metadata == {"project": "test"}
|
| 101 |
+
assert doc.pages[0].metadata == {"page_quality": "good"}
|
| 102 |
+
assert doc.pages[0].text_regions[0].lines[0].words[0].metadata == {"custom": True}
|