fix: prefer embedded PDF text over OCR for hi_res table tokens

Author: claytonlin1110

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.22.27
+
+### Fixes
+
+- **Prefer embedded PDF text for hi-res table cell content**: When `partition_pdf(..., strategy="hi_res", infer_table_structure=True)` processes digital PDFs, table tokenization now uses PDFMiner extracted text inside table regions before OCR fallback. This prevents OCR substitutions in table HTML (for example, replacing `AUTOSAR Administration` with incorrect OCR output) while preserving OCR behavior for scanned tables.
+
 ## 0.22.26
 
 ### Enhancements

test_unstructured/partition/pdf_image/test_ocr.py

@@ -486,6 +486,48 @@ def test_get_table_tokens(mock_ocr_layout):
         assert table_tokens == expected_tokens
 
 
+def test_get_table_tokens_prefers_extracted_regions_over_ocr(mock_ocr_layout):
+    extracted_regions = TextRegions.from_list(
+        [
+            EmbeddedTextRegion.from_coords(
+                20, 30, 40, 50, text="AUTOSAR Administration", source=None
+            ),
+            EmbeddedTextRegion.from_coords(45, 30, 55, 50, text="2.1.0", source=None),
+        ]
+    )
+    with patch.object(OCRAgentTesseract, "get_layout_from_image", return_value=mock_ocr_layout):
+        ocr_agent = OCRAgent.get_agent(language="eng")
+        table_tokens = ocr.get_table_tokens(
+            table_element_image=Image.new("RGB", (80, 80)),
+            ocr_agent=ocr_agent,
+            extracted_regions=extracted_regions,
+            table_bbox=(10, 20, 70, 70),
+            padding=0,
+        )
+
+        assert [token["text"] for token in table_tokens] == ["AUTOSAR Administration", "2.1.0"]
+        assert table_tokens[0]["bbox"] == [10, 10, 30, 30]
+
+
+def test_get_table_tokens_falls_back_to_ocr_when_extracted_is_sparse(mock_ocr_layout):
+    extracted_regions = TextRegions.from_list(
+        [
+            EmbeddedTextRegion.from_coords(20, 30, 40, 50, text="only-one-token", source=None),
+        ]
+    )
+    with patch.object(OCRAgentTesseract, "get_layout_from_image", return_value=mock_ocr_layout):
+        ocr_agent = OCRAgent.get_agent(language="eng")
+        table_tokens = ocr.get_table_tokens(
+            table_element_image=Image.new("RGB", (80, 80)),
+            ocr_agent=ocr_agent,
+            extracted_regions=extracted_regions,
+            table_bbox=(10, 20, 70, 70),
+            padding=0,
+        )
+
+        assert [token["text"] for token in table_tokens] == ["Token1", "Token2"]
+
+
 def test_auto_zoom_not_exceed_tesseract_limit(monkeypatch):
     monkeypatch.setenv("TESSERACT_MIN_TEXT_HEIGHT", "1000")
     monkeypatch.setenv("TESSERACT_OPTIMUM_TEXT_HEIGHT", "100000")

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.22.26"  # pragma: no cover
+__version__ = "0.22.27"  # pragma: no cover

unstructured/partition/pdf_image/ocr.py

@@ -312,17 +312,26 @@ def supplement_element_with_table_extraction(
     table_elements = elements.slice(table_ele_indices)
     padding = env_config.TABLE_IMAGE_CROP_PAD
     for i, element_coords in enumerate(table_elements.element_coords):
+        table_bbox = (
+            float(element_coords[0]),
+            float(element_coords[1]),
+            float(element_coords[2]),
+            float(element_coords[3]),
+        )
         cropped_image = image.crop(
             (
-                element_coords[0] - padding,
-                element_coords[1] - padding,
-                element_coords[2] + padding,
-                element_coords[3] + padding,
+                table_bbox[0] - padding,
+                table_bbox[1] - padding,
+                table_bbox[2] + padding,
+                table_bbox[3] + padding,
             ),
         )
         table_tokens = get_table_tokens(
             table_element_image=cropped_image,
             ocr_agent=ocr_agent,
+            extracted_regions=extracted_regions,
+            table_bbox=table_bbox,
+            padding=padding,
         )
         tatr_cells = tables_agent.predict(
             cropped_image, ocr_tokens=table_tokens, result_format="cells"
@@ -344,13 +353,15 @@ def supplement_element_with_table_extraction(
 def get_table_tokens(
     table_element_image: PILImage.Image,
     ocr_agent: OCRAgent,
+    extracted_regions: Optional[TextRegions] = None,
+    table_bbox: Optional[tuple[float, float, float, float]] = None,
+    padding: float = 0,
 ) -> List[dict[str, Any]]:
-    """Get OCR tokens from either paddleocr or tesseract"""
-
+    """Get table tokens, preferring embedded PDF text when coverage is sufficient."""
     ocr_layout = ocr_agent.get_layout_from_image(image=table_element_image)
-    table_tokens = []
+    ocr_tokens = []
     for i, text in enumerate(ocr_layout.texts):
-        table_tokens.append(
+        ocr_tokens.append(
             {
                 "bbox": [
                     ocr_layout.x1[i],
@@ -367,6 +378,99 @@ def get_table_tokens(
             }
         )
 
+    if extracted_regions is None or table_bbox is None:
+        return ocr_tokens
+
+    extracted_tokens = _get_table_tokens_from_extracted_regions(
+        extracted_regions=extracted_regions,
+        table_bbox=table_bbox,
+        table_image_size=table_element_image.size,
+        padding=padding,
+    )
+    if _prefer_extracted_table_tokens(extracted_tokens, ocr_tokens):
+        return extracted_tokens
+
+    return ocr_tokens
+
+
+def _prefer_extracted_table_tokens(
+    extracted_tokens: List[dict[str, Any]],
+    ocr_tokens: List[dict[str, Any]],
+    token_ratio_threshold: float = 0.8,
+    text_ratio_threshold: float = 0.8,
+) -> bool:
+    """Choose extracted tokens only when they have comparable coverage to OCR."""
+    if not extracted_tokens:
+        return False
+    if not ocr_tokens:
+        return True
+
+    extracted_count = len(extracted_tokens)
+    ocr_count = len(ocr_tokens)
+    extracted_chars = sum(len(str(token.get("text", ""))) for token in extracted_tokens)
+    ocr_chars = sum(len(str(token.get("text", ""))) for token in ocr_tokens)
+
+    return (
+        extracted_count >= token_ratio_threshold * ocr_count
+        and extracted_chars >= text_ratio_threshold * ocr_chars
+    )
+
+
+def _get_table_tokens_from_extracted_regions(
+    extracted_regions: TextRegions,
+    table_bbox: tuple[float, float, float, float],
+    table_image_size: tuple[int, int],
+    padding: float,
+) -> List[dict[str, Any]]:
+    if len(extracted_regions) == 0:
+        return []
+
+    mask = (
+        bboxes1_is_almost_subregion_of_bboxes2(
+            extracted_regions.element_coords,
+            np.array([table_bbox]),
+            env_config.OCR_LAYOUT_SUBREGION_THRESHOLD,
+        )
+        .sum(axis=1)
+        .astype(bool)
+    )
+    if not np.any(mask):
+        return []
+
+    selected_regions = extracted_regions.slice(mask)
+    left = table_bbox[0] - padding
+    top = table_bbox[1] - padding
+    width, height = table_image_size
+
+    valid = [
+        (idx, text) for idx, text in enumerate(selected_regions.texts) if text and str(text).strip()
+    ]
+    if not valid:
+        return []
+
+    # Keep deterministic reading order (top-to-bottom then left-to-right).
+    sorted_indices = sorted(
+        valid,
+        key=lambda item: (selected_regions.y1[item[0]], selected_regions.x1[item[0]]),
+    )
+    table_tokens = []
+    for span_num, (idx, text) in enumerate(sorted_indices):
+        x1 = max(0, min(width, int(round(selected_regions.x1[idx] - left))))
+        y1 = max(0, min(height, int(round(selected_regions.y1[idx] - top))))
+        x2 = max(0, min(width, int(round(selected_regions.x2[idx] - left))))
+        y2 = max(0, min(height, int(round(selected_regions.y2[idx] - top))))
+        if x2 <= x1 or y2 <= y1:
+            continue
+        table_tokens.append(
+            {
+                "bbox": [x1, y1, x2, y2],
+                "text": str(text),
+                "span_num": span_num,
+                "line_num": 0,
+                "block_num": 0,
+            }
+        )
+
     return table_tokens