From 4e80ca29867121147e2abf7da423c5b20ef8d5b2 Mon Sep 17 00:00:00 2001 From: Nishchay Mahor Date: Sat, 4 Jul 2026 23:43:54 -0700 Subject: [PATCH] fix: map long-s-t ligature (U+FB05) to 'st' not 'ft' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clean_ligatures mapped U+FB05 (LATIN SMALL LIGATURE LONG S T, the long-s + t glyph 'ſt') to 'ft', which corrupted real text: words like last/first/best typeset with that ligature became laft/firft/beft. Unicode NFKC normalizes U+FB05 to 'st', and the adjacent U+FB06 ('st') is already mapped to 'st' in the same table. Map U+FB05 to 'st' and fix the test case that encoded the old behavior (the prior example used the st-ligature to spell 'craftsman', which is not what the glyph represents). --- test_unstructured/cleaners/test_core.py | 2 +- unstructured/cleaners/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_unstructured/cleaners/test_core.py b/test_unstructured/cleaners/test_core.py index 6414292d99..212b5ce57b 100644 --- a/test_unstructured/cleaners/test_core.py +++ b/test_unstructured/cleaners/test_core.py @@ -75,7 +75,7 @@ def test_clean_ordered_bullets(text, expected): ("She had a flower in her hair.", "She had a flower in her hair."), ("The coffin was placed in the grave.", "The coffin was placed in the grave."), ("The buffle zone was clearly marked.", "The buffle zone was clearly marked."), - ("The craſtsman worked with dedication.", "The craftsman worked with dedication."), + ("The laſt chapter was the best.", "The last chapter was the best."), ("The symbol ʪ is very rare.", "The symbol ls is very rare."), ("The word 'cœur' means 'heart' in French.", "The word 'coeur' means 'heart' in French."), ("The word 'Œuvre' refers to the works", "The word 'OEuvre' refers to the works"), diff --git a/unstructured/cleaners/core.py b/unstructured/cleaners/core.py index b64c7bd19c..3698aec47e 100644 --- a/unstructured/cleaners/core.py +++ b/unstructured/cleaners/core.py @@ -89,7 +89,7 @@ def clean_ligatures(text) -> str: "fl": "fl", "ffi": "ffi", "ffl": "ffl", - "ſt": "ft", + "ſt": "st", # U+FB05 LATIN SMALL LIGATURE LONG S T -> "st" (per Unicode NFKC) "ʪ": "ls", "œ": "oe", "Œ": "OE",