Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions test_unstructured/partition/html/test_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,43 @@ def test_partition_html_processes_chinese_chracters():
assert elements[0].text == "每日新闻"


def test_partition_html_preserves_numeric_table_cell_text_in_chunked_html():
html_text = """
<html>
<body>
<h1>Financial metrics</h1>
<table>
<tr><th>metric</th><th>value</th></tr>
<tr><td>revenue</td><td>478923</td></tr>
<tr><td>shares</td><td>1234567</td></tr>
<tr><td>ticker</td><td>000001</td></tr>
<tr><td>ratio</td><td>0.000123</td></tr>
<tr><td>market cap</td><td>999999999999999999</td></tr>
</table>
</body>
</html>
"""

elements = partition_html(
text=html_text,
chunking_strategy="by_title",
max_characters=80,
new_after_n_chars=80,
)

table_html = "".join(
e.metadata.text_as_html or "" for e in elements if isinstance(e, TableChunk)
)

assert "<td>478923</td>" in table_html
assert "<td>1234567</td>" in table_html
assert "<td>000001</td>" in table_html
assert "<td>0.000123</td>" in table_html
assert "<td>999999999999999999</td>" in table_html
assert "e+" not in table_html.lower()
assert "e-" not in table_html.lower()


def test_emoji_appears_with_emoji_utf8_code():
assert partition_html(text='<html charset="utf-8"><p>Hello &#128512;</p></html>') == [
Text("Hello 😀")
Expand Down
26 changes: 26 additions & 0 deletions test_unstructured/partition/test_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,32 @@ def test_partition_csv_from_file_with_metadata_filename():
assert elements[0].metadata.filename == "test"


def test_partition_csv_preserves_numeric_cell_text_in_table_html(tmp_path):
file_path = tmp_path / "financial-metrics.csv"
file_path.write_text(
"metric,value\n"
"revenue,478923\n"
"shares,1234567\n"
"ticker,000001\n"
"ratio,0.000123\n"
"market cap,999999999999999999\n",
encoding="utf-8",
)

table = partition_csv(filename=str(file_path))[0]

assert table.metadata.text_as_html == (
"<table>"
"<tr><td>metric</td><td>value</td></tr>"
"<tr><td>revenue</td><td>478923</td></tr>"
"<tr><td>shares</td><td>1234567</td></tr>"
"<tr><td>ticker</td><td>000001</td></tr>"
"<tr><td>ratio</td><td>0.000123</td></tr>"
"<tr><td>market cap</td><td>999999999999999999</td></tr>"
"</table>"
)


# -- .metadata.last_modified ---------------------------------------------------------------------


Expand Down
26 changes: 26 additions & 0 deletions test_unstructured/partition/test_tsv.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,32 @@ def test_partition_tsv_from_file_with_metadata_filename():
assert all(element.metadata.filename == "test" for element in elements)


def test_partition_tsv_preserves_numeric_cell_text_in_table_html(tmp_path):
file_path = tmp_path / "financial-metrics.tsv"
file_path.write_text(
"metric\tvalue\n"
"revenue\t478923\n"
"shares\t1234567\n"
"ticker\t000001\n"
"ratio\t0.000123\n"
"market cap\t999999999999999999\n",
encoding="utf-8",
)

table = partition_tsv(filename=str(file_path), include_header=False)[0]

assert table.metadata.text_as_html == (
"<table>"
"<tr><td>metric</td><td>value</td></tr>"
"<tr><td>revenue</td><td>478923</td></tr>"
"<tr><td>shares</td><td>1234567</td></tr>"
"<tr><td>ticker</td><td>000001</td></tr>"
"<tr><td>ratio</td><td>0.000123</td></tr>"
"<tr><td>market cap</td><td>999999999999999999</td></tr>"
"</table>"
)


# -- .metadata.last_modified ---------------------------------------------------------------------


Expand Down
21 changes: 16 additions & 5 deletions unstructured/partition/csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import pandas as pd

from unstructured.chunking import add_chunking_strategy
from unstructured.common.html_table import HtmlTable
from unstructured.common.html_table import HtmlTable, htmlify_matrix_of_cell_texts
from unstructured.documents.elements import Element, ElementMetadata, Table
from unstructured.file_utils.model import FileType
from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
Expand Down Expand Up @@ -58,15 +58,19 @@ def partition_csv(

csv.field_size_limit(CSV_FIELD_LIMIT)
with ctx.open() as file:
read_kw: dict = {"header": ctx.header, "sep": ctx.delimiter, "encoding": ctx.encoding}
read_kw: dict = {
"header": ctx.header,
"sep": ctx.delimiter,
"encoding": ctx.encoding,
"dtype": str,
"keep_default_na": False,
}
# sep=None is not supported by the C engine; use Python engine to avoid ParserWarning.
if ctx.delimiter is None:
read_kw["engine"] = "python"
dataframe = pd.read_csv(file, **read_kw)

html_table = HtmlTable.from_html_text(
dataframe.to_html(index=False, header=include_header, na_rep="")
)
html_table = HtmlTable.from_html_text(_dataframe_to_html_text(dataframe, include_header))

metadata = ElementMetadata(
filename=filename,
Expand All @@ -78,6 +82,13 @@ def partition_csv(
return [Table(text=html_table.text, metadata=metadata, detection_origin=DETECTION_ORIGIN)]


def _dataframe_to_html_text(dataframe: pd.DataFrame, include_header: bool) -> str:
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Outdated
"""Render a dataframe as table HTML without pandas numeric formatting."""
rows = dataframe.astype(str).values.tolist()
matrix = [[str(column) for column in dataframe.columns]] + rows if include_header else rows
return htmlify_matrix_of_cell_texts(matrix)


class _CsvPartitioningContext:
"""Encapsulates the partitioning-run details.

Expand Down
23 changes: 17 additions & 6 deletions unstructured/partition/tsv.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pandas as pd

from unstructured.chunking import add_chunking_strategy
from unstructured.common.html_table import HtmlTable
from unstructured.common.html_table import HtmlTable, htmlify_matrix_of_cell_texts
from unstructured.documents.elements import Element, ElementMetadata, Table
from unstructured.file_utils.model import FileType
from unstructured.partition.common.common import (
Expand Down Expand Up @@ -41,18 +41,22 @@ def partition_tsv(

header = 0 if include_header else None

read_kw: dict[str, Any] = {
"sep": "\t",
"header": header,
"dtype": str,
"keep_default_na": False,
}
if filename:
dataframe = pd.read_csv(filename, sep="\t", header=header)
dataframe = pd.read_csv(filename, **read_kw)
else:
assert file is not None
# -- Note(scanny): `SpooledTemporaryFile` on Python<3.11 does not implement `.readable()`
# -- which triggers an exception on `pd.DataFrame.read_csv()` call.
f = spooled_to_bytes_io_if_needed(file)
dataframe = pd.read_csv(f, sep="\t", header=header)
dataframe = pd.read_csv(f, **read_kw)

html_table = HtmlTable.from_html_text(
dataframe.to_html(index=False, header=include_header, na_rep="")
)
html_table = HtmlTable.from_html_text(_dataframe_to_html_text(dataframe, include_header))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: Custom agent: Enforce Pragmatic Test Coverage

Change to TSV partition core logic lacks test coverage

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. At unstructured/partition/tsv.py, line 59:

<comment>Change to TSV partition core logic lacks test coverage</comment>

<file context>
@@ -41,18 +41,22 @@ def partition_tsv(
-    html_table = HtmlTable.from_html_text(
-        dataframe.to_html(index=False, header=include_header, na_rep="")
-    )
+    html_table = HtmlTable.from_html_text(_dataframe_to_html_text(dataframe, include_header))
 
     metadata = ElementMetadata(
</file context>


metadata = ElementMetadata(
filename=filename,
Expand All @@ -62,3 +66,10 @@ def partition_tsv(
metadata.detection_origin = DETECTION_ORIGIN

return [Table(text=html_table.text, metadata=metadata)]


def _dataframe_to_html_text(dataframe: pd.DataFrame, include_header: bool) -> str:
"""Render a dataframe as table HTML without pandas numeric formatting."""
rows = dataframe.astype(str).values.tolist()
matrix = [[str(column) for column in dataframe.columns]] + rows if include_header else rows
return htmlify_matrix_of_cell_texts(matrix)