From dd3d837fcc153d2ab8e3355afa28f9c40d2cd04c Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 17:22:21 -0500 Subject: [PATCH 01/19] fix: stop duplicating merged-cell text in DOCX text_as_html DOCX table HTML built one per grid position that a gridSpan/vMerge merge visually covered, repeating that cell's text into each one with no colspan/rowspan attribute. Track the underlying tc element's identity across grid positions and collapse matching runs into a single carrying the appropriate colspan/rowspan, so merged-cell text appears exactly once and the merge geometry is recoverable from the HTML. --- test_unstructured/common/test_html_table.py | 83 ++++++++++++++ test_unstructured/partition/test_docx.py | 80 ++++++++++--- unstructured/common/html_table.py | 118 ++++++++++++++++++-- unstructured/partition/docx.py | 52 ++++++--- 4 files changed, 294 insertions(+), 39 deletions(-) diff --git a/test_unstructured/common/test_html_table.py b/test_unstructured/common/test_html_table.py index 50b37dc82d..3ad7cf389c 100644 --- a/test_unstructured/common/test_html_table.py +++ b/test_unstructured/common/test_html_table.py @@ -11,7 +11,9 @@ HtmlCell, HtmlRow, HtmlTable, + collapse_matrix_of_keyed_cells_to_spans, htmlify_matrix_of_cell_texts, + htmlify_matrix_of_spanned_cell_texts, ) @@ -43,6 +45,87 @@ def test_htmlify_matrix_handles_empty_matrix(self): assert htmlify_matrix_of_cell_texts([]) == "" +class Describe_htmlify_matrix_of_spanned_cell_texts: + """Unit-test suite for `html_table.htmlify_matrix_of_spanned_cell_texts()`.""" + + def it_emits_plain_cells_when_no_span_is_greater_than_1(self): + assert htmlify_matrix_of_spanned_cell_texts( + [[("a", 1, 1), ("b", 1, 1)], [("c", 1, 1), ("d", 1, 1)]] + ) == ("
ab
cd
") + + def it_emits_colspan_and_rowspan_attributes_only_when_greater_than_1(self): + assert htmlify_matrix_of_spanned_cell_texts([[("a", 2, 3)], [("b", 1, 1)]]) == ( + '
a
b
' + ) + + def it_emits_a_void_td_for_an_empty_spanned_cell(self): + assert htmlify_matrix_of_spanned_cell_texts([[("", 2, 1)]]) == ( + '
' + ) + + def it_suppresses_rows_with_no_cells(self): + assert htmlify_matrix_of_spanned_cell_texts([[("a", 1, 1)], []]) == ( + "
a
" + ) + + def it_handles_an_empty_matrix(self): + assert htmlify_matrix_of_spanned_cell_texts([]) == "" + + +class Describe_collapse_matrix_of_keyed_cells_to_spans: + """Unit-test suite for `html_table.collapse_matrix_of_keyed_cells_to_spans()`.""" + + def it_leaves_a_matrix_with_no_repeated_keys_unchanged(self): + matrix = [[("a", 1), ("b", 2)], [("c", 3), ("d", 4)]] + assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [ + [("a", 1, 1), ("b", 1, 1)], + [("c", 1, 1), ("d", 1, 1)], + ] + + def it_collapses_a_horizontal_run_of_matching_keys_into_a_colspan(self): + matrix = [[("a", 1), ("a", 1), ("b", 2)]] + assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [[("a", 2, 1), ("b", 1, 1)]] + + def it_collapses_a_vertical_run_of_matching_keys_into_a_rowspan(self): + matrix = [[("a", 1)], [("a", 1)], [("b", 2)]] + assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [ + [("a", 1, 2)], + [], + [("b", 1, 1)], + ] + + def it_collapses_a_rectangular_region_into_a_single_cell_with_colspan_and_rowspan(self): + """Reproduces the docx-tables.docx merged-cell fixture geometry. + + +---+-------+ + | a | b | + | +---+---+ + | | c | d | + +---+---+ | + | e | | + +-------+---+ + """ + matrix = [ + [("a", 1), ("b", 2), ("b", 2)], + [("a", 1), ("c", 3), ("d", 4)], + [("e", 5), ("e", 5), ("d", 4)], + ] + assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [ + [("a", 1, 2), ("b", 2, 1)], + [("c", 1, 1), ("d", 1, 2)], + [("e", 2, 1)], + ] + + def it_treats_distinct_keys_as_never_merged_even_when_their_text_matches(self): + matrix = [[("", 1), ("", 2)]] + assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [ + [("", 1, 1), ("", 1, 1)], + ] + + def it_handles_an_empty_matrix(self): + assert collapse_matrix_of_keyed_cells_to_spans([]) == [] + + class DescribeHtmlTable: """Unit-test suite for `unstructured.common.html_table.HtmlTable`.""" diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index 6911fefc52..1e1438b2ef 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -149,6 +149,35 @@ def test_partition_docx_processes_table(): assert elements[0].metadata.filename == "fake_table.docx" +def test_partition_docx_table_with_merged_cells_reports_spans_instead_of_duplicating_text(): + """A merged cell's text must appear in exactly one ``, marked with colspan/rowspan. + + Fixture table is: + + +---+-------+ + | a | b | + | +---+---+ + | | c | d | + +---+---+ | + | e | | + +-------+---+ + """ + elements = partition_docx(example_doc_path("docx-tables.docx"), infer_table_structure=True) + tables = [e for e in elements if isinstance(e, Table)] + table = next(t for t in tables if t.text == "a b c d e") + + assert table.metadata.text_as_html == ( + "" + '' + '' + '' + "
ab
cd
e
" + ) + # -- no cell's text is duplicated across more than one `` -- + for letter in "abcde": + assert table.metadata.text_as_html.count(f">{letter}<") == 1 + + def test_partition_docx_grabs_header_and_footer(): elements = partition_docx(example_doc_path("handbook-1p.docx")) @@ -1072,6 +1101,31 @@ def but_the_text_of_a_merged_cell_appears_only_once(self, opts_args: dict[str, A table = docx.Document(example_doc_path("docx-tables.docx")).tables[2] assert " ".join(_DocxPartitioner(opts)._iter_table_texts(table)) == "a b c d e" + def and_the_html_of_a_merged_cell_carries_colspan_and_rowspan_instead_of_repeating_text( + self, opts_args: dict[str, Any] + ): + """ + Fixture table is: + + +---+-------+ + | a | b | + | +---+---+ + | | c | d | + +---+---+ | + | e | | + +-------+---+ + """ + opts = DocxPartitionerOptions(**opts_args) + table = docx.Document(example_doc_path("docx-tables.docx")).tables[2] + + assert _DocxPartitioner(opts)._convert_table_to_html(table) == ( + "" + '' + '' + '' + "
ab
cd
e
" + ) + def it_can_partition_tables_with_incomplete_rows(self): """DOCX permits table rows to start late and end early. @@ -1128,7 +1182,7 @@ def it_can_partition_tables_with_incomplete_rows(self): assert e.text == "a b c d", f"actual {e.text=}" assert e.metadata.text_as_html == ( "" - "" + '' "" "
aa
a
bcd
" ), f"actual {e.metadata.text_as_html=}" @@ -1143,8 +1197,8 @@ def it_can_partition_tables_with_incomplete_rows(self): assert e.text == "a b c d", f"actual {e.text=}" assert e.metadata.text_as_html == ( "" - "" - "" + '' + "" "
ab
acd
ab
cd
" ), f"actual {e.metadata.text_as_html=}" # -- late-start, early-end, and >2 rows vertical span -- @@ -1162,10 +1216,10 @@ def it_can_partition_tables_with_incomplete_rows(self): assert e.text == "a b c d e f", f"actual {e.text=}" assert e.metadata.text_as_html == ( "" - "" - "" - "" - "" + '' + '' + "" + "" "
aabc
dd
eddf
dd
abc
d
ef
" ), f"actual {e.metadata.text_as_html=}" # -- @@ -1175,14 +1229,14 @@ def it_can_partition_tables_with_incomplete_rows(self): assert e.text == "Data More Dato WTF? Strange Format", f"actual {e.text=}" assert e.metadata.text_as_html == ( "" - "" - "" - "" + '' + "" + "" "" "" - "" - "" - "" + '' + '' + '' "
DataData
DataData
DataData
Data
More
Dato
WTF?WTF?
StrangeStrange
FormatFormat
WTF?
Strange
Format
" ), f"actual {e.metadata.text_as_html=}" diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index b0c366b8fb..56a11823db 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -11,10 +11,37 @@ from lxml import etree from lxml.html import fragment_fromstring +from typing_extensions import TypeAlias if TYPE_CHECKING: from lxml.html import HtmlElement +# -- (cell_text, colspan, rowspan) for one HTML `` -- +SpannedCell: TypeAlias = "tuple[str, int, int]" + + +def _format_td(cell_text: str, colspan: int = 1, rowspan: int = 1) -> str: + """Format a single `` element, escaping and normalizing `cell_text`. + + `colspan`/`rowspan` attributes are only emitted when greater than 1 (the implicit default), + to minimize character overhead in the common no-span case. + """ + # -- take care of things like '<' and '>' in the text -- + s = html.escape(cell_text) + # -- substitute
elements for line-feeds in the text -- + s = "
".join(s.split("\n")) + # -- normalize whitespace in cell -- + text = " ".join(s.split()) + + attrs = "" + if colspan > 1: + attrs += f' colspan="{colspan}"' + if rowspan > 1: + attrs += f' rowspan="{rowspan}"' + + # -- emit void `` when cell text is empty string -- + return f"{text}" if text else f"" + def htmlify_matrix_of_cell_texts(matrix: Sequence[Sequence[str]]) -> str: """Form an HTML table from "rows" and "columns" of `matrix`. @@ -32,22 +59,89 @@ def iter_trs(rows_of_cell_strs: Sequence[Sequence[str]]) -> Iterator[str]: # -- suppress emission of rows with no cells -- if not row_cell_strs: continue - yield f"{''.join(iter_tds(row_cell_strs))}" - - def iter_tds(row_cell_strs: Sequence[str]) -> Iterator[str]: - for s in row_cell_strs: - # -- take care of things like '<' and '>' in the text -- - s = html.escape(s) - # -- substitute
elements for line-feeds in the text -- - s = "
".join(s.split("\n")) - # -- normalize whitespace in cell -- - cell_text = " ".join(s.split()) - # -- emit void `` when cell text is empty string -- - yield f"{cell_text}" if cell_text else "" + yield f"{''.join(_format_td(s) for s in row_cell_strs)}" return f"{''.join(iter_trs(matrix))}
" if matrix else "" +def htmlify_matrix_of_spanned_cell_texts(matrix: Sequence[Sequence[SpannedCell]]) -> str: + """Like `htmlify_matrix_of_cell_texts()` but each cell can also carry a colspan/rowspan. + + Each row of `matrix` is a sequence of `(cell_text, colspan, rowspan)` triples, one for each + grid-position that is the top-left corner of a (possibly 1x1) cell. A grid-position covered by + the colspan/rowspan of an earlier cell (in the same row or a prior row) must simply be omitted + from `matrix` by the caller; this function has no notion of the overall grid-shape, only of the + cells it is told to emit. + """ + + def iter_trs(rows: Sequence[Sequence[SpannedCell]]) -> Iterator[str]: + for row in rows: + # -- suppress emission of rows with no cells -- + if not row: + continue + tds = (_format_td(text, colspan, rowspan) for text, colspan, rowspan in row) + yield f"{''.join(tds)}" + + return f"{''.join(iter_trs(matrix))}
" if matrix else "" + + +def collapse_matrix_of_keyed_cells_to_spans( + matrix: Sequence[Sequence[tuple[str, object]]], +) -> list[list[SpannedCell]]: + """Collapse a full row/column grid of `(cell_text, merge_key)` cells into merged spans. + + `matrix` must be "rectangular" in the sense that it represents every grid-position of the + table, including positions covered by a merge, unlike the `matrix` consumed by + `htmlify_matrix_of_spanned_cell_texts()`. Two grid-positions belong to the same merged region + exactly when their `merge_key` compares equal with `==`; a grid-position that is not merged + with any other must be given a `merge_key` that compares equal only to itself (e.g. a unique + `object()` instance). + + Only rectangular merged regions are supported (as is guaranteed by, e.g., DOCX and XLSX merge + semantics) -- an "L-shaped" or otherwise irregular region of matching keys produces undefined + (but not exception-raising) results. + + Returns one row per row of `matrix`, each containing a `(cell_text, colspan, rowspan)` triple + for each cell that "originates" a merged region (or an unmerged 1x1 cell), in left-to-right + order. A grid-position covered by the colspan/rowspan of such a cell is omitted. + """ + n_rows = len(matrix) + consumed = [[False] * len(row) for row in matrix] + spanned_rows: list[list[SpannedCell]] = [] + + for r, row in enumerate(matrix): + spanned_row: list[SpannedCell] = [] + for c, (text, key) in enumerate(row): + if consumed[r][c]: + continue + consumed[r][c] = True + + # -- extend rightward while the merge-key matches -- + colspan = 1 + while c + colspan < len(row) and row[c + colspan][1] == key: + consumed[r][c + colspan] = True + colspan += 1 + + # -- extend downward while the same colspan-wide run of merge-keys matches -- + rowspan = 1 + next_r = r + 1 + while next_r < n_rows: + next_row = matrix[next_r] + if c + colspan > len(next_row): + break + if any(next_row[c + i][1] != key for i in range(colspan)): + break + for i in range(colspan): + consumed[next_r][c + i] = True + rowspan += 1 + next_r += 1 + + spanned_row.append((text, colspan, rowspan)) + spanned_rows.append(spanned_row) + + return spanned_rows + + class HtmlTable: """A `` element.""" diff --git a/unstructured/partition/docx.py b/unstructured/partition/docx.py index 280a3206b0..bab54a341c 100644 --- a/unstructured/partition/docx.py +++ b/unstructured/partition/docx.py @@ -27,7 +27,10 @@ from unstructured.chunking import add_chunking_strategy from unstructured.cleaners.core import clean_bullets -from unstructured.common.html_table import htmlify_matrix_of_cell_texts +from unstructured.common.html_table import ( + collapse_matrix_of_keyed_cells_to_spans, + htmlify_matrix_of_spanned_cell_texts, +) from unstructured.documents.elements import ( Address, Element, @@ -496,6 +499,10 @@ def _convert_table_to_html(self, table: DocxTable) -> str:
+ A merged cell (`gridSpan` and/or `vMerge`) is emitted as a single `` carrying the + appropriate `colspan`/`rowspan` attribute rather than being repeated into every grid + position it visually covers. + `is_nested` is used for recursive calls when a nested table is encountered. Certain behaviors are different in that case, but the caller can safely ignore that parameter and allow it to take its default value. @@ -513,35 +520,52 @@ def iter_cell_block_items(cell: _Cell) -> Iterator[str]: yield paragraph.text elif isinstance(table := block_item, DocxTable): for row in table.rows: - yield from iter_row_cells_as_text(row) + yield from (text for text, _ in iter_row_cells(row)) + + def cell_text(cell: _Cell) -> str: + """The normalized text of `cell`, including that of any table nested in it.""" + text = " ".join(iter_cell_block_items(cell)) + return " ".join(text.split()) - def iter_row_cells_as_text(row: _Row) -> Iterator[str]: - """Generate the normalized text of each cell in `row` as a separate string. + def iter_row_cells(row: _Row) -> Iterator[tuple[str, _Cell | None]]: + """Generate (cell_text, cell) for each layout-grid position in `row`. - The text of each paragraph within a cell is not separated. A table nested in a cell is - converted to a normalized string of its contents and combined with the text of the - cell that contains the table. + `cell` is `None` for a grid-position with no `tc` element -- the (rare) case of a row + that starts late or ends early, or one where `row.cells` raises because the table has + merged or malformed cells; `cell_text` is the empty string in each such case. """ # -- Each omitted cell at the start of the row (pretty rare) gets the empty string. # -- This preserves column alignment when one or more initial cells are omitted. for _ in range(row.grid_cols_before): - yield "" + yield "", None try: # -- row.cells may introduce `ValueError: no tc element at grid_offset=X` if the # -- table has merged or malformed cells. always wrap in try/except. for cell in row.cells: - cell_text = " ".join(iter_cell_block_items(cell)) - yield " ".join(cell_text.split()) + yield cell_text(cell), cell except Exception as e: - logging.warning(f"Skipping cell in _iter_row_cells_as_text due to: {e}") - yield "" + logging.warning(f"Skipping cell in _convert_table_to_html due to: {e}") + yield "", None # -- Each omitted cell at the end of the row (also rare) gets the empty string. -- for _ in range(row.grid_cols_after): - yield "" + yield "", None + + def iter_row_merge_keyed_texts(row: _Row) -> Iterator[tuple[str, object]]: + """Generate (cell_text, merge_key) for each layout-grid position in `row`. + + `merge_key` is shared by every grid-position spanned by the same merged cell (DOCX + resolves both `gridSpan` and `vMerge="continue"` to the same underlying `tc` element), + and is otherwise unique, so it can be fed to `collapse_matrix_of_keyed_cells_to_spans()` + to recover the original merge geometry. + """ + for text, cell in iter_row_cells(row): + yield text, (cell._tc if cell is not None else object()) - return htmlify_matrix_of_cell_texts([list(iter_row_cells_as_text(r)) for r in table.rows]) + matrix = [list(iter_row_merge_keyed_texts(row)) for row in table.rows] + spanned_matrix = collapse_matrix_of_keyed_cells_to_spans(matrix) + return htmlify_matrix_of_spanned_cell_texts(spanned_matrix) @cached_property def _document(self) -> Document: From 2ceac727d15b64cf7a717c876957960adc44447f Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 17:24:05 -0500 Subject: [PATCH 02/19] chore: bump version to 0.27.6 --- CHANGELOG.md | 6 ++++++ unstructured/__version__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e7efc627..39efa107e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.6 + +### Fixes + +- **Stop duplicating merged-cell text in DOCX `text_as_html`.** A merged cell (`gridSpan`/`vMerge`) was repeated into every `` its merge visually covered, with no `colspan`/`rowspan` attribute marking the merge. Merged cells are now emitted once, with `colspan`/`rowspan` reflecting the true geometry. + ## 0.27.5 ### Fixes diff --git a/unstructured/__version__.py b/unstructured/__version__.py index a6976fb92c..a71228fe17 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.5" # pragma: no cover +__version__ = "0.27.6" # pragma: no cover From 832fafd0c4320763800c441f5d18213c5ddbfe00 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 17:34:33 -0500 Subject: [PATCH 03/19] fix: stop dropping empty rows in spanned HTML table output htmlify_matrix_of_spanned_cell_texts() suppressed rows with no originating cells, but such a row can legitimately represent a real grid-row entirely covered by a rowspan from an earlier row. Since HTML rowspan counts actual elements rather than "rows that had content", dropping the row shifted the column-placement of every subsequent row. Add a regression test through the public partition_docx() API, plus a combined collapse-and-render test covering a full-width (every-column) vertical merge -- the collapse and render steps were previously only unit-tested in isolation, which is how this went unnoticed. --- test_unstructured/common/test_html_table.py | 31 +++++++++++++++++++-- test_unstructured/partition/test_docx.py | 28 +++++++++++++++++++ unstructured/common/html_table.py | 8 ++++-- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/test_unstructured/common/test_html_table.py b/test_unstructured/common/test_html_table.py index 3ad7cf389c..0b790118b9 100644 --- a/test_unstructured/common/test_html_table.py +++ b/test_unstructured/common/test_html_table.py @@ -63,14 +63,39 @@ def it_emits_a_void_td_for_an_empty_spanned_cell(self): '
' ) - def it_suppresses_rows_with_no_cells(self): - assert htmlify_matrix_of_spanned_cell_texts([[("a", 1, 1)], []]) == ( - "
a
" + def it_emits_an_empty_tr_for_a_row_entirely_consumed_by_a_rowspan(self): + """A row with no originating cells still needs its own ``. + + HTML `rowspan` counts actual `` elements, not "rows that happened to have content"; + dropping this row would shift the column-placement of every row after it. + """ + assert htmlify_matrix_of_spanned_cell_texts([[("a", 1, 2)], [], [("b", 1, 1)]]) == ( + '
a
b
' ) def it_handles_an_empty_matrix(self): assert htmlify_matrix_of_spanned_cell_texts([]) == "" + def it_renders_a_full_width_vertical_merge_via_the_full_collapse_and_render_pipeline(self): + """Regression: `collapse_matrix_of_keyed_cells_to_spans()` and + `htmlify_matrix_of_spanned_cell_texts()` were each unit-tested individually, but their + interaction was not -- the collapse step legitimately produces an empty row for a grid-row + entirely covered by a multi-column `rowspan`, and the render step must still emit a `` + for it rather than suppressing it. + """ + keyed_matrix = [ + [("a", "M"), ("a", "M")], + [("a", "M"), ("a", "M")], + [("c", "C1"), ("d", "C2")], + ] + spanned_matrix = collapse_matrix_of_keyed_cells_to_spans(keyed_matrix) + assert spanned_matrix == [[("a", 2, 2)], [], [("c", 1, 1), ("d", 1, 1)]] + assert htmlify_matrix_of_spanned_cell_texts(spanned_matrix) == ( + '' + "" + "
a
cd
" + ) + class Describe_collapse_matrix_of_keyed_cells_to_spans: """Unit-test suite for `html_table.collapse_matrix_of_keyed_cells_to_spans()`.""" diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index 1e1438b2ef..8c75ebffce 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -178,6 +178,34 @@ def test_partition_docx_table_with_merged_cells_reports_spans_instead_of_duplica assert table.metadata.text_as_html.count(f">{letter}<") == 1 +def test_partition_docx_table_with_full_width_vertical_merge_reports_a_tr_for_every_row(tmp_path): + """A grid-row entirely covered by a `rowspan` (no originating cells of its own) still needs + its own `` in the output. HTML `rowspan` counts actual `` elements, not "rows that + happened to have content" -- suppressing this row would shift the column-placement of every + row after it. + + +---+ + | A | + | | + +---+ + | B | + +---+ + """ + document = docx.Document() + table = document.add_table(rows=3, cols=1) + table.cell(0, 0).merge(table.cell(1, 0)).text = "A" + table.cell(2, 0).text = "B" + docx_path = tmp_path / "vertical-merge.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path), infer_table_structure=True) + table_element = next(e for e in elements if isinstance(e, Table)) + + assert table_element.metadata.text_as_html == ( + '
A
B
' + ) + + def test_partition_docx_grabs_header_and_footer(): elements = partition_docx(example_doc_path("handbook-1p.docx")) diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index 56a11823db..fb88cdf807 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -76,9 +76,11 @@ def htmlify_matrix_of_spanned_cell_texts(matrix: Sequence[Sequence[SpannedCell]] def iter_trs(rows: Sequence[Sequence[SpannedCell]]) -> Iterator[str]: for row in rows: - # -- suppress emission of rows with no cells -- - if not row: - continue + # -- Unlike `htmlify_matrix_of_cell_texts()`, an empty row here is NOT suppressed: it + # -- represents a real grid-row entirely covered by a `rowspan` from an earlier row + # -- (its cells were already emitted there). Suppressing it would drop a ``, which + # -- shifts the column-placement of every subsequent row under HTML's rowspan model + # -- (rowspan counts actual `` elements, not "rows that happened to have content"). tds = (_format_td(text, colspan, rowspan) for text, colspan, rowspan in row) yield f"{''.join(tds)}" From ab6303156a6bf1d4e5c425e30584ddc6803d971f Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 17:51:18 -0500 Subject: [PATCH 04/19] fix: keep rowspan-bound rows together when chunking a table The table chunker split purely on row text length, with no awareness of an active `rowspan`. A split falling inside one left the origin chunk claiming more rows than were present, and shifted every cell in the continuation chunk into the wrong column (a fresh standalone `` has no earlier row to carry the span forward). Rows spanned by an earlier row's `rowspan` are now grouped and chunked as one atomic unit, computed via the standard overlapping-interval merge over each row's declared span reach. A group that doesn't fit even alone is emitted as one (necessarily oversized) chunk, the same tolerance already granted a single oversized row or cell. --- CHANGELOG.md | 6 ++ test_unstructured/chunking/test_base.py | 86 ++++++++++++++++++++++--- unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 85 ++++++++++++++++++------ unstructured/common/html_table.py | 13 ++++ 5 files changed, 161 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39efa107e6..8b7726430e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.7 + +### Fixes + +- **Keep rows bound by an active `rowspan` in the same table chunk.** Table chunking split purely on row *text* length, unaware that a `rowspan` crossing the boundary would overclaim rows in one chunk and shift cells into the wrong column in the next. Such rows are now kept together as one unit, same tolerance already given a single oversized row or cell. + ## 0.27.6 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 19c21abe90..7205f31133 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3019,6 +3019,72 @@ def and_it_uses_the_configured_measurement_units_for_row_fitting( ), ] + def and_it_keeps_rows_bound_by_an_active_rowspan_in_the_same_chunk(self): + """A split between rows still covered by an earlier row's `rowspan` would leave that + `rowspan` claiming more rows than are present in its chunk, and would shift every cell + in the continuation chunk into the wrong column (its `
` has no earlier row to + carry the span forward). The row-fit measurement only counts cell *text*, so a short-text, + markup-heavy table like this one can silently cross that boundary without the fix.""" + opts = ChunkingOptions(max_characters=50) + html_table = HtmlTable.from_html_text( + """ +
+ + + +
AAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
zzzzzzzzzzzzzzzzzzzz
+ """ + ) + + # -- the whole rowspan-bound group is emitted as one chunk, even though it exceeds + # -- `max_characters`, same tolerance already granted a single oversized row or cell -- + assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + ( + "AAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzzzzz", + "" + '' + "" + "" + "
AAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
zzzzzzzzzzzzzzzzzzzz
", + ), + ] + + def and_it_keeps_a_fully_consumed_continuation_row_with_its_rowspan_origin(self): + """The empty `` a fully-consumed continuation row emits (so a `rowspan` still counts + actual `` elements) must never be separated from the row whose `rowspan` covers it, + and a later, independent row may still join the same chunk when there's room.""" + opts = ChunkingOptions(max_characters=70) + html_table = HtmlTable.from_html_text( + """ + + + + + + +
BIGMERGEBIGMERGEBIGMERGE
ppppppppppppppppppppqqqqqqqqqqqqqqqqqqqq
rrrrrrrrrrrrrrrrrrrrssssssssssssssssssss
ttttttttttttttttttttuuuuuuuuuuuuuuuuuuuu
+ """ + ) + + assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + ( + "BIGMERGEBIGMERGEBIGMERGE pppppppppppppppppppp qqqqqqqqqqqqqqqqqqqq", + "" + '' + "" + "" + "
BIGMERGEBIGMERGEBIGMERGE
ppppppppppppppppppppqqqqqqqqqqqqqqqqqqqq
", + ), + ( + "rrrrrrrrrrrrrrrrrrrr ssssssssssssssssssss", + "
rrrrrrrrrrrrrrrrrrrrssssssssssssssssssss
", + ), + ( + "tttttttttttttttttttt uuuuuuuuuuuuuuuuuuuu", + "
ttttttttttttttttttttuuuuuuuuuuuuuuuuuuuu
", + ), + ] + class Describe_TextSplitter: """Unit-test suite for `unstructured.chunking.base._TextSplitter` objects.""" @@ -3217,7 +3283,7 @@ def it_accumulates_rows_added_to_it(self): accum = _RowAccumulator(maxlen=100) row = HtmlRow(fragment_fromstring("foobar")) - accum.add_row(row) + accum.add_rows([row]) assert accum._rows == [row] assert accum._row_text_len == len("foo bar") @@ -3226,8 +3292,8 @@ def and_it_uses_the_configured_measurement_units_for_remaining_space(self): accum = _RowAccumulator(maxlen=3, measure=lambda text: len(text.split())) row = HtmlRow(fragment_fromstring("supercalifragilisticexpialidocious")) - assert accum.will_fit(row) is True - accum.add_row(row) + assert accum.will_fit([row]) is True + accum.add_rows([row]) # -- one token of text plus one separator leaves one token of space -- assert accum._remaining_space == 1 @@ -3251,7 +3317,7 @@ def it_will_fit_a_row_with_text_shorter_than_maxlen_when_empty( accum = _RowAccumulator(maxlen=21) row = HtmlRow(fragment_fromstring(row_html)) - assert accum.will_fit(row) is expected_value + assert accum.will_fit([row]) is expected_value @pytest.mark.parametrize( ("row_html", "expected_value"), @@ -3270,15 +3336,17 @@ def and_it_will_fit_a_row_with_text_shorter_than_remaining_space_when_not_empty( ): """There is no overhead beyond row HTML for additional rows.""" accum = _RowAccumulator(maxlen=48) - accum.add_row(HtmlRow(fragment_fromstring("abcdefghijklmnopqrstuvwxyz"))) + accum.add_rows( + [HtmlRow(fragment_fromstring("abcdefghijklmnopqrstuvwxyz"))] + ) # -- remaining space is 48 - 26 = 21 -- row = HtmlRow(fragment_fromstring(row_html)) - assert accum.will_fit(row) is expected_value + assert accum.will_fit([row]) is expected_value def it_generates_a_TextAndHtml_pair_and_resets_itself_to_empty_when_flushed(self): accum = _RowAccumulator(maxlen=100) - accum.add_row(HtmlRow(fragment_fromstring("abcde fghij klmno"))) + accum.add_rows([HtmlRow(fragment_fromstring("abcde fghij klmno"))]) text, html = next(accum.flush()) @@ -3289,8 +3357,8 @@ def it_generates_a_TextAndHtml_pair_and_resets_itself_to_empty_when_flushed(self def and_the_HTML_contains_as_many_rows_as_were_accumulated(self): accum = _RowAccumulator(maxlen=100) - accum.add_row(HtmlRow(fragment_fromstring("abcde fghij klmno"))) - accum.add_row(HtmlRow(fragment_fromstring("pqrst uvwxy z"))) + accum.add_rows([HtmlRow(fragment_fromstring("abcde fghij klmno"))]) + accum.add_rows([HtmlRow(fragment_fromstring("pqrst uvwxy z"))]) text, html = next(accum.flush()) diff --git a/unstructured/__version__.py b/unstructured/__version__.py index a71228fe17..7c588e685f 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.6" # pragma: no cover +__version__ = "0.27.7" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 46c434fd73..9aebcbaf94 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -6,7 +6,7 @@ import copy import uuid from functools import cached_property -from typing import Any, Callable, DefaultDict, Iterable, Iterator, cast +from typing import Any, Callable, DefaultDict, Iterable, Iterator, Sequence, cast import regex from lxml.etree import ParserError, tostring @@ -1240,26 +1240,43 @@ def iter_subtables( def _iter_subtables(self) -> Iterator[TextAndHtml]: """Generate (text, html) pairs containing as many whole rows as will fit in window. - Falls back to splitting rows into whole cells when a single row is by itself too big to - fit in the chunking window. + Rows joined by an active `rowspan` are kept together as one atomic group — splitting + between them would leave a `rowspan` whose declared count exceeds the rows actually + present in its chunk, and would shift every following row in the continuation chunk into + the wrong column (that chunk's `` has no earlier row to carry the span forward). + Falls back to splitting rows into whole cells when a single row (or, when rowspan-bound, + a whole such group) is by itself too big to fit in the chunking window. """ is_first_chunk = True accum = _RowAccumulator(maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure) - for row in self._table_element.iter_rows(): - # -- if row won't fit, any WIP chunk is done, send it on its way -- - if not accum.will_fit(row): + for group in self._iter_rowspan_bound_row_groups(): + # -- if group won't fit, any WIP chunk is done, send it on its way -- + if not accum.will_fit(group): for text, html in accum.flush(): yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False accum = _RowAccumulator( maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure ) - # -- if row fits, add it to accumulator -- - if accum.will_fit(row): - accum.add_row(row) - else: # -- otherwise, single row is bigger than chunking window -- - for text, html in self._iter_row_splits(row, maxlen=self._maxlen(is_first_chunk)): + # -- if group fits, add it to accumulator -- + if accum.will_fit(group): + accum.add_rows(group) + elif len(group) == 1: # -- a single row is bigger than the chunking window -- + for text, html in self._iter_row_splits( + group[0], maxlen=self._maxlen(is_first_chunk) + ): + yield self._prepend_repeated_headers(text, html, is_first_chunk) + is_first_chunk = False + accum = _RowAccumulator( + maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure + ) + else: + # -- A rowspan-bound group doesn't fit even in an empty chunking window. Splitting + # -- it would corrupt the span it exists to protect, so it's emitted whole, same + # -- tolerance the codebase already grants a single oversized row/cell. + accum.add_rows(group) + for text, html in accum.flush(): yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False accum = _RowAccumulator( @@ -1270,6 +1287,26 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False + def _iter_rowspan_bound_row_groups(self) -> Iterator[tuple[HtmlRow, ...]]: + """Group consecutive rows that a `rowspan` binds together. + + A row whose cell declares `rowspan=N` binds the next `N-1` rows to it (they carry that + cell's continuation and would misplace their own cells, or overclaim the span's row + count, if split into a different chunk). Spans starting in different rows of the same + group can reach further than the row that opened the group, so the group's far edge is + the max reach of every span opened before it closes — the standard overlapping-interval + merge. A run of rows with no multi-row `rowspan` at all yields one-row groups, identical + to the pre-grouping behavior. + """ + rows = list(self._table_element.iter_rows()) + group_start = 0 + group_end = -1 # -- index of the furthest row any span opened so far reaches -- + for idx, row in enumerate(rows): + group_end = max(group_end, idx + row.max_rowspan - 1) + if idx == group_end: + yield tuple(rows[group_start : idx + 1]) + group_start = idx + 1 + def _iter_row_splits(self, row: HtmlRow, maxlen: int) -> Iterator[TextAndHtml]: """Split oversized row into (text, html) pairs containing as many cells as will fit.""" accum = _CellAccumulator(maxlen=maxlen) @@ -1707,10 +1744,13 @@ def __init__(self, maxlen: int, measure: Callable[[str], int] = len): self._rows: list[HtmlRow] = [] self._row_text_len = 0 - def add_row(self, row: HtmlRow) -> None: - """Add `row` to this accumulation. Caller is responsible for ensuring it will fit.""" - self._rows.append(row) - self._row_text_len += self._measured_row_text_len(row) + def add_rows(self, rows: Sequence[HtmlRow]) -> None: + """Add `rows` (a rowspan-bound group, possibly of length 1) to this accumulation. + + Caller is responsible for ensuring the group will fit. + """ + self._rows.extend(rows) + self._row_text_len += self._measured_rows_text_len(rows) def flush(self) -> Iterator[TextAndHtml]: """Generate zero-or-one (text, html) pairs for accumulated sub-table.""" @@ -1723,9 +1763,9 @@ def flush(self) -> Iterator[TextAndHtml]: self._row_text_len = 0 yield text, html - def will_fit(self, row: HtmlRow) -> bool: - """True when `row` will fit within remaining space left by accummulated rows.""" - return self._remaining_space >= self._measured_row_text_len(row) + def will_fit(self, rows: Sequence[HtmlRow]) -> bool: + """True when `rows` (a rowspan-bound group) will fit in space left by accumulated rows.""" + return self._remaining_space >= self._measured_rows_text_len(rows) def _iter_cell_texts(self) -> Iterator[str]: """Generate contents of each row cell as a separate string. @@ -1743,9 +1783,12 @@ def _remaining_space(self) -> int: separators_len = len(self._rows) return self._maxlen - separators_len - self._row_text_len - def _measured_row_text_len(self, row: HtmlRow) -> int: - """Length of `row` text in configured chunk-size units.""" - return self._measure(" ".join(row.iter_cell_texts())) + def _measured_rows_text_len(self, rows: Sequence[HtmlRow]) -> int: + """Length of the joined cell text of `rows` in configured chunk-size units.""" + texts: list[str] = [] + for row in rows: + texts.extend(row.iter_cell_texts()) + return self._measure(" ".join(texts)) # ================================================================================================ diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index fb88cdf807..a479ae8b4b 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -285,6 +285,11 @@ def text_len(self) -> int: """Length of the normalized text, as it would appear in `element.text`.""" return len(" ".join(self.iter_cell_texts())) + @cached_property + def max_rowspan(self) -> int: + """Largest `rowspan` declared by any cell in this row, `1` when none span multiple rows.""" + return max((cell.rowspan for cell in self.iter_cells()), default=1) + class HtmlCell: """A ``/``/`` boundaries, so it resolves to the rest of the table.""" + opts = ChunkingOptions(max_characters=15) + html_table = HtmlTable.from_html_text( + """ +
` element.""" @@ -301,3 +306,11 @@ def html(self) -> str: def text(self) -> str: """Text inside `` element, empty string when no text.""" return " ".join(self._td.text_content().split()) + + @cached_property + def rowspan(self) -> int: + """Declared `rowspan` for this cell, `1` when absent or unparseable.""" + try: + return max(1, int(self._td.attrib.get("rowspan", 1))) + except (TypeError, ValueError): + return 1 From 97b6e1649863e1e675863bcc4847af83a485a048 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 18:01:43 -0500 Subject: [PATCH 05/19] fix: clamp rowspan reach to available rows and honor rowspan="0" The rowspan-aware chunking boundary could silently lose an entire table's trailing rows when a declared rowspan named more rows than the table had (the group-closing index was never reached, so the final group was never yielded), and coerced rowspan="0" (HTML's "span every remaining row") to 1, letting a chunk boundary fall through an active maximal span. Both now resolve to "the rest of the table." --- CHANGELOG.md | 6 ++++ test_unstructured/chunking/test_base.py | 48 +++++++++++++++++++++++++ unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 12 ++++++- unstructured/common/html_table.py | 25 +++++++++---- 5 files changed, 85 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b7726430e..0d6b51d384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.8 + +### Fixes + +- **Don't drop a table's trailing rows when a `rowspan` reaches past the last row, and honor `rowspan="0"`.** The rowspan-aware chunking boundary added in 0.27.7 could silently lose an entire table's tail when a declared `rowspan` named more rows than the table had, and treated `rowspan="0"` (HTML's "span every remaining row") as no span at all, letting a chunk boundary fall through it. Both now resolve to "the rest of the table" and are never split or dropped. + ## 0.27.7 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 7205f31133..7e60308589 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3085,6 +3085,54 @@ def and_it_keeps_a_fully_consumed_continuation_row_with_its_rowspan_origin(self) ), ] + def and_it_does_not_drop_a_rowspan_that_reaches_past_the_last_row(self): + """A malformed but browser-tolerated `rowspan` naming more rows than the table has must + still yield its rows as one group, not disappear because the group-closing index it + names is never reached.""" + opts = ChunkingOptions(max_characters=25) + html_table = HtmlTable.from_html_text( + """ + + + +
Axxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
+ """ + ) + + assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + ( + "A xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", + "" + '' + "" + "
Axxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", + ), + ] + + def and_it_treats_rowspan_0_as_spanning_every_remaining_row(self): + """`rowspan="0"` is HTML's spelling for "spans every remaining row in the row group" — + the largest possible span, not the absence of one. This model doesn't track + `
+ + +
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
+ """ + ) + + assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + ( + "Region xxxxxxxxxxxxx yyyyyyyyyyyyy", + "" + '' + "" + "
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
", + ), + ] + class Describe_TextSplitter: """Unit-test suite for `unstructured.chunking.base._TextSplitter` objects.""" diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 7c588e685f..fd75f2ac73 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.7" # pragma: no cover +__version__ = "0.27.8" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 9aebcbaf94..e8051a567b 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1297,15 +1297,25 @@ def _iter_rowspan_bound_row_groups(self) -> Iterator[tuple[HtmlRow, ...]]: the max reach of every span opened before it closes — the standard overlapping-interval merge. A run of rows with no multi-row `rowspan` at all yields one-row groups, identical to the pre-grouping behavior. + + A declared span can reach past the last row the table actually has (a malformed but + browser-tolerated document, which clips it to the rows present) or be `rowspan="0"` + (spans every remaining row) — both resolve to "the rest of the table" here, and the + final, possibly-still-open group is always yielded rather than silently dropped. """ rows = list(self._table_element.iter_rows()) group_start = 0 group_end = -1 # -- index of the furthest row any span opened so far reaches -- for idx, row in enumerate(rows): - group_end = max(group_end, idx + row.max_rowspan - 1) + row_reach = len(rows) - 1 if row.max_rowspan is None else idx + row.max_rowspan - 1 + group_end = max(group_end, row_reach) if idx == group_end: yield tuple(rows[group_start : idx + 1]) group_start = idx + 1 + # -- a span reaching past the last row (or `rowspan="0"`) leaves a final group that + # -- never hits `idx == group_end` inside the loop; emit it rather than drop it -- + if group_start < len(rows): + yield tuple(rows[group_start:]) def _iter_row_splits(self, row: HtmlRow, maxlen: int) -> Iterator[TextAndHtml]: """Split oversized row into (text, html) pairs containing as many cells as will fit.""" diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index a479ae8b4b..1d46082762 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -286,9 +286,16 @@ def text_len(self) -> int: return len(" ".join(self.iter_cell_texts())) @cached_property - def max_rowspan(self) -> int: - """Largest `rowspan` declared by any cell in this row, `1` when none span multiple rows.""" - return max((cell.rowspan for cell in self.iter_cells()), default=1) + def max_rowspan(self) -> int | None: + """Largest `rowspan` declared by any cell in this row, `1` when none span multiple rows. + + `None` when any cell in this row declares `rowspan="0"` (HTML's "span every remaining + row"), since it reaches farther than any positive count could name. + """ + spans = [cell.rowspan for cell in self.iter_cells()] + if any(span is None for span in spans): + return None + return max((span for span in spans if span is not None), default=1) class HtmlCell: @@ -308,9 +315,15 @@ def text(self) -> str: return " ".join(self._td.text_content().split()) @cached_property - def rowspan(self) -> int: - """Declared `rowspan` for this cell, `1` when absent or unparseable.""" + def rowspan(self) -> int | None: + """Declared `rowspan` for this cell, `1` when absent or unparseable. + + `None` for `rowspan="0"`, HTML's spelling for "spans every remaining row in the + containing row group." This model doesn't track ``/``/`` + boundaries (see `HtmlTable`), so that resolves to the end of the table. + """ try: - return max(1, int(self._td.attrib.get("rowspan", 1))) + value = int(self._td.attrib.get("rowspan", 1)) except (TypeError, ValueError): return 1 + return None if value == 0 else max(1, value) From 7b3d6829c7e513926770d7e551e860f27fa85688 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 18:15:03 -0500 Subject: [PATCH 06/19] fix: scope rowspan chunk-boundary grouping to its own thead/tbody/tfoot row-group A rowspan="0" cell (or any positive rowspan) was resolved as reaching to the end of the whole table rather than the end of its own row-group, so a short header section could swallow an entire, otherwise-boundable body section into one unbounded chunk. Rows are now grouped by the identity of their actual containing // element (or the table itself, for a row with no section wrapper), and a span can no longer bind rows across a real section boundary. --- CHANGELOG.md | 6 ++ test_unstructured/chunking/test_base.py | 84 ++++++++++++++++++++++++ test_unstructured/partition/test_docx.py | 37 +++++++++++ unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 41 ++++++++++-- unstructured/common/html_table.py | 46 +++++++++++-- 6 files changed, 206 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d6b51d384..abfbc9041f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.9 + +### Fixes + +- **Scope `rowspan` chunk-boundary protection to its actual ``/``/`` row-group, not the whole table.** A `rowspan="0"` cell in a short `` (or any positive `rowspan`) was previously resolved as reaching to the end of the entire table rather than the end of its own row-group, so a real, bounded `` following it could get swallowed into one unbounded chunk alongside the header. Rows are now grouped by their actual containing section (or the whole table, when there is none), and a span can no longer bind rows across a real section boundary. + ## 0.27.8 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 7e60308589..a9b3dfb64d 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3133,6 +3133,90 @@ def and_it_treats_rowspan_0_as_spanning_every_remaining_row(self): ), ] + def and_it_bounds_a_rowspan_0_header_to_its_own_thead_instead_of_the_whole_table(self): + """`rowspan="0"` spans every remaining row in its OWN row-group, not the whole table. A + one-row `` closes the header's span there; the following `` must still + chunk normally instead of being swallowed into one unbounded group with the header.""" + opts = ChunkingOptions(max_characters=200) + body_rows = "".join(f"{i:040d}" for i in range(50)) + html_table = HtmlTable.from_html_text( + f""" + + + {body_rows} +
Header
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # -- bounded into many chunks, not one ~2KB chunk holding the header + all 50 body rows -- + assert len(chunks) > 1 + for _, html in chunks: + assert len(html) < 300 + # -- the header's own one-row group is still emitted intact, on its own -- + assert chunks[0][1].startswith('') + + def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): + """A `rowspan` declared inside one `` must not reach into a following `` or + `` — each row-group bounds its own spans, however far they claim to reach.""" + opts = ChunkingOptions(max_characters=70) + html_table = HtmlTable.from_html_text( + """ +
Header
+ + + + + + + +
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
zzzzzzzzzzzzzzzzzzzz
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # -- the tbody's 2-row group (bound by the rowspan, clipped to the tbody's own last row, + # -- not the declared rowspan=5) is one chunk; the tfoot row is independent and separate -- + assert chunks == [ + ( + "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", + "" + '' + "" + "
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", + ), + ( + "zzzzzzzzzzzzzzzzzzzz", + "
zzzzzzzzzzzzzzzzzzzz
", + ), + ] + + def and_it_still_spans_the_whole_group_for_a_sectionless_rowspan_0_table(self): + """The original motivating case (no explicit ``/``/``) must keep + working exactly as before row-groups were introduced: a table with no section wrapper is + itself one row-group, so `rowspan="0"` still reaches every row the table has.""" + opts = ChunkingOptions(max_characters=15) + html_table = HtmlTable.from_html_text( + """ + + + +
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
+ """ + ) + + assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + ( + "Region xxxxxxxxxxxxx yyyyyyyyyyyyy", + "" + '' + "" + "
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
", + ), + ] + class Describe_TextSplitter: """Unit-test suite for `unstructured.chunking.base._TextSplitter` objects.""" diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index 8c75ebffce..30e683da7f 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -206,6 +206,43 @@ def test_partition_docx_table_with_full_width_vertical_merge_reports_a_tr_for_ev ) +def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geometry(tmp_path): + """A DOCX table with a real vertical merge, partitioned then chunked with a small window, + must never split between rows an active `rowspan` still covers -- doing so would leave a + continuation `TableChunk` with cells shifted into the wrong column. This is the DOCX-specific + fix (real `colspan`/`rowspan` output) and the general rowspan-aware chunker exercised together + end to end, rather than only unit-tested in isolation. + """ + document = docx.Document() + table = document.add_table(rows=4, cols=2) + table.cell(0, 0).merge(table.cell(1, 0)).merge(table.cell(2, 0)).text = "REGIONWIDE TOTAL" + table.cell(0, 1).text = "alpha bravo charlie" + table.cell(1, 1).text = "delta echo foxtrot" + table.cell(2, 1).text = "golf hotel india" + table.cell(3, 0).text = "juliet" + table.cell(3, 1).text = "kilo lima mike" + docx_path = tmp_path / "merged-cell-chunking.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path), infer_table_structure=True) + table_element = next(e for e in elements if isinstance(e, Table)) + assert 'rowspan="3"' in table_element.metadata.text_as_html + + chunks = chunk_by_title([table_element], max_characters=60) + + assert len(chunks) > 1, "fixture should be oversized enough to actually require a split" + for chunk in chunks: + assert isinstance(chunk, TableChunk) + # -- every emitted chunk must itself be well-formed, parseable HTML -- + html = chunk.metadata.text_as_html + assert html.startswith("") + assert html.endswith("
") + # -- no cell's text is lost or duplicated across the whole set of chunks -- + combined_text = " ".join(chunk.text for chunk in chunks) + for word in ("REGIONWIDE", "alpha", "delta", "golf", "juliet", "kilo"): + assert combined_text.count(word) == 1 + + def test_partition_docx_grabs_header_and_footer(): elements = partition_docx(example_doc_path("handbook-1p.docx")) diff --git a/unstructured/__version__.py b/unstructured/__version__.py index fd75f2ac73..11757d2a5f 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.8" # pragma: no cover +__version__ = "0.27.9" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index e8051a567b..115875bbe0 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1300,23 +1300,54 @@ def _iter_rowspan_bound_row_groups(self) -> Iterator[tuple[HtmlRow, ...]]: A declared span can reach past the last row the table actually has (a malformed but browser-tolerated document, which clips it to the rows present) or be `rowspan="0"` - (spans every remaining row) — both resolve to "the rest of the table" here, and the - final, possibly-still-open group is always yielded rather than silently dropped. + (spans every remaining row) — both resolve to "the rest of the table's own row-group" + here (see `_group_last_idx`), and the final, possibly-still-open group is always yielded + rather than silently dropped. """ rows = list(self._table_element.iter_rows()) + group_last_idx = self._group_last_idx(rows) group_start = 0 group_end = -1 # -- index of the furthest row any span opened so far reaches -- for idx, row in enumerate(rows): - row_reach = len(rows) - 1 if row.max_rowspan is None else idx + row.max_rowspan - 1 + own_group_last = group_last_idx[idx] + row_reach = ( + own_group_last + if row.max_rowspan is None + else min(idx + row.max_rowspan - 1, own_group_last) + ) group_end = max(group_end, row_reach) if idx == group_end: yield tuple(rows[group_start : idx + 1]) group_start = idx + 1 - # -- a span reaching past the last row (or `rowspan="0"`) leaves a final group that - # -- never hits `idx == group_end` inside the loop; emit it rather than drop it -- + # -- a span reaching past the last row of its own row-group (or `rowspan="0"`) leaves a + # -- final group that never hits `idx == group_end` inside the loop; emit it rather than + # -- drop it -- if group_start < len(rows): yield tuple(rows[group_start:]) + @staticmethod + def _group_last_idx(rows: Sequence[HtmlRow]) -> list[int]: + """For each row-index in `rows`, the index of the last row sharing its row-group. + + Rows are grouped by identity of `HtmlRow.row_group_key` (a specific ``/``/ + `` element, or the `` itself for a row with no section wrapper), so a + `rowspan` — including `rowspan="0"`, HTML's "spans every remaining row in the row group" + — can never bind rows across a real section boundary. A table with no explicit sections + has exactly one row-group (the whole table), matching the simpler pre-row-group behavior. + """ + n = len(rows) + last_idx = [0] * n + i = 0 + while i < n: + key = rows[i].row_group_key + j = i + while j + 1 < n and rows[j + 1].row_group_key is key: + j += 1 + for k in range(i, j + 1): + last_idx[k] = j + i = j + 1 + return last_idx + def _iter_row_splits(self, row: HtmlRow, maxlen: int) -> Iterator[TextAndHtml]: """Split oversized row into (text, html) pairs containing as many cells as will fit.""" accum = _CellAccumulator(maxlen=maxlen) diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index 1d46082762..a438f02471 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -152,10 +152,12 @@ def __init__( table: HtmlElement, header_row_idxs: set[int] | None = None, source_row_htmls: Sequence[str] | None = None, + row_group_keys: Sequence[object] | None = None, ): self._table = table self._header_row_idxs = header_row_idxs or set() self._source_row_htmls = tuple(source_row_htmls or ()) + self._row_group_keys = tuple(row_group_keys or ()) @classmethod def from_html_text(cls, html_text: str) -> HtmlTable: @@ -166,7 +168,8 @@ def from_html_text(cls, html_text: str) -> HtmlTable: raise ValueError("`html_text` contains no `
` element") table = tables[0] - # -- capture header semantics and source row HTML before compactification strips details -- + # -- capture header semantics, source row HTML, and row-group identity before + # -- compactification strips those details -- rows = cast("list[HtmlElement]", table.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr")) source_row_htmls = tuple(etree.tostring(tr, encoding=str) for tr in rows) header_row_idxs = { @@ -174,6 +177,12 @@ def from_html_text(cls, html_text: str) -> HtmlTable: for idx, tr in enumerate(rows) if tr.getparent().tag == "thead" or bool(tr.xpath("./th")) } + # -- Each row's row-group is identified by its immediate parent element: a specific + # -- ``/``/`` when present, or the `
` itself for a row with no + # -- section wrapper. Captured now (identity survives the `.drop_tag()` below even though + # -- the dropped element becomes detached) so a `rowspan` can later be prevented from + # -- binding rows across a real section boundary. + row_group_keys = tuple(tr.getparent() for tr in rows) # -- remove ``, ``, and `` noise elements when present -- noise_elements = table.xpath(".//thead | .//tbody | .//tfoot") @@ -214,7 +223,12 @@ def from_html_text(cls, html_text: str) -> HtmlTable: suffix = " " if e.tail[-1].isspace() else "" e.tail = prefix + " ".join(parts) + suffix - return cls(table, header_row_idxs=header_row_idxs, source_row_htmls=source_row_htmls) + return cls( + table, + header_row_idxs=header_row_idxs, + source_row_htmls=source_row_htmls, + row_group_keys=row_group_keys, + ) @cached_property def html(self) -> str: @@ -232,7 +246,13 @@ def iter_rows(self) -> Iterator[HtmlRow]: rows = cast("list[HtmlElement]", self._table.xpath("./tr")) for idx, tr in enumerate(rows): source_html = self._source_row_htmls[idx] if idx < len(self._source_row_htmls) else None - yield HtmlRow(tr, is_header=(idx in self._header_row_idxs), source_html=source_html) + row_group_key = self._row_group_keys[idx] if idx < len(self._row_group_keys) else None + yield HtmlRow( + tr, + is_header=(idx in self._header_row_idxs), + source_html=source_html, + row_group_key=row_group_key, + ) @cached_property def text(self) -> str: @@ -245,10 +265,17 @@ def text(self) -> str: class HtmlRow: """A `` element.""" - def __init__(self, tr: HtmlElement, is_header: bool = False, source_html: str | None = None): + def __init__( + self, + tr: HtmlElement, + is_header: bool = False, + source_html: str | None = None, + row_group_key: object = None, + ): self._tr = tr self._is_header = is_header self._source_html = source_html + self._row_group_key = row_group_key @cached_property def html(self) -> str: @@ -269,6 +296,17 @@ def source_html(self) -> str | None: """Original source `` HTML captured before compactification, when available.""" return self._source_html + @property + def row_group_key(self) -> object: + """Identity of this row's containing row-group (a ``/``/`` element, + or the `
` itself for a row with no section wrapper). + + Two rows compare equal on this value (`is`) exactly when they belong to the same row-group + for `rowspan` purposes. `None` when unknown (e.g. an `HtmlRow` constructed directly rather + than via `HtmlTable.iter_rows()`), in which case all such rows are treated as one group. + """ + return self._row_group_key + def iter_cell_texts(self) -> Iterator[str]: """Generate contents of each cell of this row as a separate string. From 41676f39118ce5127ff62c10343d3ce3948a7b14 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 18:32:37 -0500 Subject: [PATCH 07/19] fix: never let a chunk cross a row-group boundary through a rowspan="0" cell The chunk accumulator packed row-groups together purely by character budget, so a rowspan="0" header bound to its own one-row could still end up in the same chunk as following rows once section wrappers are stripped from the emitted HTML -- reintroducing the column-shift corruption this feature exists to prevent. A row-group change is now a hard boundary specifically when a rowspan="0" row is involved; ordinary rows, and rows with a positive rowspan meant to carry across a boundary (e.g. a repeated header), are unaffected -- an earlier version of this fix flushed on every row-group change unconditionally and broke that intentional behavior. --- CHANGELOG.md | 6 +++ test_unstructured/chunking/test_base.py | 63 ++++++++++++++++++++++++- unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 48 +++++++++++++++++++ 4 files changed, 116 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abfbc9041f..1d3007aed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.10 + +### Fixes + +- **Never let a chunk's rows span more than one source ``/``/`` row-group when a `rowspan="0"` cell is involved.** Row-group boundaries were correctly used to compute rowspan-bound groups, but the chunk accumulator could still pack an already-grouped `rowspan="0"` header together with a following row-group's rows when both fit the character budget — since section wrappers are stripped from emitted chunk HTML, the reparsed, flattened result let that span reach into rows it was never meant to bind. A row-group change is now a hard chunk boundary whenever a `rowspan="0"` row is involved; ordinary rows (and rows with a real, positive `rowspan` meant to carry across a boundary, e.g. a repeated header) are unaffected. + ## 0.27.9 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index a9b3dfb64d..cab449cbe9 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3154,8 +3154,67 @@ def and_it_bounds_a_rowspan_0_header_to_its_own_thead_instead_of_the_whole_table assert len(chunks) > 1 for _, html in chunks: assert len(html) < 300 - # -- the header's own one-row group is still emitted intact, on its own -- - assert chunks[0][1].startswith('
') + # -- the header's own one-row group is emitted ALONE -- not merely first, with body rows + # -- trailing along behind it in the same chunk (that would leave the header chunk's flat, + # -- wrapper-less HTML with no visible row-group boundary, so its `rowspan="0"` would + # -- legitimately -- by HTML's own rules -- reach into the very body rows this test exists + # -- to keep separate) -- + assert chunks[0][1] == '
Header
Header
' + # -- no body row leaked into the header's chunk -- + assert chunks[0][1].count("") == 1 + + def and_it_bounds_a_rowspan_0_header_even_when_a_huge_window_would_otherwise_merge_sections( + self, + ): + """The row-group boundary must hold even when the character budget alone would happily + pack the header and every body row into one chunk -- it is model-derived, not a lucky + side-effect of a small `max_characters` accidentally forcing separate chunks.""" + opts = ChunkingOptions(max_characters=100_000) + body_rows = "".join(f"{i:040d}" for i in range(50)) + html_table = HtmlTable.from_html_text( + f""" + + + {body_rows} +
Header
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # -- two chunks: the header's own row-group, then every body row (which itself fits the + # -- huge window as one chunk, since nothing bounds it from below except its own group) -- + assert len(chunks) == 2 + assert chunks[0][1] == '
Header
' + assert chunks[1][1].count("") == 50 + + def and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group(self): + """General invariant check: reparsing each emitted chunk must never reveal a `rowspan` + binding rows that came from two different source row-groups -- if it did, the chunk's + own `rowspan` count would exceed the rows the chunk actually has (or would have, on a + different split), silently reintroducing the corruption this whole feature prevents.""" + opts = ChunkingOptions(max_characters=200) + body_rows = "".join(f"{i:040d}" for i in range(50)) + html_table = HtmlTable.from_html_text( + f""" + + + {body_rows} +
Header
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + for _, html in chunks: + reparsed = HtmlTable.from_html_text(html) + rows = list(reparsed.iter_rows()) + # -- a chunk is emitted wrapper-less, so it is itself exactly one row-group; every + # -- row's declared rowspan must therefore resolve within THIS chunk's own row count, + # -- never claiming more rows than the chunk actually contains -- + for idx, row in enumerate(rows): + if row.max_rowspan is not None: + assert idx + row.max_rowspan - 1 < len(rows) def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): """A `rowspan` declared inside one `` must not reach into a following `` or diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 11757d2a5f..654374eb31 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.9" # pragma: no cover +__version__ = "0.27.10" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 115875bbe0..84c7fa964a 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1251,6 +1251,23 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: accum = _RowAccumulator(maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure) for group in self._iter_rowspan_bound_row_groups(): + # -- Crossing a row-group boundary is only unsafe when a span already accumulated + # -- could reach further than the one row it currently occupies (see + # -- `crosses_a_row_group_unsafely_if_extended`) -- an ordinary header row (or any + # -- row with no real rowspan) is always safe to pack with whatever comes next, + # -- row-group or not, and forcing a flush there would needlessly fragment perfectly + # -- normal tables that were never at risk. + if ( + accum.row_group_key is not None + and group[0].row_group_key is not accum.row_group_key + and accum.crosses_a_row_group_unsafely_if_extended + ): + for text, html in accum.flush(): + yield self._prepend_repeated_headers(text, html, is_first_chunk) + is_first_chunk = False + accum = _RowAccumulator( + maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure + ) # -- if group won't fit, any WIP chunk is done, send it on its way -- if not accum.will_fit(group): for text, html in accum.flush(): @@ -1808,6 +1825,37 @@ def will_fit(self, rows: Sequence[HtmlRow]) -> bool: """True when `rows` (a rowspan-bound group) will fit in space left by accumulated rows.""" return self._remaining_space >= self._measured_rows_text_len(rows) + @property + def row_group_key(self) -> object | None: + """Row-group identity shared by the rows already accumulated, `None` if empty. + + A rowspan-bound group never itself spans two row-groups (`_iter_rowspan_bound_row_groups` + guarantees that), so any one row's key stands for the whole accumulation. + """ + return self._rows[0].row_group_key if self._rows else None + + @property + def crosses_a_row_group_unsafely_if_extended(self) -> bool: + """True when appending a row from a DIFFERENT row-group could corrupt a span already + held here. + + Only `rowspan="0"` (`max_rowspan is None`) is row-group-scoped by the HTML spec -- + "spans every remaining row IN THE ROW GROUP". A group of size 1 whose row declares it + was clipped down to exactly itself only because it sits last in its OWN row-group + (`_iter_rowspan_bound_row_groups` guarantees a size-1 group never means that for any + other reason). `HtmlTable.from_html_text` strips section wrappers when building the row + model, so appending a different row-group's rows after such a row would emit a flat, + wrapper-less `` in which `rowspan="0"` legitimately -- by HTML's own rules, absent + any visible section boundary -- reaches into rows it was never meant to bind. + + A POSITIVE declared span (e.g. `rowspan="2"`) is NOT row-group-scoped -- it is meant to, + and is already elsewhere in this codebase treated as intended to, extend into whatever + row follows it regardless of section (e.g. a `` header row repeated/carried + forward into the body). Flushing on those would fragment ordinary, correct tables that + were never at risk -- see `and_it_preserves_source_header_row_html_for_carried_rows`. + """ + return any(row.max_rowspan is None for row in self._rows) + def _iter_cell_texts(self) -> Iterator[str]: """Generate contents of each row cell as a separate string. From 99b81446a0298aa5a00c8fb88fee1d436f2053d4 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 18:53:25 -0500 Subject: [PATCH 08/19] fix: also treat a clipped positive rowspan as a hard chunk boundary A positive rowspan declaring more rows than its own thead/tbody/tfoot row-group actually has is clipped when grouping rows for chunking, but its emitted HTML still carries the original, uncorrected value. If that group got packed into the same chunk as a following row-group's rows, the value would legitimately reach into rows it was never meant to bind once section wrappers are stripped. A row-group change is now also a hard chunk boundary when the preceding group was clipped this way. A thead row's positive span is exempt, since it's already handled separately as a repeated/carried-forward header. --- CHANGELOG.md | 6 ++ test_unstructured/chunking/test_base.py | 98 ++++++++++++++++++++++++ unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 99 ++++++++++++++++--------- 4 files changed, 171 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d3007aed4..b008ba4e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.11 + +### Fixes + +- **Also treat a clipped positive `rowspan` as a hard chunk boundary, not only `rowspan="0"`.** A positive `rowspan` that declares more rows than its own ``/``/`` row-group actually has is clipped to that row-group when grouping rows for chunking, but its emitted HTML still carries the original, uncorrected declared value. If such a group got packed into the same chunk as a following row-group's rows, that value would legitimately reach into rows it was never meant to bind, once section wrappers are stripped. A row-group change is now also a hard chunk boundary whenever the preceding group's span was clipped this way — a `` row's positive span is unaffected, since it's already handled separately as a repeated/carried-forward header. + ## 0.27.10 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index cab449cbe9..739c39d0aa 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -28,6 +28,7 @@ is_title, ) from unstructured.chunking.dispatch import reconstruct_table_from_chunks +from unstructured.chunking.title import chunk_by_title from unstructured.common.html_table import HtmlCell, HtmlRow, HtmlTable from unstructured.documents.elements import ( CheckBox, @@ -3252,6 +3253,103 @@ def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): ), ] + def and_it_clips_a_positive_rowspan_even_when_a_huge_window_would_otherwise_merge_sections( + self, + ): + """The row-group boundary must hold for a CLIPPED positive `rowspan` too, not only + `rowspan="0"` -- and must hold even when the character budget alone would happily pack + the clipped group and the next row-group's rows into one chunk. A tight budget (as in + `and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody`) would pass even without + this protection, purely by accident; this uses a window large enough that only the + clipped-group tracking itself can be responsible for keeping the groups apart.""" + opts = ChunkingOptions(max_characters=100_000) + html_table = HtmlTable.from_html_text( + """ +
+ + + + + + + + +
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
zzzzzzzzzzzzzzzzzzzz
wwwwwwwwwwwwwwwwwwww
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # -- the tbody's clipped 2-row group is its own chunk (declared rowspan=5, but its own + # -- row-group only has 2 rows); the tfoot's rows are a separate chunk -- despite an + # -- enormous window that would gladly merge both into one -- + assert chunks == [ + ( + "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", + "" + '' + "" + "
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", + ), + ( + "zzzzzzzzzzzzzzzzzzzz wwwwwwwwwwwwwwwwwwww", + "" + "" + "" + "
zzzzzzzzzzzzzzzzzzzz
wwwwwwwwwwwwwwwwwwww
", + ), + ] + + def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self): + """Same clipped-positive-rowspan protection, exercised through the actual public + `chunk_by_title()` entry point rather than only the internal splitter class. + + Note: a reparsed chunk's own `max_rowspan` can legitimately exceed ITS OWN row count -- + an overdeclared span clipped to its source row-group is tolerated as-is (not rewritten), + same as the pre-existing `rowspan` reaching past the last row of a table. So the bound + checked by `and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group` isn't a + valid way to catch THIS bug -- it can't tell "overdeclared within its own group" from + "reached into a different group's rows" from the reparsed chunk alone. What actually + distinguishes the fixed from the broken behavior is whether tbody and tfoot content ever + land in the same chunk -- check that directly instead.""" + html = ( + "" + "" + '' + "" + "" + "" + "" + "" + "" + "
alpha bravo charliedelta echo foxtrot
golf hotel india
juliet kilo lima
mike november oscar
" + ) + text = ( + "alpha bravo charlie delta echo foxtrot golf hotel india " + "juliet kilo lima mike november oscar" + ) + table = Table(text, metadata=ElementMetadata(text_as_html=html)) + + chunks = chunk_by_title([table], max_characters=75) + + assert len(chunks) > 1 + for chunk in chunks: + assert isinstance(chunk, TableChunk) + # -- every emitted chunk is well-formed, parseable HTML -- + html = chunk.metadata.text_as_html + assert html.startswith("") + assert html.endswith("
") + # -- the tbody group and the tfoot group never land in the same chunk; if they did, + # -- the tbody's rowspan="5" would legitimately (per HTML's own rules, once section + # -- wrappers are stripped) reach into the tfoot rows and shift them a column over -- + has_tbody_content = "golf" in html + has_tfoot_content = "juliet" in html or "mike" in html + assert not (has_tbody_content and has_tfoot_content) + # -- no cell text lost or duplicated across the whole set of chunks -- + combined_text = " ".join(chunk.text for chunk in chunks) + for word in ("alpha", "delta", "golf", "juliet", "mike"): + assert combined_text.count(word) == 1 + def and_it_still_spans_the_whole_group_for_a_sectionless_rowspan_0_table(self): """The original motivating case (no explicit ``/``/``) must keep working exactly as before row-groups were introduced: a table with no section wrapper is diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 654374eb31..bef9bb955e 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.10" # pragma: no cover +__version__ = "0.27.11" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 84c7fa964a..b474d27c25 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1250,13 +1250,13 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: is_first_chunk = True accum = _RowAccumulator(maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure) - for group in self._iter_rowspan_bound_row_groups(): + for group, group_is_clipped in self._iter_rowspan_bound_row_groups(): # -- Crossing a row-group boundary is only unsafe when a span already accumulated - # -- could reach further than the one row it currently occupies (see - # -- `crosses_a_row_group_unsafely_if_extended`) -- an ordinary header row (or any - # -- row with no real rowspan) is always safe to pack with whatever comes next, - # -- row-group or not, and forcing a flush there would needlessly fragment perfectly - # -- normal tables that were never at risk. + # -- was clipped by ITS OWN row-group boundary (see + # -- `crosses_a_row_group_unsafely_if_extended`) -- an ordinary row, or a span that + # -- fits entirely within its own row-group as declared, is always safe to pack with + # -- whatever comes next, row-group or not, and forcing a flush there would needlessly + # -- fragment perfectly normal tables that were never at risk. if ( accum.row_group_key is not None and group[0].row_group_key is not accum.row_group_key @@ -1278,7 +1278,7 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: ) # -- if group fits, add it to accumulator -- if accum.will_fit(group): - accum.add_rows(group) + accum.add_rows(group, is_clipped=group_is_clipped) elif len(group) == 1: # -- a single row is bigger than the chunking window -- for text, html in self._iter_row_splits( group[0], maxlen=self._maxlen(is_first_chunk) @@ -1292,7 +1292,7 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: # -- A rowspan-bound group doesn't fit even in an empty chunking window. Splitting # -- it would corrupt the span it exists to protect, so it's emitted whole, same # -- tolerance the codebase already grants a single oversized row/cell. - accum.add_rows(group) + accum.add_rows(group, is_clipped=group_is_clipped) for text, html in accum.flush(): yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False @@ -1304,7 +1304,9 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False - def _iter_rowspan_bound_row_groups(self) -> Iterator[tuple[HtmlRow, ...]]: + def _iter_rowspan_bound_row_groups( + self, + ) -> Iterator[tuple[tuple[HtmlRow, ...], bool]]: """Group consecutive rows that a `rowspan` binds together. A row whose cell declares `rowspan=N` binds the next `N-1` rows to it (they carry that @@ -1320,27 +1322,47 @@ def _iter_rowspan_bound_row_groups(self) -> Iterator[tuple[HtmlRow, ...]]: (spans every remaining row) — both resolve to "the rest of the table's own row-group" here (see `_group_last_idx`), and the final, possibly-still-open group is always yielded rather than silently dropped. + + Each yielded group is paired with a `bool`: whether the group's far edge was *clipped* by + its own row-group boundary in a way that's unsafe to extend across that boundary. + `rowspan="0"` is always clipped this way, regardless of section — there is no literal + count for it to fall back on, so its emitted HTML would let it reach into whatever + follows once section wrappers are stripped. A POSITIVE span (e.g. `rowspan="5"`) that + declares more rows than its own row-group actually has (`idx + max_rowspan - 1 > + own_group_last`) is clipped the same way -- UNLESS its own row-group is a ``: a + `` row's positive span is already established, elsewhere in this codebase, as + intentionally extending past its own section (a header repeated/carried forward into the + body via `_prepend_repeated_headers`, independent of this grouping mechanism entirely) -- + see `and_it_preserves_source_header_row_html_for_carried_rows`. A span that fits entirely + within its own row-group as declared is never clipped — its literal value already stops + at the right row regardless of what follows, so packing more rows after it is always safe. """ rows = list(self._table_element.iter_rows()) group_last_idx = self._group_last_idx(rows) group_start = 0 group_end = -1 # -- index of the furthest row any span opened so far reaches -- + group_clipped = False for idx, row in enumerate(rows): own_group_last = group_last_idx[idx] - row_reach = ( - own_group_last - if row.max_rowspan is None - else min(idx + row.max_rowspan - 1, own_group_last) - ) + is_thead = getattr(row.row_group_key, "tag", None) == "thead" + if row.max_rowspan is None: + row_reach = own_group_last + row_clipped = True + else: + declared_reach = idx + row.max_rowspan - 1 + row_reach = min(declared_reach, own_group_last) + row_clipped = declared_reach > own_group_last and not is_thead group_end = max(group_end, row_reach) + group_clipped = group_clipped or row_clipped if idx == group_end: - yield tuple(rows[group_start : idx + 1]) + yield tuple(rows[group_start : idx + 1]), group_clipped group_start = idx + 1 + group_clipped = False # -- a span reaching past the last row of its own row-group (or `rowspan="0"`) leaves a # -- final group that never hits `idx == group_end` inside the loop; emit it rather than - # -- drop it -- + # -- drop it. It's always clipped -- that's exactly why it never closed on its own. -- if group_start < len(rows): - yield tuple(rows[group_start:]) + yield tuple(rows[group_start:]), True @staticmethod def _group_last_idx(rows: Sequence[HtmlRow]) -> list[int]: @@ -1801,14 +1823,22 @@ def __init__(self, maxlen: int, measure: Callable[[str], int] = len): self._measure = measure self._rows: list[HtmlRow] = [] self._row_text_len = 0 + self._has_clipped_group = False - def add_rows(self, rows: Sequence[HtmlRow]) -> None: + def add_rows(self, rows: Sequence[HtmlRow], is_clipped: bool = False) -> None: """Add `rows` (a rowspan-bound group, possibly of length 1) to this accumulation. + `is_clipped` is whether the group's far edge was clipped by its own row-group boundary + (see `_iter_rowspan_bound_row_groups`) rather than reflecting a span's literal declared + value -- once true for any group in this accumulation, it stays true (a single clipped + group anywhere in the accumulated rows is enough to make extending across a row-group + boundary unsafe). + Caller is responsible for ensuring the group will fit. """ self._rows.extend(rows) self._row_text_len += self._measured_rows_text_len(rows) + self._has_clipped_group = self._has_clipped_group or is_clipped def flush(self) -> Iterator[TextAndHtml]: """Generate zero-or-one (text, html) pairs for accumulated sub-table.""" @@ -1819,6 +1849,7 @@ def flush(self) -> Iterator[TextAndHtml]: html = f"{trs_str}
" self._rows.clear() self._row_text_len = 0 + self._has_clipped_group = False yield text, html def will_fit(self, rows: Sequence[HtmlRow]) -> bool: @@ -1839,22 +1870,24 @@ def crosses_a_row_group_unsafely_if_extended(self) -> bool: """True when appending a row from a DIFFERENT row-group could corrupt a span already held here. - Only `rowspan="0"` (`max_rowspan is None`) is row-group-scoped by the HTML spec -- - "spans every remaining row IN THE ROW GROUP". A group of size 1 whose row declares it - was clipped down to exactly itself only because it sits last in its OWN row-group - (`_iter_rowspan_bound_row_groups` guarantees a size-1 group never means that for any - other reason). `HtmlTable.from_html_text` strips section wrappers when building the row - model, so appending a different row-group's rows after such a row would emit a flat, - wrapper-less `` in which `rowspan="0"` legitimately -- by HTML's own rules, absent - any visible section boundary -- reaches into rows it was never meant to bind. - - A POSITIVE declared span (e.g. `rowspan="2"`) is NOT row-group-scoped -- it is meant to, - and is already elsewhere in this codebase treated as intended to, extend into whatever - row follows it regardless of section (e.g. a `` header row repeated/carried - forward into the body). Flushing on those would fragment ordinary, correct tables that - were never at risk -- see `and_it_preserves_source_header_row_html_for_carried_rows`. + True whenever any group accumulated so far was *clipped* by its own row-group boundary + (tracked via `add_rows`'s `is_clipped` argument) -- `rowspan="0"` (which is row-group- + scoped by the HTML spec, "spans every remaining row IN THE ROW GROUP") is always clipped + by construction, and so is a positive declared span (e.g. `rowspan="5"`) that named more + rows than its own row-group actually had. In either case the emitted cell's HTML still + carries the original, uncorrected declared value -- never rewritten to match the clip -- + and `HtmlTable.from_html_text` strips section wrappers when building the row model, so + appending a different row-group's rows after such a group would emit a flat, wrapper-less + `
` in which that declared value legitimately -- by HTML's own rules, absent any + visible section boundary -- reaches into rows it was never meant to bind. + + A span that fit entirely within its own row-group AS DECLARED (never clipped) is safe to + extend across a row-group boundary -- its literal value already stops at the right row + regardless of what follows (e.g. a `` header row's `rowspan="2"` correctly repeated + /carried forward into the body). Flushing on those would fragment ordinary, correct tables + that were never at risk -- see `and_it_preserves_source_header_row_html_for_carried_rows`. """ - return any(row.max_rowspan is None for row in self._rows) + return self._has_clipped_group def _iter_cell_texts(self) -> Iterator[str]: """Generate contents of each row cell as a separate string. From 6490396c7705f28061b55ef460d65c169aac2344 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 21:03:20 -0500 Subject: [PATCH 09/19] Rewrite chunked-table rowspan handling to self-correct emitted span values Replace chunk-boundary bookkeeping ("is this group unsafe to extend across a row-group boundary") with a structural safety net: at the point each chunk's rows are finalized, any cell whose declared rowspan (including rowspan="0") would claim more rows than are actually present in that same chunk gets its rowspan rewritten to match reality. An emitted span can no longer overreach regardless of what else gets packed into the same chunk, closing this class of bug by construction rather than by enumerating unsafe cases one at a time. The existing row-group-boundary flush logic is kept (it still avoids visually merging a clipped group's own row-group with unrelated content), but two real gaps in it are fixed along the way: - An overdeclared rowspan was unconditionally exempted from clipping, even when header repetition isn't actually configured/ active for it -- nothing else protects such a row, so it's now only exempted when it's a genuine, active carried-forward header row. - The accumulator compared a candidate group's row-group identity against the FIRST row it had ever accumulated rather than the most recently accumulated one, which could miss a real transition when an earlier and later row happened to share the same row-group identity (e.g. two direct rows around an explicit ). --- CHANGELOG.md | 6 + test_unstructured/chunking/test_base.py | 159 +++++++++++++++++---- unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 179 ++++++++++++++++-------- unstructured/common/html_table.py | 47 ++++++- 5 files changed, 306 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b008ba4e9c..ea30afd3a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.12 + +### Fixes + +- **Rewrite chunked-table `rowspan` handling to self-correct emitted span values, instead of tracking which chunk boundaries are unsafe to cross.** A clipped or `rowspan="0"` cell's emitted HTML previously always carried its original declared value, relying on chunk-boundary bookkeeping to keep it away from rows it didn't truly cover. That value is now rewritten, at the point each chunk is assembled, to the number of rows actually present in that same chunk — an emitted `rowspan` can no longer overreach regardless of what else the chunker decides to pack alongside it. Also fixes two remaining gaps in the boundary bookkeeping this replaces: an overdeclared `` `rowspan` was exempted from clipping even when header repetition isn't active for it (so nothing else protected it), and the accumulator compared a candidate row-group against the FIRST row it had accumulated rather than the most recent one, which could miss a real transition when an earlier and later row happened to share the same row-group identity. + ## 0.27.11 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 739c39d0aa..c6f1dc8a4a 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3089,7 +3089,9 @@ def and_it_keeps_a_fully_consumed_continuation_row_with_its_rowspan_origin(self) def and_it_does_not_drop_a_rowspan_that_reaches_past_the_last_row(self): """A malformed but browser-tolerated `rowspan` naming more rows than the table has must still yield its rows as one group, not disappear because the group-closing index it - names is never reached.""" + names is never reached -- and the emitted `rowspan` is rewritten from its overdeclared + "3" down to "2", the number of rows the table (and this chunk) actually has, so the + emitted HTML is self-consistent regardless of what a caller inspects it in isolation.""" opts = ChunkingOptions(max_characters=25) html_table = HtmlTable.from_html_text( """ @@ -3104,7 +3106,7 @@ def and_it_does_not_drop_a_rowspan_that_reaches_past_the_last_row(self): ( "A xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", "
" - '' + '' "" "
Axxxxxxxxxxxxxxxxxxxx
Axxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", ), @@ -3113,7 +3115,9 @@ def and_it_does_not_drop_a_rowspan_that_reaches_past_the_last_row(self): def and_it_treats_rowspan_0_as_spanning_every_remaining_row(self): """`rowspan="0"` is HTML's spelling for "spans every remaining row in the row group" — the largest possible span, not the absence of one. This model doesn't track - ``/``/`` boundaries, so it resolves to the rest of the table.""" + ``/``/`` boundaries here (no explicit sections), so it resolves to + the rest of the table -- emitted as the literal, self-consistent count ("2") rather than + the ambiguous "0", since both rows land in this same chunk.""" opts = ChunkingOptions(max_characters=15) html_table = HtmlTable.from_html_text( """ @@ -3128,7 +3132,7 @@ def and_it_treats_rowspan_0_as_spanning_every_remaining_row(self): ( "Region xxxxxxxxxxxxx yyyyyyyyyyyyy", "" - '' + '' "" "
Regionxxxxxxxxxxxxx
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
", ), @@ -3156,11 +3160,11 @@ def and_it_bounds_a_rowspan_0_header_to_its_own_thead_instead_of_the_whole_table for _, html in chunks: assert len(html) < 300 # -- the header's own one-row group is emitted ALONE -- not merely first, with body rows - # -- trailing along behind it in the same chunk (that would leave the header chunk's flat, - # -- wrapper-less HTML with no visible row-group boundary, so its `rowspan="0"` would - # -- legitimately -- by HTML's own rules -- reach into the very body rows this test exists - # -- to keep separate) -- - assert chunks[0][1] == '
Header
' + # -- trailing along behind it in the same chunk. Its `rowspan="0"` is also rewritten to + # -- match: within its own one-row group it claims no further rows at all, so the emitted + # -- cell carries no `rowspan` attribute (the implicit default is 1) rather than the + # -- ambiguous "0" -- self-consistent even if this chunk were inspected on its own -- + assert chunks[0][1] == "
Header
" # -- no body row leaked into the header's chunk -- assert chunks[0][1].count("") == 1 @@ -3186,7 +3190,7 @@ def and_it_bounds_a_rowspan_0_header_even_when_a_huge_window_would_otherwise_mer # -- two chunks: the header's own row-group, then every body row (which itself fits the # -- huge window as one chunk, since nothing bounds it from below except its own group) -- assert len(chunks) == 2 - assert chunks[0][1] == '
Header
' + assert chunks[0][1] == "
Header
" assert chunks[1][1].count("") == 50 def and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group(self): @@ -3219,7 +3223,10 @@ def and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group(self): def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): """A `rowspan` declared inside one `` must not reach into a following `` or - `` — each row-group bounds its own spans, however far they claim to reach.""" + `` — each row-group bounds its own spans, however far they claim to reach. The + overdeclared "5" is rewritten to "2" (the tbody's own row count), so the emitted chunk is + self-consistent even inspected on its own, not merely safe because the tfoot landed + elsewhere.""" opts = ChunkingOptions(max_characters=70) html_table = HtmlTable.from_html_text( """ @@ -3243,7 +3250,7 @@ def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): ( "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", "" - '' + '' "" "
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", ), @@ -3280,14 +3287,14 @@ def and_it_clips_a_positive_rowspan_even_when_a_huge_window_would_otherwise_merg chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- the tbody's clipped 2-row group is its own chunk (declared rowspan=5, but its own - # -- row-group only has 2 rows); the tfoot's rows are a separate chunk -- despite an - # -- enormous window that would gladly merge both into one -- + # -- the tbody's clipped 2-row group is its own chunk (declared rowspan=5, rewritten to + # -- "2", its own row-group's true count); the tfoot's rows are a separate chunk -- + # -- despite an enormous window that would gladly merge both into one -- assert chunks == [ ( "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", "" - '' + '' "" "
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", ), @@ -3304,14 +3311,12 @@ def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self) """Same clipped-positive-rowspan protection, exercised through the actual public `chunk_by_title()` entry point rather than only the internal splitter class. - Note: a reparsed chunk's own `max_rowspan` can legitimately exceed ITS OWN row count -- - an overdeclared span clipped to its source row-group is tolerated as-is (not rewritten), - same as the pre-existing `rowspan` reaching past the last row of a table. So the bound - checked by `and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group` isn't a - valid way to catch THIS bug -- it can't tell "overdeclared within its own group" from - "reached into a different group's rows" from the reparsed chunk alone. What actually - distinguishes the fixed from the broken behavior is whether tbody and tfoot content ever - land in the same chunk -- check that directly instead.""" + The overdeclared `rowspan="5"` is rewritten to match the tbody's own true row count, so + `and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group`'s general reparse + check would in fact also catch a regression here -- this test additionally checks the + thing that check can't see on its own: that tbody and tfoot content never land in the + same chunk in the first place, which is what the row-group-boundary flush (not the + rewrite) is responsible for.""" html = ( "" "" @@ -3353,7 +3358,8 @@ def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self) def and_it_still_spans_the_whole_group_for_a_sectionless_rowspan_0_table(self): """The original motivating case (no explicit ``/``/``) must keep working exactly as before row-groups were introduced: a table with no section wrapper is - itself one row-group, so `rowspan="0"` still reaches every row the table has.""" + itself one row-group, so `rowspan="0"` still reaches every row the table has -- emitted + as the literal count ("2") rather than the ambiguous "0".""" opts = ChunkingOptions(max_characters=15) html_table = HtmlTable.from_html_text( """ @@ -3368,12 +3374,113 @@ def and_it_still_spans_the_whole_group_for_a_sectionless_rowspan_0_table(self): ( "Region xxxxxxxxxxxxx yyyyyyyyyyyyy", "
" - '' + '' "" "
Regionxxxxxxxxxxxxx
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
", ), ] + def and_an_exactly_fitting_positive_rowspan_is_emitted_unchanged(self): + """A `rowspan` whose declared value already matches its own row-group's row count is + never rewritten -- the self-correction is a no-op whenever the declared value was + already honest, so ordinary, correct tables see no behavior change at all.""" + opts = ChunkingOptions(max_characters=100) + html_table = HtmlTable.from_html_text( + """ + + + +
AB
C
+ """ + ) + + assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + ( + "A B C", + '
AB
C
', + ), + ] + + def and_it_clips_an_overdeclared_thead_rowspan_when_header_repetition_is_not_active(self): + """A `` row's overdeclared positive `rowspan` is only exempt from clipping when + header repetition is actually configured and active for it -- `_prepend_repeated_headers` + is what makes reaching past the header's own section safe, and that mechanism plays no + part when `repeat_table_headers=False`. Without it, nothing else protects an overreaching + `` span from binding body rows it was never meant to cover, so it must be clipped + exactly like a ``/`` span would be.""" + opts = ChunkingOptions(max_characters=200) + html_table = HtmlTable.from_html_text( + """ + + + + + + +
HeaderHX
AB
CD
+ """ + ) + + # -- header_row_count=0 (the default) means repetition is never configured, so the + # -- `` row gets no carry-forward exemption regardless of `repeat_table_headers`, + # -- and (now correctly recognized as clipped) is kept isolated from the tbody rows by the + # -- same row-group-boundary flush that already protects ``/`` spans -- + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert len(chunks) == 2 + for _, html in chunks: + reparsed = HtmlTable.from_html_text(html) + rows = list(reparsed.iter_rows()) + for idx, row in enumerate(rows): + if row.max_rowspan is not None: + assert idx + row.max_rowspan - 1 < len(rows) + # -- the header's declared "3" is additionally clipped down to "1" by the same rewrite + # -- that protects every other row-group (its own thead has just the one row), so even + # -- inspected on its own the header's chunk carries no false claim over body content -- + assert chunks[0][1] == "
HeaderHX
" + assert ( + chunks[1][1] + == "
AB
CD
" + ) + + def and_it_does_not_let_a_clipped_span_bind_a_later_direct_row_of_the_same_key(self): + """`_RowAccumulator` must compare a candidate group's row-group identity against the + MOST RECENTLY accumulated row-group, not the first one ever added to this accumulator -- + a direct (sectionless) row before and after an explicit `` both key by the same + `` element, so comparing against the first accumulated row can mistake a real + transition (leaving the clipped tbody group) for "no change", letting the clipped span + bind the trailing direct row it was never meant to cover.""" + opts = ChunkingOptions(max_characters=200) + html_table = HtmlTable.from_html_text( + """ +
+ + + +
LeadX
ScopedInside
AfterY
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # -- comparing against the LAST accumulated row's key (tbody) rather than the first + # -- (table) correctly detects "After" as a real transition away from the clipped + # -- "Scoped" group, forcing a flush that keeps them apart -- + assert len(chunks) == 2 + for _, html in chunks: + reparsed = HtmlTable.from_html_text(html) + rows = list(reparsed.iter_rows()) + for idx, row in enumerate(rows): + if row.max_rowspan is not None: + assert idx + row.max_rowspan - 1 < len(rows) + # -- "Scoped"'s declared "2" is additionally clipped to "1" (its own tbody has only its + # -- own one row), so even inspected on its own its chunk claims nothing past itself -- + assert ( + chunks[0][1] == "" + "
LeadX
ScopedInside
" + ) + assert chunks[1][1] == "
AfterY
" + class Describe_TextSplitter: """Unit-test suite for `unstructured.chunking.base._TextSplitter` objects.""" diff --git a/unstructured/__version__.py b/unstructured/__version__.py index bef9bb955e..f1527a87a7 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.11" # pragma: no cover +__version__ = "0.27.12" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index b474d27c25..791726f244 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1250,16 +1250,19 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: is_first_chunk = True accum = _RowAccumulator(maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure) - for group, group_is_clipped in self._iter_rowspan_bound_row_groups(): + for group, group_bounds, group_is_clipped in self._iter_rowspan_bound_row_groups(): # -- Crossing a row-group boundary is only unsafe when a span already accumulated # -- was clipped by ITS OWN row-group boundary (see # -- `crosses_a_row_group_unsafely_if_extended`) -- an ordinary row, or a span that # -- fits entirely within its own row-group as declared, is always safe to pack with # -- whatever comes next, row-group or not, and forcing a flush there would needlessly - # -- fragment perfectly normal tables that were never at risk. + # -- fragment perfectly normal tables that were never at risk. This check is a + # -- semantic nicety, not a correctness requirement: `group_bounds` (below) is what + # -- actually guarantees an emitted `rowspan` can never overreach, independent of + # -- whether this flush decision fires correctly. if ( - accum.row_group_key is not None - and group[0].row_group_key is not accum.row_group_key + accum.last_row_group_key is not None + and group[0].row_group_key is not accum.last_row_group_key and accum.crosses_a_row_group_unsafely_if_extended ): for text, html in accum.flush(): @@ -1278,7 +1281,7 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: ) # -- if group fits, add it to accumulator -- if accum.will_fit(group): - accum.add_rows(group, is_clipped=group_is_clipped) + accum.add_rows(group, group_bounds, is_clipped=group_is_clipped) elif len(group) == 1: # -- a single row is bigger than the chunking window -- for text, html in self._iter_row_splits( group[0], maxlen=self._maxlen(is_first_chunk) @@ -1292,7 +1295,7 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: # -- A rowspan-bound group doesn't fit even in an empty chunking window. Splitting # -- it would corrupt the span it exists to protect, so it's emitted whole, same # -- tolerance the codebase already grants a single oversized row/cell. - accum.add_rows(group, is_clipped=group_is_clipped) + accum.add_rows(group, group_bounds, is_clipped=group_is_clipped) for text, html in accum.flush(): yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False @@ -1306,7 +1309,7 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: def _iter_rowspan_bound_row_groups( self, - ) -> Iterator[tuple[tuple[HtmlRow, ...], bool]]: + ) -> Iterator[tuple[tuple[HtmlRow, ...], tuple[int | None, ...], bool]]: """Group consecutive rows that a `rowspan` binds together. A row whose cell declares `rowspan=N` binds the next `N-1` rows to it (they carry that @@ -1323,46 +1326,82 @@ def _iter_rowspan_bound_row_groups( here (see `_group_last_idx`), and the final, possibly-still-open group is always yielded rather than silently dropped. - Each yielded group is paired with a `bool`: whether the group's far edge was *clipped* by - its own row-group boundary in a way that's unsafe to extend across that boundary. - `rowspan="0"` is always clipped this way, regardless of section — there is no literal - count for it to fall back on, so its emitted HTML would let it reach into whatever - follows once section wrappers are stripped. A POSITIVE span (e.g. `rowspan="5"`) that - declares more rows than its own row-group actually has (`idx + max_rowspan - 1 > - own_group_last`) is clipped the same way -- UNLESS its own row-group is a ``: a - `` row's positive span is already established, elsewhere in this codebase, as - intentionally extending past its own section (a header repeated/carried forward into the - body via `_prepend_repeated_headers`, independent of this grouping mechanism entirely) -- - see `and_it_preserves_source_header_row_html_for_carried_rows`. A span that fits entirely - within its own row-group as declared is never clipped — its literal value already stops - at the right row regardless of what follows, so packing more rows after it is always safe. + Each yielded group is paired with two things: + + - A same-length tuple of per-row **safe rowspan bounds**: for the row at position `p` + within the group, either the true number of rows (starting at that row, within its own + row-group) it may safely claim, or `None` when the row is an actively carried-forward + header row (see below) and should be emitted with its span exactly as declared. This is + consumed by `HtmlRow.html_clipped_to_rows()` at emission time and is the *structural* + safety net: whatever else this class's grouping/accumulation logic decides to pack into + the same chunk, an emitted `rowspan` can never claim a row that isn't genuinely present + in that same chunk, because it is rewritten to match this bound regardless. + - A `bool`: whether the group's far edge was *clipped* by its own row-group boundary in a + way that makes it unsafe to *also* pack a different row-group's rows into the same + chunk (used only by `_RowAccumulator.crosses_a_row_group_unsafely_if_extended` to decide + when to keep a clipped group isolated in its own chunk, a semantic — not correctness — + concern now that the per-row bounds above independently guarantee correctness). + + `rowspan="0"` is always clipped — there is no literal count for it to fall back on. A + POSITIVE span (e.g. `rowspan="5"`) that declares more rows than its own row-group actually + has (`idx + max_rowspan - 1 > own_group_last`) is clipped the same way — UNLESS it is an + **actively carried-forward header row**: a `` row, within the configured header-row + count, whose header text is actually going to be repeated onto continuation chunks via + `_prepend_repeated_headers` (`self._should_repeat_headers`). That combination is already + established elsewhere in this codebase as intentionally extending a header's span past its + own section — see `and_it_preserves_source_header_row_html_for_carried_rows` — so such a + row is exempted from both the per-row bound (`None`, emit as declared) and the group-clip + flag. A `` row whose span merely happens to overreach with no active repetition + configured (e.g. `repeat_table_headers=False`) gets no such exemption: its bound is clipped + like any other, since nothing else protects it from binding rows it was never meant to + cover. A span that fits entirely within its own row-group as declared is never clipped — + its literal value already stops at the right row regardless of what follows. """ rows = list(self._table_element.iter_rows()) + n = len(rows) group_last_idx = self._group_last_idx(rows) - group_start = 0 - group_end = -1 # -- index of the furthest row any span opened so far reaches -- - group_clipped = False + reach = [0] * n + clipped = [False] * n + bound: list[int | None] = [None] * n for idx, row in enumerate(rows): own_group_last = group_last_idx[idx] is_thead = getattr(row.row_group_key, "tag", None) == "thead" + is_carried_header = ( + is_thead and idx < self._header_row_count and self._should_repeat_headers + ) if row.max_rowspan is None: - row_reach = own_group_last - row_clipped = True + reach[idx] = own_group_last + clipped[idx] = True + bound[idx] = own_group_last - idx + 1 else: declared_reach = idx + row.max_rowspan - 1 - row_reach = min(declared_reach, own_group_last) - row_clipped = declared_reach > own_group_last and not is_thead - group_end = max(group_end, row_reach) - group_clipped = group_clipped or row_clipped + overreaches = declared_reach > own_group_last + reach[idx] = min(declared_reach, own_group_last) + clipped[idx] = overreaches and not is_carried_header + bound[idx] = ( + None if (overreaches and is_carried_header) else (own_group_last - idx + 1) + ) + + group_start = 0 + group_end = -1 # -- index of the furthest row any span opened so far reaches -- + for idx in range(n): + group_end = max(group_end, reach[idx]) if idx == group_end: - yield tuple(rows[group_start : idx + 1]), group_clipped + yield ( + tuple(rows[group_start : idx + 1]), + tuple(bound[group_start : idx + 1]), + any(clipped[group_start : idx + 1]), + ) group_start = idx + 1 - group_clipped = False # -- a span reaching past the last row of its own row-group (or `rowspan="0"`) leaves a # -- final group that never hits `idx == group_end` inside the loop; emit it rather than # -- drop it. It's always clipped -- that's exactly why it never closed on its own. -- - if group_start < len(rows): - yield tuple(rows[group_start:]), True + if group_start < n: + yield ( + tuple(rows[group_start:]), + tuple(bound[group_start:]), + True, + ) @staticmethod def _group_last_idx(rows: Sequence[HtmlRow]) -> list[int]: @@ -1822,21 +1861,36 @@ def __init__(self, maxlen: int, measure: Callable[[str], int] = len): self._maxlen = maxlen self._measure = measure self._rows: list[HtmlRow] = [] + self._bounds: list[int | None] = [] self._row_text_len = 0 self._has_clipped_group = False - def add_rows(self, rows: Sequence[HtmlRow], is_clipped: bool = False) -> None: + def add_rows( + self, + rows: Sequence[HtmlRow], + bounds: Sequence[int | None] | None = None, + is_clipped: bool = False, + ) -> None: """Add `rows` (a rowspan-bound group, possibly of length 1) to this accumulation. + `bounds` is `rows`' own per-row safe-rowspan-bound (see `_iter_rowspan_bound_row_groups`), + carried alongside so `flush()` can rewrite an overreaching cell's `rowspan` to match -- + the structural guarantee that an emitted span can never claim a row that isn't genuinely + present, independent of whatever else this accumulator goes on to hold. Omitted (`None`) + means "no bound for any of these rows" -- every declared span is trusted as-is, the + pre-existing behavior for callers unconcerned with row-group correctness. + `is_clipped` is whether the group's far edge was clipped by its own row-group boundary - (see `_iter_rowspan_bound_row_groups`) rather than reflecting a span's literal declared - value -- once true for any group in this accumulation, it stays true (a single clipped - group anywhere in the accumulated rows is enough to make extending across a row-group - boundary unsafe). + rather than reflecting a span's literal declared value -- once true for any group in this + accumulation, it stays true (a single clipped group anywhere in the accumulated rows is + enough to make extending across a row-group boundary unsafe -- see + `crosses_a_row_group_unsafely_if_extended`, a semantic nicety independent of `bounds`' + correctness guarantee). Caller is responsible for ensuring the group will fit. """ self._rows.extend(rows) + self._bounds.extend(bounds if bounds is not None else (None,) * len(rows)) self._row_text_len += self._measured_rows_text_len(rows) self._has_clipped_group = self._has_clipped_group or is_clipped @@ -1845,9 +1899,15 @@ def flush(self) -> Iterator[TextAndHtml]: if not self._rows: return text = " ".join(self._iter_cell_texts()) - trs_str = "".join(r.html for r in self._rows) + trs_str = "".join( + row.html_clipped_to_rows(bound) + if bound is not None and (row.max_rowspan is None or row.max_rowspan > bound) + else row.html + for row, bound in zip(self._rows, self._bounds) + ) html = f"{trs_str}
" self._rows.clear() + self._bounds.clear() self._row_text_len = 0 self._has_clipped_group = False yield text, html @@ -1857,29 +1917,32 @@ def will_fit(self, rows: Sequence[HtmlRow]) -> bool: return self._remaining_space >= self._measured_rows_text_len(rows) @property - def row_group_key(self) -> object | None: - """Row-group identity shared by the rows already accumulated, `None` if empty. - - A rowspan-bound group never itself spans two row-groups (`_iter_rowspan_bound_row_groups` - guarantees that), so any one row's key stands for the whole accumulation. + def last_row_group_key(self) -> object | None: + """Row-group identity of the most recently accumulated row, `None` if empty. + + Used to decide whether the NEXT group differs from what's already accumulated -- a + rowspan-bound group never itself spans two row-groups (`_iter_rowspan_bound_row_groups` + guarantees that), but the accumulation as a WHOLE can span several, one appended after + another, so it's the last one added -- not the first -- that the next comparison is + against. """ - return self._rows[0].row_group_key if self._rows else None + return self._rows[-1].row_group_key if self._rows else None @property def crosses_a_row_group_unsafely_if_extended(self) -> bool: - """True when appending a row from a DIFFERENT row-group could corrupt a span already - held here. - - True whenever any group accumulated so far was *clipped* by its own row-group boundary - (tracked via `add_rows`'s `is_clipped` argument) -- `rowspan="0"` (which is row-group- - scoped by the HTML spec, "spans every remaining row IN THE ROW GROUP") is always clipped - by construction, and so is a positive declared span (e.g. `rowspan="5"`) that named more - rows than its own row-group actually had. In either case the emitted cell's HTML still - carries the original, uncorrected declared value -- never rewritten to match the clip -- - and `HtmlTable.from_html_text` strips section wrappers when building the row model, so - appending a different row-group's rows after such a group would emit a flat, wrapper-less - `` in which that declared value legitimately -- by HTML's own rules, absent any - visible section boundary -- reaches into rows it was never meant to bind. + """True when appending a row from a DIFFERENT row-group would blend content that never + belonged together, even though `flush()`'s `bounds`-based rewrite (see `add_rows`) already + guarantees this can never corrupt column placement. + + This is a semantic boundary, not a correctness one: `rowspan="0"` (which is row-group- + scoped by the HTML spec, "spans every remaining row IN THE ROW GROUP") and a positive + declared span (e.g. `rowspan="5"`) that named more rows than its own row-group actually + had are both cases where the cell's declared value doesn't reflect a real, intentional + span past its own section -- `flush()` will rewrite the emitted `rowspan` to match reality + regardless of what gets appended here, so the only thing this property still protects + against is a clipped group's row-group getting visually absorbed into unrelated content + that happens to fit the same chunking window, which is undesirable even though it would no + longer be geometrically wrong. A span that fit entirely within its own row-group AS DECLARED (never clipped) is safe to extend across a row-group boundary -- its literal value already stops at the right row diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index a438f02471..230a726062 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -64,6 +64,12 @@ def iter_trs(rows_of_cell_strs: Sequence[Sequence[str]]) -> Iterator[str]: return f"
{''.join(iter_trs(matrix))}
" if matrix else "" +def _tr_html(cells: Sequence[SpannedCell]) -> str: + """Serialize one `` from `(cell_text, colspan, rowspan)` triples.""" + tds = (_format_td(text, colspan, rowspan) for text, colspan, rowspan in cells) + return f"{''.join(tds)}" + + def htmlify_matrix_of_spanned_cell_texts(matrix: Sequence[Sequence[SpannedCell]]) -> str: """Like `htmlify_matrix_of_cell_texts()` but each cell can also carry a colspan/rowspan. @@ -81,8 +87,7 @@ def iter_trs(rows: Sequence[Sequence[SpannedCell]]) -> Iterator[str]: # -- (its cells were already emitted there). Suppressing it would drop a ``, which # -- shifts the column-placement of every subsequent row under HTML's rowspan model # -- (rowspan counts actual `` elements, not "rows that happened to have content"). - tds = (_format_td(text, colspan, rowspan) for text, colspan, rowspan in row) - yield f"{''.join(tds)}" + yield _tr_html(row) return f"{''.join(iter_trs(matrix))}
" if matrix else "" @@ -335,6 +340,31 @@ def max_rowspan(self) -> int | None: return None return max((span for span in spans if span is not None), default=1) + def html_clipped_to_rows(self, max_rowspan: int) -> str: + """Serialize this row's ``, clipping any cell's `rowspan` down to `max_rowspan` when + its declared value (or `rowspan="0"`, HTML's "spans every remaining row") would otherwise + claim more rows than `max_rowspan` names. + + `max_rowspan` is supplied by the caller as the number of rows -- including this one -- + that are actually going to be present, in order, starting at this row in the emitted + fragment; this method has no notion of the wider table or chunking context. Passing the + row's own true remaining reach here is what makes the emitted `rowspan` self-correct: it + can never claim more rows than truly follow it, regardless of what a caller subsequently + decides to place after this row. + + Cells whose declared span already fits within `max_rowspan` are unaffected -- only an + actually-overreaching declaration is rewritten. + """ + cells = [ + ( + cell.text, + cell.colspan, + max_rowspan if cell.rowspan is None else min(cell.rowspan, max_rowspan), + ) + for cell in self.iter_cells() + ] + return _tr_html(cells) + class HtmlCell: """A `` element.""" @@ -365,3 +395,16 @@ def rowspan(self) -> int | None: except (TypeError, ValueError): return 1 return None if value == 0 else max(1, value) + + @cached_property + def colspan(self) -> int: + """Declared `colspan` for this cell, `1` when absent, unparseable, or non-positive. + + Unlike `rowspan`, HTML gives `colspan="0"` no special "spans every remaining column" + meaning, so it is simply treated as the default of `1`. + """ + try: + value = int(self._td.attrib.get("colspan", 1)) + except (TypeError, ValueError): + return 1 + return max(1, value) From 6329162b8524c3da09a71052f53c2915ecae57d8 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 21:25:22 -0500 Subject: [PATCH 10/19] fix: preserve cell content when correcting a clipped rowspan, and bound the thead row's original occurrence correctly html_clipped_to_rows() previously reconstructed each cell from its plain text when correcting an overreaching rowspan, discarding any nested table, hyperlink, image, or other markup/attributes the source cell carried. It now deep-copies the real and mutates only the rowspan attribute on cells that need correction, leaving everything else byte-for-byte unchanged. Separately, the carried-header exemption in _iter_rowspan_bound_row_groups was keyed on whether header repetition was configured at all, which incorrectly also exempted the thead row's own ORIGINAL, wrapper-less occurrence -- not just an actual repeated/carried copy (built separately by _as_header_row_html, wrapped in its own real , and never routed through this bound at all). Every row is now bounded to its own row-group uniformly; only a genuinely repeated copy escapes correction, because it was never subject to it in the first place. --- CHANGELOG.md | 6 ++ test_unstructured/chunking/test_base.py | 108 +++++++++++++++++++++++- unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 48 +++++------ unstructured/common/html_table.py | 28 +++--- 5 files changed, 152 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea30afd3a9..77d2d4878e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.13 + +### Fixes + +- **Preserve a cell's actual HTML when correcting a clipped or `rowspan="0"` `rowspan`, and bound the `` row's own original occurrence even when header repetition is configured.** Correcting an overreaching `rowspan` previously reconstructed the cell from its plain text, discarding any nested table, hyperlink, image, or other markup and attribute the source cell carried. Only the `rowspan` attribute is now rewritten on a copy of the real cell; everything else survives unchanged. Separately, a `` row's ORIGINAL, wrapper-less occurrence was being exempted from this correction whenever header repetition was merely configured — not only for an actual repeated/carried copy (a different artifact, already safely wrapped in its own real ``) — leaving that first occurrence still vulnerable to exactly the cross-row-group corruption this mechanism exists to prevent. The exemption is now scoped correctly: every row is bounded to its own row-group, full stop. + ## 0.27.12 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index c6f1dc8a4a..f88abddf01 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -4,9 +4,11 @@ from __future__ import annotations +import io import logging from typing import Any, Sequence +import pandas as pd import pytest from lxml.html import fragment_fromstring @@ -1577,7 +1579,17 @@ def and_it_preserves_source_header_row_html_for_carried_rows(self): repeat_table_headers=True, ) - assert len(chunks) == 3 + # -- 4 chunks, not 3: the header's own ORIGINAL occurrence is now correctly bounded to its + # -- own 1-row thead group (its declared rowspan="2" overreaches that group by one row), + # -- so it can no longer share a chunk with any body row and is isolated into its own + # -- leading chunk -- separate from (and unaffected by) the carried/repeated copies below -- + assert len(chunks) == 4 + original_html = chunks[0].metadata.text_as_html + assert original_html is not None + original_table = fragment_fromstring(original_html) + # -- the original occurrence's overreaching rowspan is corrected away (not left as "2") -- + assert original_table.xpath("./tr[1]/td[1]/@rowspan") == [] + continuation_html = chunks[1].metadata.text_as_html assert continuation_html is not None continuation_table = fragment_fromstring(continuation_html) @@ -3481,6 +3493,100 @@ def and_it_does_not_let_a_clipped_span_bind_a_later_direct_row_of_the_same_key(s ) assert chunks[1][1] == "
AfterY
" + def and_it_preserves_non_text_cell_content_when_correcting_a_clipped_rowspan(self): + """`HtmlRow.html_clipped_to_rows()` must only ever touch the `rowspan` attribute -- a + nested table, a hyperlink, an image-only cell, and any other cell attribute must survive + a correction completely unchanged. Reconstructing a cell from its plain `.text` (the + pre-fix behavior) would flatten all of this into concatenated text or an emptied ``. + """ + opts = ChunkingOptions(max_characters=100_000) + html_table = HtmlTable.from_html_text( + """ + + + + + + + + + +
Group +
Q1100
+ details + 		Chart
Otherrowcontent
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # -- `rowspan="0"` unconditionally goes through the correction path (its true reach is + # -- always rewritten to a literal count), which is exactly what exercises this cell's + # -- non-text content on every run -- not just when a chunk boundary happens to force it -- + assert chunks == [ + ( + "Group Q1100details Other row content", + "" + '' + "" + "" + "" + "
Group
Q1100
details
Otherrowcontent
", + ) + ] + + def and_it_bounds_the_theads_own_original_occurrence_even_when_repetition_is_configured( + self, + ): + """The carried-header exemption must apply only to a REPEATED copy + `_prepend_repeated_headers` injects onto a continuation chunk (a separate artifact built + from `row.source_html`/`row.html`, wrapped in its own real ``) -- never to the + header row's own ORIGINAL, wrapper-less occurrence. Before this fix, + `repeat_table_headers=True` alone was enough to exempt that original occurrence too, so + its overreaching `rowspan` could merge with body rows from a different row-group in the + same wrapper-less chunk and shift their columns. + + Verified by reparsing each chunk's own emitted HTML with `pandas.read_html` (which + correctly honors `rowspan`/`colspan` when building a grid) and checking that no body + value has been shifted into the wrong column -- a genuine geometry check, not just a + string/row-count comparison.""" + html = ( + "" + "" + '' + "" + "" + "" + "" + "" + "" + "
RegionQuarter
NWQ1
Southwest TerritoryQ2
Midwest TerritoryQ3
" + ) + text = "Region Quarter NW Q1 Southwest Territory Q2 Midwest Territory Q3" + table = Table(text, metadata=ElementMetadata(text_as_html=html)) + + chunks = chunk_by_title([table], max_characters=60, repeat_table_headers=True) + + assert len(chunks) > 1 + for chunk in chunks: + html_out = chunk.metadata.text_as_html + assert html_out is not None + grid = pd.read_html(io.StringIO(html_out))[0].to_numpy().tolist() + for row in grid: + if "NW" in row: + assert row[0] == "NW" + assert row[1] == "Q1" + if "Southwest Territory" in row: + assert row[0] == "Southwest Territory" + assert row[1] == "Q2" + if "Midwest Territory" in row: + assert row[0] == "Midwest Territory" + assert row[1] == "Q3" + # -- no cell text lost or duplicated across the whole set of chunks -- + combined_text = " ".join(chunk.text for chunk in chunks) + for word in ("NW", "Southwest", "Midwest"): + assert combined_text.count(word) == 1 + class Describe_TextSplitter: """Unit-test suite for `unstructured.chunking.base._TextSplitter` objects.""" diff --git a/unstructured/__version__.py b/unstructured/__version__.py index f1527a87a7..5cdb02ca0c 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.12" # pragma: no cover +__version__ = "0.27.13" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 791726f244..cec9b36362 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1309,7 +1309,7 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: def _iter_rowspan_bound_row_groups( self, - ) -> Iterator[tuple[tuple[HtmlRow, ...], tuple[int | None, ...], bool]]: + ) -> Iterator[tuple[tuple[HtmlRow, ...], tuple[int, ...], bool]]: """Group consecutive rows that a `rowspan` binds together. A row whose cell declares `rowspan=N` binds the next `N-1` rows to it (they carry that @@ -1329,9 +1329,8 @@ def _iter_rowspan_bound_row_groups( Each yielded group is paired with two things: - A same-length tuple of per-row **safe rowspan bounds**: for the row at position `p` - within the group, either the true number of rows (starting at that row, within its own - row-group) it may safely claim, or `None` when the row is an actively carried-forward - header row (see below) and should be emitted with its span exactly as declared. This is + within the group, the true number of rows (starting at that row, within its own + row-group) it may safely claim. This is consumed by `HtmlRow.html_clipped_to_rows()` at emission time and is the *structural* safety net: whatever else this class's grouping/accumulation logic decides to pack into the same chunk, an emitted `rowspan` can never claim a row that isn't genuinely present @@ -1344,43 +1343,38 @@ def _iter_rowspan_bound_row_groups( `rowspan="0"` is always clipped — there is no literal count for it to fall back on. A POSITIVE span (e.g. `rowspan="5"`) that declares more rows than its own row-group actually - has (`idx + max_rowspan - 1 > own_group_last`) is clipped the same way — UNLESS it is an - **actively carried-forward header row**: a `` row, within the configured header-row - count, whose header text is actually going to be repeated onto continuation chunks via - `_prepend_repeated_headers` (`self._should_repeat_headers`). That combination is already - established elsewhere in this codebase as intentionally extending a header's span past its - own section — see `and_it_preserves_source_header_row_html_for_carried_rows` — so such a - row is exempted from both the per-row bound (`None`, emit as declared) and the group-clip - flag. A `` row whose span merely happens to overreach with no active repetition - configured (e.g. `repeat_table_headers=False`) gets no such exemption: its bound is clipped - like any other, since nothing else protects it from binding rows it was never meant to - cover. A span that fits entirely within its own row-group as declared is never clipped — - its literal value already stops at the right row regardless of what follows. + has (`idx + max_rowspan - 1 > own_group_last`) is clipped the same way, with NO exemption + for a `` row even when header repetition is configured. Every row this function + iterates is the row's own ORIGINAL, single occurrence in the source table — repeated + copies injected onto continuation chunks are an entirely separate artifact, built by + `_as_header_row_html`/`_header_rows_html` from `row.source_html`/`row.html` and wrapped in + their own real `` element, never routed through this function's bounds or through + `HtmlRow.html_clipped_to_rows()` at all. So a repeated copy's span is already scoped by an + actual `` boundary in the HTML it's emitted into (see + `and_it_preserves_source_header_row_html_for_carried_rows`) and needs no exemption here; + exempting the ORIGINAL occurrence from its own bound — as an earlier version of this + function did, reasoning from "repetition is configured" rather than "this occurrence is a + repeated copy" — left that first, wrapper-less occurrence exposed to exactly the + cross-row-group corruption this function exists to prevent. A span that fits entirely + within its own row-group as declared is never clipped — its literal value already stops at + the right row regardless of what follows. """ rows = list(self._table_element.iter_rows()) n = len(rows) group_last_idx = self._group_last_idx(rows) reach = [0] * n clipped = [False] * n - bound: list[int | None] = [None] * n + bound: list[int] = [0] * n for idx, row in enumerate(rows): own_group_last = group_last_idx[idx] - is_thead = getattr(row.row_group_key, "tag", None) == "thead" - is_carried_header = ( - is_thead and idx < self._header_row_count and self._should_repeat_headers - ) + bound[idx] = own_group_last - idx + 1 if row.max_rowspan is None: reach[idx] = own_group_last clipped[idx] = True - bound[idx] = own_group_last - idx + 1 else: declared_reach = idx + row.max_rowspan - 1 - overreaches = declared_reach > own_group_last reach[idx] = min(declared_reach, own_group_last) - clipped[idx] = overreaches and not is_carried_header - bound[idx] = ( - None if (overreaches and is_carried_header) else (own_group_last - idx + 1) - ) + clipped[idx] = declared_reach > own_group_last group_start = 0 group_end = -1 # -- index of the furthest row any span opened so far reaches -- diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index 230a726062..980291d4b1 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -5,6 +5,7 @@ from __future__ import annotations +import copy import html from functools import cached_property from typing import TYPE_CHECKING, Iterator, Sequence, cast @@ -352,18 +353,23 @@ def html_clipped_to_rows(self, max_rowspan: int) -> str: can never claim more rows than truly follow it, regardless of what a caller subsequently decides to place after this row. - Cells whose declared span already fits within `max_rowspan` are unaffected -- only an - actually-overreaching declaration is rewritten. + Cells whose declared span already fits within `max_rowspan` are left completely untouched + (not even reserialized). A cell that needs correction has ONLY its `rowspan` attribute + rewritten (set to the corrected value, or removed entirely when the correction is `1`) -- + every other tag, child element, attribute, and cell content is preserved exactly as in the + source. This operates on a deep-copied `` so nested tables, links, images, and other + markup a naive text-only reconstruction would discard all survive unchanged. """ - cells = [ - ( - cell.text, - cell.colspan, - max_rowspan if cell.rowspan is None else min(cell.rowspan, max_rowspan), - ) - for cell in self.iter_cells() - ] - return _tr_html(cells) + tr = copy.deepcopy(self._tr) + for td in tr: + rowspan = HtmlCell(td).rowspan + if rowspan is not None and rowspan <= max_rowspan: + continue # -- already fits; leave this cell untouched -- + if max_rowspan <= 1: + td.attrib.pop("rowspan", None) + else: + td.attrib["rowspan"] = str(max_rowspan) + return etree.tostring(tr, encoding=str) class HtmlCell: From 47dcebbe407217e5a3a4145f9ff4936b5be9f767 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Thu, 3 Sep 2026 21:42:31 -0500 Subject: [PATCH 11/19] fix: bound a singleton oversized row's rowspan before cell-splitting A rowspan-bound row too large to fit any chunk even alone was handed to the cell splitter with its original, uncorrected rowspan attribute -- the self-correcting rewrite only applied to rows going through the normal row accumulator. reconstruct_table_from_chunks() can reassemble separately-emitted chunks, at which point the stale span reaches into rows from a later chunk. The row's own bound is now applied via row_clipped_to_rows() before it's split cell-by-cell. --- CHANGELOG.md | 6 ++ test_unstructured/chunking/test_base.py | 96 +++++++++++++++++++++++++ unstructured/__version__.py | 2 +- unstructured/chunking/base.py | 11 ++- unstructured/common/html_table.py | 28 ++++++-- 5 files changed, 136 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77d2d4878e..2f2da473e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.14 + +### Fixes + +- **Bound a singleton oversized row's `rowspan` before splitting it cell-by-cell.** A rowspan-bound row too large to fit any chunk even alone was handed to the cell splitter with its original, uncorrected `rowspan` still attached — the self-correcting rewrite only applied to rows that pass through the normal row accumulator. A public caller reassembling separately-emitted chunks (`reconstruct_table_from_chunks()`) could see that stale span reach into rows from a later chunk, the exact corruption this mechanism exists to prevent. The row's own bound is now applied before it's split. + ## 0.27.13 ### Fixes diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index f88abddf01..167ce8ea39 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3587,6 +3587,102 @@ def and_it_bounds_the_theads_own_original_occurrence_even_when_repetition_is_con for word in ("NW", "Southwest", "Midwest"): assert combined_text.count(word) == 1 + def and_it_bounds_a_positive_rowspan_in_a_multi_cell_oversized_singleton_row(self): + """A singleton rowspan-bound group whose ROW as a whole is too big for the chunking + window, but whose FIRST cell fits on its own, is emitted via `_CellAccumulator` (which + serializes a fitting cell's real `.html`, preserving its original `rowspan`) -- not via + `_iter_cell_splits` (which only ever emits plain, span-less text fragments and so was + never at risk here). This is the reachable case the "inert" reasoning missed: the row's + OWN, uncorrected declared span survives into that first sub-chunk unless it is bounded + before being handed to the row splitter.""" + opts = ChunkingOptions(max_characters=20) + html_table = HtmlTable.from_html_text( + """ + + + + + + + + + + +
Regionzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
NWQ1
SWQ2
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # -- the one-row thead's own bound is "1" (no other row in its group), so the header's + # -- first sub-chunk must carry no rowspan claim at all -- + assert chunks[0] == ("Region", "
Region
") + + def and_it_bounds_a_rowspan_0_in_a_multi_cell_oversized_singleton_row(self): + """Same reachable gap as the positive-rowspan case above, for `rowspan="0"` specifically + -- HTML's own "spans every remaining row" form, which this codebase always treats as + clipped (there is no literal count to fall back on).""" + opts = ChunkingOptions(max_characters=20) + html_table = HtmlTable.from_html_text( + """ + + + + + + +
Regionzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert chunks[0] == ("Region", "
Region
") + + def and_it_bounds_a_singleton_oversized_rows_span_through_chunk_by_title_and_reconstruction( + self, + ): + """End-to-end through the public `chunk_by_title()` entry point, then back through + `reconstruct_table_from_chunks()` -- the actual round-trip a caller performs -- reparsing + the reconstructed table with `pandas.read_html()` (which honors `rowspan`/`colspan` when + building a grid) to catch real column-shift corruption, not just inspect chunk strings.""" + html = ( + "" + "" + '" + "" + "" + "" + "" + "" + "
Region' + "z" * 150 + "
NWQ1
Southwest TerritoryQ2
" + ) + text = "Region " + "z" * 150 + " NW Q1 Southwest Territory Q2" + table = Table(text, metadata=ElementMetadata(text_as_html=html)) + + chunks = chunk_by_title([table], max_characters=50, repeat_table_headers=False) + [reconstructed] = reconstruct_table_from_chunks(chunks) + + html_out = reconstructed.metadata.text_as_html + assert html_out is not None + grid = pd.read_html(io.StringIO(html_out))[0].to_numpy().tolist() + # -- "Region"'s uncorrected `rowspan="3"` reaches exactly 3 rows deep (this row plus the + # -- next 2), not far enough to displace the much-later NW/Southwest Territory rows -- so + # -- this checks the actual corrupted rows directly, immediately following "Region", not + # -- rows the span happens not to reach in this particular fixture -- + assert grid[0][0] == "Region" + for row in grid[1:9]: + assert row[0] != "Region", f"a later row still carries Region's uncorrected span: {row}" + for row in grid: + if "NW" in row: + assert row == ["NW", "Q1"] + if "Southwest Territory" in row: + assert row == ["Southwest Territory", "Q2"] + # -- no cell text lost or duplicated across the whole reconstructed table -- + combined_text = reconstructed.text + for word in ("Region", "NW", "Southwest"): + assert combined_text.count(word) == 1 + class Describe_TextSplitter: """Unit-test suite for `unstructured.chunking.base._TextSplitter` objects.""" diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 5cdb02ca0c..05d19bf1a2 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.13" # pragma: no cover +__version__ = "0.27.14" # pragma: no cover diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index cec9b36362..0a9022ecc9 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1283,8 +1283,17 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: if accum.will_fit(group): accum.add_rows(group, group_bounds, is_clipped=group_is_clipped) elif len(group) == 1: # -- a single row is bigger than the chunking window -- + # -- Even though this row is emitted alone (no other row from this chunk is + # -- present for its span to misplace), its literal `rowspan` attribute must still + # -- be corrected: a public caller can later reassemble multiple separately-emitted + # -- chunks (`reconstruct_table_from_chunks()`), at which point an uncorrected span + # -- from this chunk would reach into whatever rows follow in the reassembled + # -- table. Bounding it here, exactly like the multi-row accumulator path, is what + # -- keeps the emitted HTML self-correcting independent of what a caller does with + # -- it afterward. -- + bounded_row = group[0].row_clipped_to_rows(group_bounds[0]) for text, html in self._iter_row_splits( - group[0], maxlen=self._maxlen(is_first_chunk) + bounded_row, maxlen=self._maxlen(is_first_chunk) ): yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index 980291d4b1..e84c2745b5 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -341,10 +341,8 @@ def max_rowspan(self) -> int | None: return None return max((span for span in spans if span is not None), default=1) - def html_clipped_to_rows(self, max_rowspan: int) -> str: - """Serialize this row's ``, clipping any cell's `rowspan` down to `max_rowspan` when - its declared value (or `rowspan="0"`, HTML's "spans every remaining row") would otherwise - claim more rows than `max_rowspan` names. + def _clipped_tr(self, max_rowspan: int) -> HtmlElement: + """A deep-copied `` with any over-reaching cell `rowspan` clipped to `max_rowspan`. `max_rowspan` is supplied by the caller as the number of rows -- including this one -- that are actually going to be present, in order, starting at this row in the emitted @@ -369,7 +367,27 @@ def html_clipped_to_rows(self, max_rowspan: int) -> str: td.attrib.pop("rowspan", None) else: td.attrib["rowspan"] = str(max_rowspan) - return etree.tostring(tr, encoding=str) + return tr + + def html_clipped_to_rows(self, max_rowspan: int) -> str: + """Serialize this row's ``, clipping any cell's `rowspan` down to `max_rowspan` when + its declared value (or `rowspan="0"`, HTML's "spans every remaining row") would otherwise + claim more rows than `max_rowspan` names. See `_clipped_tr()` for the clipping rules. + """ + return etree.tostring(self._clipped_tr(max_rowspan), encoding=str) + + def row_clipped_to_rows(self, max_rowspan: int) -> "HtmlRow": + """This row, with any over-reaching cell `rowspan` clipped to `max_rowspan` (see + `_clipped_tr()`), as a fresh `HtmlRow` -- for callers (like `_iter_row_splits()`'s + singleton-oversized-row path) that need to keep working with a row object rather than a + serialized string, e.g. to iterate its cells for further splitting. + """ + return HtmlRow( + self._clipped_tr(max_rowspan), + is_header=self._is_header, + source_html=self._source_html, + row_group_key=self._row_group_key, + ) class HtmlCell: From fbd28d16b257054d6e8bae5f73c174ff448c6ad3 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Fri, 4 Sep 2026 11:07:27 -0500 Subject: [PATCH 12/19] Squash the CHANGELOG/version noise from 10 review rounds into one entry Each review-fix round bumped __version__ and added its own CHANGELOG entry as part of that round's commit, leaving 9 separate release-note entries and a 9-patch version jump for what is one user-facing fix. Consolidated into a single 0.27.6 entry and a one-increment version bump. --- CHANGELOG.md | 50 +------------------------------------ unstructured/__version__.py | 2 +- 2 files changed, 2 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f2da473e1..2404a59a57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,56 +1,8 @@ -## 0.27.14 - -### Fixes - -- **Bound a singleton oversized row's `rowspan` before splitting it cell-by-cell.** A rowspan-bound row too large to fit any chunk even alone was handed to the cell splitter with its original, uncorrected `rowspan` still attached — the self-correcting rewrite only applied to rows that pass through the normal row accumulator. A public caller reassembling separately-emitted chunks (`reconstruct_table_from_chunks()`) could see that stale span reach into rows from a later chunk, the exact corruption this mechanism exists to prevent. The row's own bound is now applied before it's split. - -## 0.27.13 - -### Fixes - -- **Preserve a cell's actual HTML when correcting a clipped or `rowspan="0"` `rowspan`, and bound the `` row's own original occurrence even when header repetition is configured.** Correcting an overreaching `rowspan` previously reconstructed the cell from its plain text, discarding any nested table, hyperlink, image, or other markup and attribute the source cell carried. Only the `rowspan` attribute is now rewritten on a copy of the real cell; everything else survives unchanged. Separately, a `` row's ORIGINAL, wrapper-less occurrence was being exempted from this correction whenever header repetition was merely configured — not only for an actual repeated/carried copy (a different artifact, already safely wrapped in its own real ``) — leaving that first occurrence still vulnerable to exactly the cross-row-group corruption this mechanism exists to prevent. The exemption is now scoped correctly: every row is bounded to its own row-group, full stop. - -## 0.27.12 - -### Fixes - -- **Rewrite chunked-table `rowspan` handling to self-correct emitted span values, instead of tracking which chunk boundaries are unsafe to cross.** A clipped or `rowspan="0"` cell's emitted HTML previously always carried its original declared value, relying on chunk-boundary bookkeeping to keep it away from rows it didn't truly cover. That value is now rewritten, at the point each chunk is assembled, to the number of rows actually present in that same chunk — an emitted `rowspan` can no longer overreach regardless of what else the chunker decides to pack alongside it. Also fixes two remaining gaps in the boundary bookkeeping this replaces: an overdeclared `` `rowspan` was exempted from clipping even when header repetition isn't active for it (so nothing else protected it), and the accumulator compared a candidate row-group against the FIRST row it had accumulated rather than the most recent one, which could miss a real transition when an earlier and later row happened to share the same row-group identity. - -## 0.27.11 - -### Fixes - -- **Also treat a clipped positive `rowspan` as a hard chunk boundary, not only `rowspan="0"`.** A positive `rowspan` that declares more rows than its own ``/``/`` row-group actually has is clipped to that row-group when grouping rows for chunking, but its emitted HTML still carries the original, uncorrected declared value. If such a group got packed into the same chunk as a following row-group's rows, that value would legitimately reach into rows it was never meant to bind, once section wrappers are stripped. A row-group change is now also a hard chunk boundary whenever the preceding group's span was clipped this way — a `` row's positive span is unaffected, since it's already handled separately as a repeated/carried-forward header. - -## 0.27.10 - -### Fixes - -- **Never let a chunk's rows span more than one source ``/``/`` row-group when a `rowspan="0"` cell is involved.** Row-group boundaries were correctly used to compute rowspan-bound groups, but the chunk accumulator could still pack an already-grouped `rowspan="0"` header together with a following row-group's rows when both fit the character budget — since section wrappers are stripped from emitted chunk HTML, the reparsed, flattened result let that span reach into rows it was never meant to bind. A row-group change is now a hard chunk boundary whenever a `rowspan="0"` row is involved; ordinary rows (and rows with a real, positive `rowspan` meant to carry across a boundary, e.g. a repeated header) are unaffected. - -## 0.27.9 - -### Fixes - -- **Scope `rowspan` chunk-boundary protection to its actual ``/``/`` row-group, not the whole table.** A `rowspan="0"` cell in a short `` (or any positive `rowspan`) was previously resolved as reaching to the end of the entire table rather than the end of its own row-group, so a real, bounded `` following it could get swallowed into one unbounded chunk alongside the header. Rows are now grouped by their actual containing section (or the whole table, when there is none), and a span can no longer bind rows across a real section boundary. - -## 0.27.8 - -### Fixes - -- **Don't drop a table's trailing rows when a `rowspan` reaches past the last row, and honor `rowspan="0"`.** The rowspan-aware chunking boundary added in 0.27.7 could silently lose an entire table's tail when a declared `rowspan` named more rows than the table had, and treated `rowspan="0"` (HTML's "span every remaining row") as no span at all, letting a chunk boundary fall through it. Both now resolve to "the rest of the table" and are never split or dropped. - -## 0.27.7 - -### Fixes - -- **Keep rows bound by an active `rowspan` in the same table chunk.** Table chunking split purely on row *text* length, unaware that a `rowspan` crossing the boundary would overclaim rows in one chunk and shift cells into the wrong column in the next. Such rows are now kept together as one unit, same tolerance already given a single oversized row or cell. - ## 0.27.6 ### Fixes -- **Stop duplicating merged-cell text in DOCX `text_as_html`.** A merged cell (`gridSpan`/`vMerge`) was repeated into every `` its merge visually covered, with no `colspan`/`rowspan` attribute marking the merge. Merged cells are now emitted once, with `colspan`/`rowspan` reflecting the true geometry. +- **Stop duplicating merged-cell text in DOCX `text_as_html`.** A merged cell (`gridSpan`/`vMerge`) was repeated into every `` its merge visually covered, with no `colspan`/`rowspan` attribute marking the merge; merged cells are now emitted once, with `colspan`/`rowspan` reflecting the true geometry. Since DOCX tables can now carry real spans, table chunking was also made rowspan-aware, so a chunk boundary can no longer split a table in a way that misattributes a spanned cell's rows to the wrong columns. ## 0.27.5 diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 05d19bf1a2..a71228fe17 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.14" # pragma: no cover +__version__ = "0.27.6" # pragma: no cover From 6e37aa8ea95fcd98e991448f1736eb57ea48e65a Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Fri, 4 Sep 2026 11:16:02 -0500 Subject: [PATCH 13/19] Trim excessive comments and docstrings added across the review-fix rounds --- unstructured/chunking/base.py | 148 +++++++----------------------- unstructured/common/html_table.py | 71 +++++--------- 2 files changed, 56 insertions(+), 163 deletions(-) diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 0a9022ecc9..7832006c7e 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1240,26 +1240,18 @@ def iter_subtables( def _iter_subtables(self) -> Iterator[TextAndHtml]: """Generate (text, html) pairs containing as many whole rows as will fit in window. - Rows joined by an active `rowspan` are kept together as one atomic group — splitting - between them would leave a `rowspan` whose declared count exceeds the rows actually - present in its chunk, and would shift every following row in the continuation chunk into - the wrong column (that chunk's `` has no earlier row to carry the span forward). - Falls back to splitting rows into whole cells when a single row (or, when rowspan-bound, - a whole such group) is by itself too big to fit in the chunking window. + Rows joined by an active `rowspan` are kept together as one atomic group, since splitting + them would leave a `rowspan` overclaiming rows and misplace every following row's columns. + Falls back to splitting into whole cells when a single row (or rowspan-bound group) is by + itself too big to fit in the chunking window. """ is_first_chunk = True accum = _RowAccumulator(maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure) for group, group_bounds, group_is_clipped in self._iter_rowspan_bound_row_groups(): - # -- Crossing a row-group boundary is only unsafe when a span already accumulated - # -- was clipped by ITS OWN row-group boundary (see - # -- `crosses_a_row_group_unsafely_if_extended`) -- an ordinary row, or a span that - # -- fits entirely within its own row-group as declared, is always safe to pack with - # -- whatever comes next, row-group or not, and forcing a flush there would needlessly - # -- fragment perfectly normal tables that were never at risk. This check is a - # -- semantic nicety, not a correctness requirement: `group_bounds` (below) is what - # -- actually guarantees an emitted `rowspan` can never overreach, independent of - # -- whether this flush decision fires correctly. + # -- flush before crossing a row-group boundary only if a clipped span is already + # -- accumulated (see `crosses_a_row_group_unsafely_if_extended`); `group_bounds` + # -- below is what actually guarantees an emitted rowspan can never overreach -- if ( accum.last_row_group_key is not None and group[0].row_group_key is not accum.last_row_group_key @@ -1283,14 +1275,9 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: if accum.will_fit(group): accum.add_rows(group, group_bounds, is_clipped=group_is_clipped) elif len(group) == 1: # -- a single row is bigger than the chunking window -- - # -- Even though this row is emitted alone (no other row from this chunk is - # -- present for its span to misplace), its literal `rowspan` attribute must still - # -- be corrected: a public caller can later reassemble multiple separately-emitted - # -- chunks (`reconstruct_table_from_chunks()`), at which point an uncorrected span - # -- from this chunk would reach into whatever rows follow in the reassembled - # -- table. Bounding it here, exactly like the multi-row accumulator path, is what - # -- keeps the emitted HTML self-correcting independent of what a caller does with - # -- it afterward. -- + # -- bound the span even though this row is emitted alone: a caller reassembling + # -- chunks later (`reconstruct_table_from_chunks()`) would otherwise see it reach + # -- into whatever rows follow in the reassembled table -- bounded_row = group[0].row_clipped_to_rows(group_bounds[0]) for text, html in self._iter_row_splits( bounded_row, maxlen=self._maxlen(is_first_chunk) @@ -1321,52 +1308,17 @@ def _iter_rowspan_bound_row_groups( ) -> Iterator[tuple[tuple[HtmlRow, ...], tuple[int, ...], bool]]: """Group consecutive rows that a `rowspan` binds together. - A row whose cell declares `rowspan=N` binds the next `N-1` rows to it (they carry that - cell's continuation and would misplace their own cells, or overclaim the span's row - count, if split into a different chunk). Spans starting in different rows of the same - group can reach further than the row that opened the group, so the group's far edge is - the max reach of every span opened before it closes — the standard overlapping-interval - merge. A run of rows with no multi-row `rowspan` at all yields one-row groups, identical - to the pre-grouping behavior. - - A declared span can reach past the last row the table actually has (a malformed but - browser-tolerated document, which clips it to the rows present) or be `rowspan="0"` - (spans every remaining row) — both resolve to "the rest of the table's own row-group" - here (see `_group_last_idx`), and the final, possibly-still-open group is always yielded - rather than silently dropped. - - Each yielded group is paired with two things: - - - A same-length tuple of per-row **safe rowspan bounds**: for the row at position `p` - within the group, the true number of rows (starting at that row, within its own - row-group) it may safely claim. This is - consumed by `HtmlRow.html_clipped_to_rows()` at emission time and is the *structural* - safety net: whatever else this class's grouping/accumulation logic decides to pack into - the same chunk, an emitted `rowspan` can never claim a row that isn't genuinely present - in that same chunk, because it is rewritten to match this bound regardless. - - A `bool`: whether the group's far edge was *clipped* by its own row-group boundary in a - way that makes it unsafe to *also* pack a different row-group's rows into the same - chunk (used only by `_RowAccumulator.crosses_a_row_group_unsafely_if_extended` to decide - when to keep a clipped group isolated in its own chunk, a semantic — not correctness — - concern now that the per-row bounds above independently guarantee correctness). - - `rowspan="0"` is always clipped — there is no literal count for it to fall back on. A - POSITIVE span (e.g. `rowspan="5"`) that declares more rows than its own row-group actually - has (`idx + max_rowspan - 1 > own_group_last`) is clipped the same way, with NO exemption - for a `` row even when header repetition is configured. Every row this function - iterates is the row's own ORIGINAL, single occurrence in the source table — repeated - copies injected onto continuation chunks are an entirely separate artifact, built by - `_as_header_row_html`/`_header_rows_html` from `row.source_html`/`row.html` and wrapped in - their own real `` element, never routed through this function's bounds or through - `HtmlRow.html_clipped_to_rows()` at all. So a repeated copy's span is already scoped by an - actual `` boundary in the HTML it's emitted into (see - `and_it_preserves_source_header_row_html_for_carried_rows`) and needs no exemption here; - exempting the ORIGINAL occurrence from its own bound — as an earlier version of this - function did, reasoning from "repetition is configured" rather than "this occurrence is a - repeated copy" — left that first, wrapper-less occurrence exposed to exactly the - cross-row-group corruption this function exists to prevent. A span that fits entirely - within its own row-group as declared is never clipped — its literal value already stops at - the right row regardless of what follows. + A row whose cell declares `rowspan=N` binds the next `N-1` rows to it, since splitting + them across chunks would misplace their cells or overclaim the span's row count. A group's + far edge is the max reach of every span opened within it (standard overlapping-interval + merge); a span reaching past its own row-group's last row, or `rowspan="0"`, clips to that + row-group's end (see `_group_last_idx`) and is always yielded rather than dropped. + + Yields, per group: the rows, a same-length tuple of per-row safe rowspan bounds (consumed + by `HtmlRow.html_clipped_to_rows()` so an emitted rowspan can never claim a row that isn't + actually present in its chunk), and a bool for whether the group's far edge was clipped + (used by `_RowAccumulator.crosses_a_row_group_unsafely_if_extended` to avoid packing a + clipped group together with a different row-group's rows). """ rows = list(self._table_element.iter_rows()) n = len(rows) @@ -1410,11 +1362,8 @@ def _iter_rowspan_bound_row_groups( def _group_last_idx(rows: Sequence[HtmlRow]) -> list[int]: """For each row-index in `rows`, the index of the last row sharing its row-group. - Rows are grouped by identity of `HtmlRow.row_group_key` (a specific ``/``/ - `` element, or the `
` itself for a row with no section wrapper), so a - `rowspan` — including `rowspan="0"`, HTML's "spans every remaining row in the row group" - — can never bind rows across a real section boundary. A table with no explicit sections - has exactly one row-group (the whole table), matching the simpler pre-row-group behavior. + Rows are grouped by identity of `HtmlRow.row_group_key`, so a `rowspan` can never bind + rows across a real ``/``/`` boundary. """ n = len(rows) last_idx = [0] * n @@ -1877,18 +1826,10 @@ def add_rows( """Add `rows` (a rowspan-bound group, possibly of length 1) to this accumulation. `bounds` is `rows`' own per-row safe-rowspan-bound (see `_iter_rowspan_bound_row_groups`), - carried alongside so `flush()` can rewrite an overreaching cell's `rowspan` to match -- - the structural guarantee that an emitted span can never claim a row that isn't genuinely - present, independent of whatever else this accumulator goes on to hold. Omitted (`None`) - means "no bound for any of these rows" -- every declared span is trusted as-is, the - pre-existing behavior for callers unconcerned with row-group correctness. - - `is_clipped` is whether the group's far edge was clipped by its own row-group boundary - rather than reflecting a span's literal declared value -- once true for any group in this - accumulation, it stays true (a single clipped group anywhere in the accumulated rows is - enough to make extending across a row-group boundary unsafe -- see - `crosses_a_row_group_unsafely_if_extended`, a semantic nicety independent of `bounds`' - correctness guarantee). + carried so `flush()` can rewrite an overreaching cell's `rowspan` to match; `None` means no + bound applies and every declared span is trusted as-is. `is_clipped` marks whether the + group's far edge was clipped by its own row-group boundary; once set for this accumulation + it stays set (see `crosses_a_row_group_unsafely_if_extended`). Caller is responsible for ensuring the group will fit. """ @@ -1921,37 +1862,18 @@ def will_fit(self, rows: Sequence[HtmlRow]) -> bool: @property def last_row_group_key(self) -> object | None: - """Row-group identity of the most recently accumulated row, `None` if empty. - - Used to decide whether the NEXT group differs from what's already accumulated -- a - rowspan-bound group never itself spans two row-groups (`_iter_rowspan_bound_row_groups` - guarantees that), but the accumulation as a WHOLE can span several, one appended after - another, so it's the last one added -- not the first -- that the next comparison is - against. - """ + """Row-group identity of the most recently accumulated row, `None` if empty.""" return self._rows[-1].row_group_key if self._rows else None @property def crosses_a_row_group_unsafely_if_extended(self) -> bool: - """True when appending a row from a DIFFERENT row-group would blend content that never - belonged together, even though `flush()`'s `bounds`-based rewrite (see `add_rows`) already - guarantees this can never corrupt column placement. - - This is a semantic boundary, not a correctness one: `rowspan="0"` (which is row-group- - scoped by the HTML spec, "spans every remaining row IN THE ROW GROUP") and a positive - declared span (e.g. `rowspan="5"`) that named more rows than its own row-group actually - had are both cases where the cell's declared value doesn't reflect a real, intentional - span past its own section -- `flush()` will rewrite the emitted `rowspan` to match reality - regardless of what gets appended here, so the only thing this property still protects - against is a clipped group's row-group getting visually absorbed into unrelated content - that happens to fit the same chunking window, which is undesirable even though it would no - longer be geometrically wrong. - - A span that fit entirely within its own row-group AS DECLARED (never clipped) is safe to - extend across a row-group boundary -- its literal value already stops at the right row - regardless of what follows (e.g. a `` header row's `rowspan="2"` correctly repeated - /carried forward into the body). Flushing on those would fragment ordinary, correct tables - that were never at risk -- see `and_it_preserves_source_header_row_html_for_carried_rows`. + """True when appending a row from a different row-group would blend unrelated content. + + A semantic boundary, not a correctness one -- `flush()`'s bounds-based rewrite already + guarantees column placement can't corrupt. Only true once a clipped group (one whose far + edge didn't reflect its span's literal declared value) has been accumulated; an unclipped + span is safe to extend across a row-group boundary, since its literal value already stops + at the right row regardless of what follows. """ return self._has_clipped_group diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index e84c2745b5..d91628b02f 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -74,20 +74,15 @@ def _tr_html(cells: Sequence[SpannedCell]) -> str: def htmlify_matrix_of_spanned_cell_texts(matrix: Sequence[Sequence[SpannedCell]]) -> str: """Like `htmlify_matrix_of_cell_texts()` but each cell can also carry a colspan/rowspan. - Each row of `matrix` is a sequence of `(cell_text, colspan, rowspan)` triples, one for each - grid-position that is the top-left corner of a (possibly 1x1) cell. A grid-position covered by - the colspan/rowspan of an earlier cell (in the same row or a prior row) must simply be omitted - from `matrix` by the caller; this function has no notion of the overall grid-shape, only of the - cells it is told to emit. + Each row of `matrix` is a sequence of `(cell_text, colspan, rowspan)` triples for each + grid-position that is the top-left corner of a cell; a caller must omit any grid-position + covered by an earlier cell's colspan/rowspan. """ def iter_trs(rows: Sequence[Sequence[SpannedCell]]) -> Iterator[str]: for row in rows: - # -- Unlike `htmlify_matrix_of_cell_texts()`, an empty row here is NOT suppressed: it - # -- represents a real grid-row entirely covered by a `rowspan` from an earlier row - # -- (its cells were already emitted there). Suppressing it would drop a ``, which - # -- shifts the column-placement of every subsequent row under HTML's rowspan model - # -- (rowspan counts actual `` elements, not "rows that happened to have content"). + # -- an empty row is a real grid-row fully covered by a prior row's rowspan, and must + # -- still emit a `` to keep the rowspan's row-count accounting correct -- yield _tr_html(row) return f"
{''.join(iter_trs(matrix))}
" if matrix else "" @@ -98,20 +93,13 @@ def collapse_matrix_of_keyed_cells_to_spans( ) -> list[list[SpannedCell]]: """Collapse a full row/column grid of `(cell_text, merge_key)` cells into merged spans. - `matrix` must be "rectangular" in the sense that it represents every grid-position of the - table, including positions covered by a merge, unlike the `matrix` consumed by - `htmlify_matrix_of_spanned_cell_texts()`. Two grid-positions belong to the same merged region - exactly when their `merge_key` compares equal with `==`; a grid-position that is not merged - with any other must be given a `merge_key` that compares equal only to itself (e.g. a unique - `object()` instance). + `matrix` must cover every grid-position of the table, including ones covered by a merge. + Grid-positions sharing an `==`-equal `merge_key` belong to the same merged region; give an + unmerged position a `merge_key` unique to itself (e.g. a fresh `object()`). Only rectangular + merged regions are supported. - Only rectangular merged regions are supported (as is guaranteed by, e.g., DOCX and XLSX merge - semantics) -- an "L-shaped" or otherwise irregular region of matching keys produces undefined - (but not exception-raising) results. - - Returns one row per row of `matrix`, each containing a `(cell_text, colspan, rowspan)` triple - for each cell that "originates" a merged region (or an unmerged 1x1 cell), in left-to-right - order. A grid-position covered by the colspan/rowspan of such a cell is omitted. + Returns one row per row of `matrix`, each holding a `(cell_text, colspan, rowspan)` triple for + every cell that originates a region (or unmerged 1x1 cell); covered positions are omitted. """ n_rows = len(matrix) consumed = [[False] * len(row) for row in matrix] @@ -183,11 +171,8 @@ def from_html_text(cls, html_text: str) -> HtmlTable: for idx, tr in enumerate(rows) if tr.getparent().tag == "thead" or bool(tr.xpath("./th")) } - # -- Each row's row-group is identified by its immediate parent element: a specific - # -- ``/``/`` when present, or the `` itself for a row with no - # -- section wrapper. Captured now (identity survives the `.drop_tag()` below even though - # -- the dropped element becomes detached) so a `rowspan` can later be prevented from - # -- binding rows across a real section boundary. + # -- row-group identity is each row's parent element (a ``/``/``, or + # -- the `
` itself); captured now since it survives `.drop_tag()` below -- row_group_keys = tuple(tr.getparent() for tr in rows) # -- remove ``, ``, and `` noise elements when present -- @@ -304,12 +289,10 @@ def source_html(self) -> str | None: @property def row_group_key(self) -> object: - """Identity of this row's containing row-group (a ``/``/`` element, - or the `
` itself for a row with no section wrapper). + """Identity of this row's containing row-group, for `rowspan` grouping purposes. - Two rows compare equal on this value (`is`) exactly when they belong to the same row-group - for `rowspan` purposes. `None` when unknown (e.g. an `HtmlRow` constructed directly rather - than via `HtmlTable.iter_rows()`), in which case all such rows are treated as one group. + `None` when unknown (e.g. an `HtmlRow` constructed directly rather than via + `HtmlTable.iter_rows()`), in which case all such rows are treated as one group. """ return self._row_group_key @@ -344,19 +327,9 @@ def max_rowspan(self) -> int | None: def _clipped_tr(self, max_rowspan: int) -> HtmlElement: """A deep-copied `` with any over-reaching cell `rowspan` clipped to `max_rowspan`. - `max_rowspan` is supplied by the caller as the number of rows -- including this one -- - that are actually going to be present, in order, starting at this row in the emitted - fragment; this method has no notion of the wider table or chunking context. Passing the - row's own true remaining reach here is what makes the emitted `rowspan` self-correct: it - can never claim more rows than truly follow it, regardless of what a caller subsequently - decides to place after this row. - - Cells whose declared span already fits within `max_rowspan` are left completely untouched - (not even reserialized). A cell that needs correction has ONLY its `rowspan` attribute - rewritten (set to the corrected value, or removed entirely when the correction is `1`) -- - every other tag, child element, attribute, and cell content is preserved exactly as in the - source. This operates on a deep-copied `` so nested tables, links, images, and other - markup a naive text-only reconstruction would discard all survive unchanged. + `max_rowspan` is the number of rows, including this one, actually present starting here in + the emitted fragment. Only an over-reaching cell's `rowspan` attribute is rewritten (or + removed, when the correction is `1`); everything else is preserved unchanged. """ tr = copy.deepcopy(self._tr) for td in tr: @@ -377,10 +350,8 @@ def html_clipped_to_rows(self, max_rowspan: int) -> str: return etree.tostring(self._clipped_tr(max_rowspan), encoding=str) def row_clipped_to_rows(self, max_rowspan: int) -> "HtmlRow": - """This row, with any over-reaching cell `rowspan` clipped to `max_rowspan` (see - `_clipped_tr()`), as a fresh `HtmlRow` -- for callers (like `_iter_row_splits()`'s - singleton-oversized-row path) that need to keep working with a row object rather than a - serialized string, e.g. to iterate its cells for further splitting. + """This row, with any over-reaching cell `rowspan` clipped to `max_rowspan`, as a fresh + `HtmlRow` -- for callers that need a row object rather than a serialized string. """ return HtmlRow( self._clipped_tr(max_rowspan), From 1aa5dae2f8fb2932cee3b0382ea264d617915640 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Fri, 4 Sep 2026 12:59:05 -0500 Subject: [PATCH 14/19] Trim history-narrating comments down to stating the current test contract Comments that compared a test's current assertion to what it used to be (e.g. "4 chunks, not 3", "now correctly recognized", "the pre-fix behavior") are rewritten to describe only the current contract, or removed where the assertion/code already made the point without a comment. --- test_unstructured/chunking/test_base.py | 89 ++++++------------------ test_unstructured/partition/test_docx.py | 5 +- unstructured/chunking/base.py | 2 - 3 files changed, 24 insertions(+), 72 deletions(-) diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 167ce8ea39..7334067a20 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -1579,15 +1579,11 @@ def and_it_preserves_source_header_row_html_for_carried_rows(self): repeat_table_headers=True, ) - # -- 4 chunks, not 3: the header's own ORIGINAL occurrence is now correctly bounded to its - # -- own 1-row thead group (its declared rowspan="2" overreaches that group by one row), - # -- so it can no longer share a chunk with any body row and is isolated into its own - # -- leading chunk -- separate from (and unaffected by) the carried/repeated copies below -- + # -- the header's one-row thead group is isolated into its own leading chunk -- assert len(chunks) == 4 original_html = chunks[0].metadata.text_as_html assert original_html is not None original_table = fragment_fromstring(original_html) - # -- the original occurrence's overreaching rowspan is corrected away (not left as "2") -- assert original_table.xpath("./tr[1]/td[1]/@rowspan") == [] continuation_html = chunks[1].metadata.text_as_html @@ -3167,17 +3163,11 @@ def and_it_bounds_a_rowspan_0_header_to_its_own_thead_instead_of_the_whole_table chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- bounded into many chunks, not one ~2KB chunk holding the header + all 50 body rows -- assert len(chunks) > 1 for _, html in chunks: assert len(html) < 300 - # -- the header's own one-row group is emitted ALONE -- not merely first, with body rows - # -- trailing along behind it in the same chunk. Its `rowspan="0"` is also rewritten to - # -- match: within its own one-row group it claims no further rows at all, so the emitted - # -- cell carries no `rowspan` attribute (the implicit default is 1) rather than the - # -- ambiguous "0" -- self-consistent even if this chunk were inspected on its own -- + # -- the header's one-row group is emitted alone, with an implicit rowspan (no attribute) -- assert chunks[0][1] == "
Header
" - # -- no body row leaked into the header's chunk -- assert chunks[0][1].count("") == 1 def and_it_bounds_a_rowspan_0_header_even_when_a_huge_window_would_otherwise_merge_sections( @@ -3199,8 +3189,6 @@ def and_it_bounds_a_rowspan_0_header_even_when_a_huge_window_would_otherwise_mer chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- two chunks: the header's own row-group, then every body row (which itself fits the - # -- huge window as one chunk, since nothing bounds it from below except its own group) -- assert len(chunks) == 2 assert chunks[0][1] == "
Header
" assert chunks[1][1].count("") == 50 @@ -3226,9 +3214,6 @@ def and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group(self): for _, html in chunks: reparsed = HtmlTable.from_html_text(html) rows = list(reparsed.iter_rows()) - # -- a chunk is emitted wrapper-less, so it is itself exactly one row-group; every - # -- row's declared rowspan must therefore resolve within THIS chunk's own row count, - # -- never claiming more rows than the chunk actually contains -- for idx, row in enumerate(rows): if row.max_rowspan is not None: assert idx + row.max_rowspan - 1 < len(rows) @@ -3256,8 +3241,7 @@ def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- the tbody's 2-row group (bound by the rowspan, clipped to the tbody's own last row, - # -- not the declared rowspan=5) is one chunk; the tfoot row is independent and separate -- + # -- rowspan is clipped to the tbody's own 2-row count; the tfoot row is a separate chunk -- assert chunks == [ ( "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", @@ -3299,9 +3283,7 @@ def and_it_clips_a_positive_rowspan_even_when_a_huge_window_would_otherwise_merg chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- the tbody's clipped 2-row group is its own chunk (declared rowspan=5, rewritten to - # -- "2", its own row-group's true count); the tfoot's rows are a separate chunk -- - # -- despite an enormous window that would gladly merge both into one -- + # -- rowspan is clipped to the tbody's row count; the tfoot rows stay a separate chunk -- assert chunks == [ ( "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", @@ -3320,15 +3302,9 @@ def and_it_clips_a_positive_rowspan_even_when_a_huge_window_would_otherwise_merg ] def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self): - """Same clipped-positive-rowspan protection, exercised through the actual public - `chunk_by_title()` entry point rather than only the internal splitter class. - - The overdeclared `rowspan="5"` is rewritten to match the tbody's own true row count, so - `and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group`'s general reparse - check would in fact also catch a regression here -- this test additionally checks the - thing that check can't see on its own: that tbody and tfoot content never land in the - same chunk in the first place, which is what the row-group-boundary flush (not the - rewrite) is responsible for.""" + """Same clipped-positive-rowspan protection, exercised through the public + `chunk_by_title()` entry point: tbody and tfoot content must never land in the same + chunk, and the rewritten rowspan must match the tbody's own row count.""" html = ( "" "" @@ -3356,9 +3332,8 @@ def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self) html = chunk.metadata.text_as_html assert html.startswith("
") assert html.endswith("
") - # -- the tbody group and the tfoot group never land in the same chunk; if they did, - # -- the tbody's rowspan="5" would legitimately (per HTML's own rules, once section - # -- wrappers are stripped) reach into the tfoot rows and shift them a column over -- + # -- tbody and tfoot content must never share a chunk, or the tbody's rowspan="5" + # -- would reach into the tfoot rows and shift them a column over -- has_tbody_content = "golf" in html has_tfoot_content = "juliet" in html or "mike" in html assert not (has_tbody_content and has_tfoot_content) @@ -3368,10 +3343,9 @@ def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self) assert combined_text.count(word) == 1 def and_it_still_spans_the_whole_group_for_a_sectionless_rowspan_0_table(self): - """The original motivating case (no explicit ``/``/``) must keep - working exactly as before row-groups were introduced: a table with no section wrapper is - itself one row-group, so `rowspan="0"` still reaches every row the table has -- emitted - as the literal count ("2") rather than the ambiguous "0".""" + """A table with no explicit ``/``/`` is itself one row-group, so + `rowspan="0"` reaches every row the table has -- emitted as the literal count ("2") + rather than the ambiguous "0".""" opts = ChunkingOptions(max_characters=15) html_table = HtmlTable.from_html_text( """ @@ -3433,10 +3407,7 @@ def and_it_clips_an_overdeclared_thead_rowspan_when_header_repetition_is_not_act """ ) - # -- header_row_count=0 (the default) means repetition is never configured, so the - # -- `` row gets no carry-forward exemption regardless of `repeat_table_headers`, - # -- and (now correctly recognized as clipped) is kept isolated from the tbody rows by the - # -- same row-group-boundary flush that already protects ``/`` spans -- + # -- header_row_count=0 (the default) means no repeat-header exemption applies here -- chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) assert len(chunks) == 2 @@ -3446,9 +3417,7 @@ def and_it_clips_an_overdeclared_thead_rowspan_when_header_repetition_is_not_act for idx, row in enumerate(rows): if row.max_rowspan is not None: assert idx + row.max_rowspan - 1 < len(rows) - # -- the header's declared "3" is additionally clipped down to "1" by the same rewrite - # -- that protects every other row-group (its own thead has just the one row), so even - # -- inspected on its own the header's chunk carries no false claim over body content -- + # -- the header's declared "3" is clipped to "1" (its own thead has one row) -- assert chunks[0][1] == "
HeaderHX
" assert ( chunks[1][1] @@ -3475,9 +3444,6 @@ def and_it_does_not_let_a_clipped_span_bind_a_later_direct_row_of_the_same_key(s chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- comparing against the LAST accumulated row's key (tbody) rather than the first - # -- (table) correctly detects "After" as a real transition away from the clipped - # -- "Scoped" group, forcing a flush that keeps them apart -- assert len(chunks) == 2 for _, html in chunks: reparsed = HtmlTable.from_html_text(html) @@ -3485,8 +3451,7 @@ def and_it_does_not_let_a_clipped_span_bind_a_later_direct_row_of_the_same_key(s for idx, row in enumerate(rows): if row.max_rowspan is not None: assert idx + row.max_rowspan - 1 < len(rows) - # -- "Scoped"'s declared "2" is additionally clipped to "1" (its own tbody has only its - # -- own one row), so even inspected on its own its chunk claims nothing past itself -- + # -- "Scoped"'s declared "2" is clipped to "1" (its own tbody has one row) -- assert ( chunks[0][1] == "" "
LeadX
ScopedInside
" @@ -3496,9 +3461,7 @@ def and_it_does_not_let_a_clipped_span_bind_a_later_direct_row_of_the_same_key(s def and_it_preserves_non_text_cell_content_when_correcting_a_clipped_rowspan(self): """`HtmlRow.html_clipped_to_rows()` must only ever touch the `rowspan` attribute -- a nested table, a hyperlink, an image-only cell, and any other cell attribute must survive - a correction completely unchanged. Reconstructing a cell from its plain `.text` (the - pre-fix behavior) would flatten all of this into concatenated text or an emptied ``. - """ + a correction completely unchanged.""" opts = ChunkingOptions(max_characters=100_000) html_table = HtmlTable.from_html_text( """ @@ -3520,9 +3483,8 @@ def and_it_preserves_non_text_cell_content_when_correcting_a_clipped_rowspan(sel chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- `rowspan="0"` unconditionally goes through the correction path (its true reach is - # -- always rewritten to a literal count), which is exactly what exercises this cell's - # -- non-text content on every run -- not just when a chunk boundary happens to force it -- + # -- rowspan="0" always goes through the correction path, exercising this cell's + # -- non-text content on every run, not only when a chunk boundary forces it -- assert chunks == [ ( "Group Q1100details Other row content", @@ -3589,12 +3551,9 @@ def and_it_bounds_the_theads_own_original_occurrence_even_when_repetition_is_con def and_it_bounds_a_positive_rowspan_in_a_multi_cell_oversized_singleton_row(self): """A singleton rowspan-bound group whose ROW as a whole is too big for the chunking - window, but whose FIRST cell fits on its own, is emitted via `_CellAccumulator` (which - serializes a fitting cell's real `.html`, preserving its original `rowspan`) -- not via - `_iter_cell_splits` (which only ever emits plain, span-less text fragments and so was - never at risk here). This is the reachable case the "inert" reasoning missed: the row's - OWN, uncorrected declared span survives into that first sub-chunk unless it is bounded - before being handed to the row splitter.""" + window, but whose FIRST cell fits on its own, is emitted via `_CellAccumulator`, which + serializes the cell's real `.html` including its original `rowspan` -- that span must be + bounded before reaching the row splitter, or it survives uncorrected into the sub-chunk.""" opts = ChunkingOptions(max_characters=20) html_table = HtmlTable.from_html_text( """ @@ -3666,10 +3625,8 @@ def and_it_bounds_a_singleton_oversized_rows_span_through_chunk_by_title_and_rec html_out = reconstructed.metadata.text_as_html assert html_out is not None grid = pd.read_html(io.StringIO(html_out))[0].to_numpy().tolist() - # -- "Region"'s uncorrected `rowspan="3"` reaches exactly 3 rows deep (this row plus the - # -- next 2), not far enough to displace the much-later NW/Southwest Territory rows -- so - # -- this checks the actual corrupted rows directly, immediately following "Region", not - # -- rows the span happens not to reach in this particular fixture -- + # -- an uncorrected rowspan="3" reaches 3 rows deep, so check the rows immediately + # -- following "Region" rather than the much-later NW/Southwest Territory rows -- assert grid[0][0] == "Region" for row in grid[1:9]: assert row[0] != "Region", f"a later row still carries Region's uncorrected span: {row}" diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index 30e683da7f..f292b43b52 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -209,10 +209,7 @@ def test_partition_docx_table_with_full_width_vertical_merge_reports_a_tr_for_ev def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geometry(tmp_path): """A DOCX table with a real vertical merge, partitioned then chunked with a small window, must never split between rows an active `rowspan` still covers -- doing so would leave a - continuation `TableChunk` with cells shifted into the wrong column. This is the DOCX-specific - fix (real `colspan`/`rowspan` output) and the general rowspan-aware chunker exercised together - end to end, rather than only unit-tested in isolation. - """ + continuation `TableChunk` with cells shifted into the wrong column.""" document = docx.Document() table = document.add_table(rows=4, cols=2) table.cell(0, 0).merge(table.cell(1, 0)).merge(table.cell(2, 0)).text = "REGIONWIDE TOTAL" diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 7832006c7e..ba03af03d3 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1263,7 +1263,6 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: accum = _RowAccumulator( maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure ) - # -- if group won't fit, any WIP chunk is done, send it on its way -- if not accum.will_fit(group): for text, html in accum.flush(): yield self._prepend_repeated_headers(text, html, is_first_chunk) @@ -1271,7 +1270,6 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: accum = _RowAccumulator( maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure ) - # -- if group fits, add it to accumulator -- if accum.will_fit(group): accum.add_rows(group, group_bounds, is_clipped=group_is_clipped) elif len(group) == 1: # -- a single row is bigger than the chunking window -- From 842a17099789567b338f7f29490d9e6fe7df5db3 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Tue, 8 Sep 2026 09:02:28 -0500 Subject: [PATCH 15/19] Fix duplicate merged-cell text in nested DOCX tables, remove unreachable branch, and harden tests Nested tables inside a DOCX cell are flattened to text; that flattening duplicated a merged cell's text at every grid position it covered, the same class of bug this PR already fixed for top-level tables. Also removes a table-chunking branch in _iter_rowspan_bound_row_groups that can never execute (the last row's own row-group always closes on the loop's final iteration), makes the chunking test module collectible without the optional pandas extra, drops a verbatim-duplicate test, and strengthens a chunking regression test to check exact per-chunk HTML instead of only word presence. --- test_unstructured/chunking/test_base.py | 27 +------------ test_unstructured/partition/test_docx.py | 51 +++++++++++++++++++----- unstructured/chunking/base.py | 9 ----- unstructured/partition/docx.py | 8 +++- 4 files changed, 49 insertions(+), 46 deletions(-) diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 7334067a20..4065ea8104 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -8,7 +8,6 @@ import logging from typing import Any, Sequence -import pandas as pd import pytest from lxml.html import fragment_fromstring @@ -3342,30 +3341,6 @@ def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self) for word in ("alpha", "delta", "golf", "juliet", "mike"): assert combined_text.count(word) == 1 - def and_it_still_spans_the_whole_group_for_a_sectionless_rowspan_0_table(self): - """A table with no explicit ``/``/`` is itself one row-group, so - `rowspan="0"` reaches every row the table has -- emitted as the literal count ("2") - rather than the ambiguous "0".""" - opts = ChunkingOptions(max_characters=15) - html_table = HtmlTable.from_html_text( - """ - - - -
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
- """ - ) - - assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ - ( - "Region xxxxxxxxxxxxx yyyyyyyyyyyyy", - "" - '' - "" - "
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
", - ), - ] - def and_an_exactly_fitting_positive_rowspan_is_emitted_unchanged(self): """A `rowspan` whose declared value already matches its own row-group's row count is never rewritten -- the self-correction is a no-op whenever the declared value was @@ -3512,6 +3487,7 @@ def and_it_bounds_the_theads_own_original_occurrence_even_when_repetition_is_con correctly honors `rowspan`/`colspan` when building a grid) and checking that no body value has been shifted into the wrong column -- a genuine geometry check, not just a string/row-count comparison.""" + pd = pytest.importorskip("pandas") html = ( "" "" @@ -3605,6 +3581,7 @@ def and_it_bounds_a_singleton_oversized_rows_span_through_chunk_by_title_and_rec `reconstruct_table_from_chunks()` -- the actual round-trip a caller performs -- reparsing the reconstructed table with `pandas.read_html()` (which honors `rowspan`/`colspan` when building a grid) to catch real column-shift corruption, not just inspect chunk strings.""" + pd = pytest.importorskip("pandas") html = ( "
" "" diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index f292b43b52..7484fa0add 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -206,6 +206,32 @@ def test_partition_docx_table_with_full_width_vertical_merge_reports_a_tr_for_ev ) +def test_partition_docx_table_with_merged_cell_in_nested_table_does_not_duplicate_its_text( + tmp_path, +): + """A merged cell in a table nested inside another table's cell contributes its text once. + + Nested tables are flattened to plain text rather than nested `
` HTML, but that + flattening must still collapse a merged cell to a single occurrence of its text. + """ + document = docx.Document() + outer_table = document.add_table(rows=1, cols=1) + nested_table = outer_table.cell(0, 0).add_table(rows=2, cols=2) + nested_table.cell(0, 0).merge(nested_table.cell(0, 1)).text = "MERGEDNESTED" + nested_table.cell(1, 0).text = "foo" + nested_table.cell(1, 1).text = "bar" + docx_path = tmp_path / "nested-merged-cell.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path), infer_table_structure=True) + table_element = next(e for e in elements if isinstance(e, Table)) + + assert table_element.text == "MERGEDNESTED foo bar" + assert table_element.metadata.text_as_html == ( + "
MERGEDNESTED foo bar
" + ) + + def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geometry(tmp_path): """A DOCX table with a real vertical merge, partitioned then chunked with a small window, must never split between rows an active `rowspan` still covers -- doing so would leave a @@ -227,17 +253,20 @@ def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geom chunks = chunk_by_title([table_element], max_characters=60) - assert len(chunks) > 1, "fixture should be oversized enough to actually require a split" - for chunk in chunks: - assert isinstance(chunk, TableChunk) - # -- every emitted chunk must itself be well-formed, parseable HTML -- - html = chunk.metadata.text_as_html - assert html.startswith("") - assert html.endswith("
") - # -- no cell's text is lost or duplicated across the whole set of chunks -- - combined_text = " ".join(chunk.text for chunk in chunks) - for word in ("REGIONWIDE", "alpha", "delta", "golf", "juliet", "kilo"): - assert combined_text.count(word) == 1 + assert len(chunks) == 2, "fixture should be oversized enough to actually require a split" + assert all(isinstance(chunk, TableChunk) for chunk in chunks) + # -- the rowspan-3 group is never split -- it lands whole in the first chunk -- + assert chunks[0].metadata.text_as_html == ( + "" + '' + "" + "" + "
REGIONWIDE TOTALalpha bravo charlie
delta echo foxtrot
golf hotel india
" + ) + # -- the final row lands in its own chunk with both its cells correctly positioned -- + assert chunks[1].metadata.text_as_html == ( + "
julietkilo lima mike
" + ) def test_partition_docx_grabs_header_and_footer(): diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index ba03af03d3..cc25360cd7 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1346,15 +1346,6 @@ def _iter_rowspan_bound_row_groups( any(clipped[group_start : idx + 1]), ) group_start = idx + 1 - # -- a span reaching past the last row of its own row-group (or `rowspan="0"`) leaves a - # -- final group that never hits `idx == group_end` inside the loop; emit it rather than - # -- drop it. It's always clipped -- that's exactly why it never closed on its own. -- - if group_start < n: - yield ( - tuple(rows[group_start:]), - tuple(bound[group_start:]), - True, - ) @staticmethod def _group_last_idx(rows: Sequence[HtmlRow]) -> list[int]: diff --git a/unstructured/partition/docx.py b/unstructured/partition/docx.py index bab54a341c..249959c95d 100644 --- a/unstructured/partition/docx.py +++ b/unstructured/partition/docx.py @@ -519,8 +519,14 @@ def iter_cell_block_items(cell: _Cell) -> Iterator[str]: # -- structure only yield paragraph.text elif isinstance(table := block_item, DocxTable): + seen_tcs: set[Any] = set() for row in table.rows: - yield from (text for text, _ in iter_row_cells(row)) + for text, nested_cell in iter_row_cells(row): + if nested_cell is not None: + if nested_cell._tc in seen_tcs: + continue + seen_tcs.add(nested_cell._tc) + yield text def cell_text(cell: _Cell) -> str: """The normalized text of `cell`, including that of any table nested in it.""" From 3da7cb76a0122e59b7737cb5d00f31ba7f0ee8d7 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Tue, 8 Sep 2026 10:12:17 -0500 Subject: [PATCH 16/19] Fix rowspan handling across row-group boundaries and oversized rowspan groups A rowspan may legitimately reach from a into a (or into ); table chunking previously clipped any positive rowspan to its own row-group, silently truncating a valid span and shifting later rows into the wrong column. It's now bound only by the table's actual last row (rowspan "0" still clips to its own row-group, since that's what the HTML spec scopes it to). Repeating a table header on continuation chunks could re-inject a header cell's rowspan uncorrected, letting it reach past the repeated header block into the continuation's own body content. The repeated copy is now clipped to the number of header rows actually carried over. Finally, a rowspan-bound group too large to fit in one chunk was previously emitted whole, violating the chunk size limit -- increasingly likely once a positive rowspan's larger legitimate reach (above) is accounted for. It's now split on a row boundary like an ordinary oversized row, with the covering cell's rowspan rewritten per fragment and re-materialized in any fragment that doesn't include the row that originally declared it. --- test_unstructured/chunking/test_base.py | 355 +++++++++++++++-------- test_unstructured/partition/test_docx.py | 22 +- unstructured/chunking/base.py | 185 ++++++++++-- 3 files changed, 411 insertions(+), 151 deletions(-) diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 4065ea8104..bc2e4afeb3 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -1578,12 +1578,13 @@ def and_it_preserves_source_header_row_html_for_carried_rows(self): repeat_table_headers=True, ) - # -- the header's one-row thead group is isolated into its own leading chunk -- - assert len(chunks) == 4 + # -- Region's rowspan="2" legitimately reaches into Northwest's row, its own group, so + # -- both land in the leading chunk together -- + assert len(chunks) == 3 original_html = chunks[0].metadata.text_as_html assert original_html is not None original_table = fragment_fromstring(original_html) - assert original_table.xpath("./tr[1]/td[1]/@rowspan") == [] + assert original_table.xpath("./tr[1]/td[1]/@rowspan") == ["2"] continuation_html = chunks[1].metadata.text_as_html assert continuation_html is not None @@ -1592,7 +1593,9 @@ def and_it_preserves_source_header_row_html_for_carried_rows(self): assert continuation_table.xpath("./thead/tr[1]/@data-role") == ["header-row"] assert continuation_table.xpath("./thead/tr[1]/th[1]/@scope") == ["col"] assert continuation_table.xpath("./thead/tr[1]/th[1]/@abbr") == ["region-code"] - assert continuation_table.xpath("./thead/tr[1]/th[1]/@rowspan") == ["2"] + # -- only one header row is ever prepended, so a repeated copy's rowspan="2" -- which + # -- would otherwise reach into the continuation's own body row -- is clipped away -- + assert continuation_table.xpath("./thead/tr[1]/th[1]/@rowspan") == [] assert continuation_table.xpath("./thead/tr[1]/th[2]/@class") == ["sales-cell"] assert continuation_table.xpath("./thead/tr[1]/th[2]/@data-k") == ["1"] assert continuation_table.xpath("./thead/tr[1]/th[2]/@colspan") == ["2"] @@ -2460,7 +2463,10 @@ def and_it_preserves_header_attributes_in_reconstructed_canonical_thead(self): assert reconstructed.xpath("./thead/tr[1]/th[1]/@abbr") == ["region-code"] assert reconstructed.xpath("./thead/tr[1]/th[2]/@colspan") == ["2"] assert reconstructed.xpath("./thead/tr[2]/th[1]/@headers") == ["sales-group"] - assert reconstructed.xpath("./thead/tr[2]/th[2]/@rowspan") == ["2"] + # -- "Revenue"'s rowspan="2" reaches one row past the header block into the first body + # -- row (Northwest's); only 2 header rows are ever carried into a repeated copy, so that + # -- reach is clipped away rather than claiming an arbitrary continuation's body row -- + assert reconstructed.xpath("./thead/tr[2]/th[2]/@rowspan") == [] assert reconstructed.xpath("./tr[1]/th") == [] assert self._row_texts(table.metadata.text_as_html) == expected_rows @@ -3027,12 +3033,13 @@ def and_it_uses_the_configured_measurement_units_for_row_fitting( ), ] - def and_it_keeps_rows_bound_by_an_active_rowspan_in_the_same_chunk(self): - """A split between rows still covered by an earlier row's `rowspan` would leave that - `rowspan` claiming more rows than are present in its chunk, and would shift every cell - in the continuation chunk into the wrong column (its `` has no earlier row to - carry the span forward). The row-fit measurement only counts cell *text*, so a short-text, - markup-heavy table like this one can silently cross that boundary without the fix.""" + def and_it_splits_an_oversized_rowspan_bound_group_instead_of_emitting_it_whole(self): + """A rowspan-bound group too big to fit even an empty chunking window is split on a row + boundary, like an ordinary oversized row, rather than emitted whole in violation of + `max_characters`. The covering cell's `rowspan` is rewritten in the first fragment to the + rows it actually contains there, and re-materialized -- with its own rewritten `rowspan` + -- in the next fragment, which doesn't include the row that originally declared it.""" + pd = pytest.importorskip("pandas") opts = ChunkingOptions(max_characters=50) html_table = HtmlTable.from_html_text( """ @@ -3044,18 +3051,77 @@ def and_it_keeps_rows_bound_by_an_active_rowspan_in_the_same_chunk(self): """ ) - # -- the whole rowspan-bound group is emitted as one chunk, even though it exceeds - # -- `max_characters`, same tolerance already granted a single oversized row or cell -- - assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert chunks == [ ( - "AAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzzzzz", + "AAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", "
" - '' + '' "" - "" "
AAAAAxxxxxxxxxxxxxxxxxxxx
AAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
zzzzzzzzzzzzzzzzzzzz
", ), + ( + "AAAAA zzzzzzzzzzzzzzzzzzzz", + "
AAAAAzzzzzzzzzzzzzzzzzzzz
", + ), ] + for text, _ in chunks: + assert len(text) <= 50 + for _, html in chunks: + grid = pd.read_html(io.StringIO(html))[0].to_numpy().tolist() + for row in grid: + assert row[0] == "AAAAA" + + def and_it_splits_a_hallucinated_rowspan_that_reaches_the_tables_last_row(self): + """A model/OCR-hallucinated `rowspan` (e.g. "20" on a cell whose real span is more like + 2-3 rows) is bound by finding-1's fix to its true reach -- the table's last row -- rather + than to its own row-group, so the resulting group is now realistically bigger than + before. When that bigger group doesn't fit even an empty chunking window, splitting it + must still honor `max_characters`/`max_tokens`, not fall back to emitting it whole.""" + pd = pytest.importorskip("pandas") + opts = ChunkingOptions(max_characters=60) + body_rows = "".join( + f"row{i} extra padding text here" for i in range(1, 8) + ) + html_table = HtmlTable.from_html_text( + f""" + + + + + + {body_rows} +
covering cell text here paddingrow0 extra padding
+ """ + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + # (a) every emitted chunk is at or below `max_characters` -- + for text, _ in chunks: + assert len(text) <= 60 + # (b) every emitted chunk is well-formed, parseable HTML, and (d) each fragment's + # -- rewritten `rowspan` reflects only the rows actually present in that fragment -- + # -- (a fragment with N rows never declares a covering `rowspan` bigger than N) -- + for _, html in chunks: + reparsed = HtmlTable.from_html_text(html) + rows = list(reparsed.iter_rows()) + assert rows, f"chunk has no rows: {html}" + for idx, row in enumerate(rows): + if row.max_rowspan is not None: + assert idx + row.max_rowspan - 1 < len(rows) + # -- the covering cell's own text correctly lands in every row it claims to cover -- + for _, html in chunks: + grid = pd.read_html(io.StringIO(html))[0].to_numpy().tolist() + for row in grid: + assert row[0] == "covering cell text here padding" + # (c) no cell text is silently dropped across the full set of fragments -- every row's + # -- own (non-covering) text appears exactly once -- + combined_text = " ".join(text for text, _ in chunks) + for i in range(1, 8): + assert combined_text.count(f"row{i} extra padding text here") == 1 + assert combined_text.count("row0 extra padding") == 1 def and_it_keeps_a_fully_consumed_continuation_row_with_its_rowspan_origin(self): """The empty `` a fully-consumed continuation row emits (so a `rowspan` still counts @@ -3095,10 +3161,11 @@ def and_it_keeps_a_fully_consumed_continuation_row_with_its_rowspan_origin(self) def and_it_does_not_drop_a_rowspan_that_reaches_past_the_last_row(self): """A malformed but browser-tolerated `rowspan` naming more rows than the table has must - still yield its rows as one group, not disappear because the group-closing index it - names is never reached -- and the emitted `rowspan` is rewritten from its overdeclared - "3" down to "2", the number of rows the table (and this chunk) actually has, so the - emitted HTML is self-consistent regardless of what a caller inspects it in isolation.""" + still yield its rows, not disappear because the group-closing index it names is never + reached -- and, since the group is too big to fit even an empty chunking window here, + each fragment's `rowspan` is rewritten to only the rows it actually contains, with the + covering cell's text re-materialized into the fragment that doesn't include the row that + originally declared it.""" opts = ChunkingOptions(max_characters=25) html_table = HtmlTable.from_html_text( """ @@ -3109,22 +3176,29 @@ def and_it_does_not_drop_a_rowspan_that_reaches_past_the_last_row(self): """ ) - assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert chunks == [ ( - "A xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", - "" - '' - "" - "
Axxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", + "A xxxxxxxxxxxxxxxxxxxx", + "
Axxxxxxxxxxxxxxxxxxxx
", + ), + ( + "A yyyyyyyyyyyyyyyyyyyy", + "
Ayyyyyyyyyyyyyyyyyyyy
", ), ] + for text, _ in chunks: + assert len(text) <= 25 def and_it_treats_rowspan_0_as_spanning_every_remaining_row(self): """`rowspan="0"` is HTML's spelling for "spans every remaining row in the row group" — the largest possible span, not the absence of one. This model doesn't track ``/``/`` boundaries here (no explicit sections), so it resolves to - the rest of the table -- emitted as the literal, self-consistent count ("2") rather than - the ambiguous "0", since both rows land in this same chunk.""" + the rest of the table, binding both rows into one rowspan-bound group. The window here is + too small even for the first row's own cells, so the group degrades all the way to + cell-level splitting -- the same tolerance already granted a single oversized cell -- + rather than being emitted whole in violation of `max_characters`.""" opts = ChunkingOptions(max_characters=15) html_table = HtmlTable.from_html_text( """ @@ -3135,15 +3209,17 @@ def and_it_treats_rowspan_0_as_spanning_every_remaining_row(self): """ ) - assert list(_HtmlTableSplitter.iter_subtables(html_table, opts)) == [ - ( - "Region xxxxxxxxxxxxx yyyyyyyyyyyyy", - "" - '' - "" - "
Regionxxxxxxxxxxxxx
yyyyyyyyyyyyy
", - ), + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert chunks == [ + ("Region", "
Region
"), + ("xxxxxxxxxxxxx", "
xxxxxxxxxxxxx
"), + ("yyyyyyyyyyyyy", "
yyyyyyyyyyyyy
"), ] + # -- no cell text lost or duplicated across the whole set of chunks -- + combined_text = " ".join(text for text, _ in chunks) + for word in ("Region", "xxxxxxxxxxxxx", "yyyyyyyyyyyyy"): + assert combined_text.count(word) == 1 def and_it_bounds_a_rowspan_0_header_to_its_own_thead_instead_of_the_whole_table(self): """`rowspan="0"` spans every remaining row in its OWN row-group, not the whole table. A @@ -3217,12 +3293,12 @@ def and_no_emitted_chunk_lets_a_span_reparse_across_its_source_row_group(self): if row.max_rowspan is not None: assert idx + row.max_rowspan - 1 < len(rows) - def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): - """A `rowspan` declared inside one `` must not reach into a following `` or - `` — each row-group bounds its own spans, however far they claim to reach. The - overdeclared "5" is rewritten to "2" (the tbody's own row count), so the emitted chunk is - self-consistent even inspected on its own, not merely safe because the tfoot landed - elsewhere.""" + def and_it_lets_a_positive_rowspan_reach_from_one_tbody_into_a_following_tfoot(self): + """A positive `rowspan` declared inside one `` may legitimately reach into a + following `` or `` -- that is valid HTML, the continuation row in the next + section omits the covered column, relying on the earlier section's cell to still cover + it. The overdeclared "5" is rewritten to "3", the table's actual total row count, since + that is as far as it can truly reach, not to the tbody's own 2-row count.""" opts = ChunkingOptions(max_characters=70) html_table = HtmlTable.from_html_text( """ @@ -3240,7 +3316,9 @@ def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- rowspan is clipped to the tbody's own 2-row count; the tfoot row is a separate chunk -- + # -- the merged group (tbody + tfoot) is bigger than `max_characters`, so it's split on a + # -- row boundary; the covering cell is re-materialized (rowspan="1") in the tfoot's own + # -- fragment, which doesn't include the row that originally declared the span -- assert chunks == [ ( "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", @@ -3250,20 +3328,16 @@ def and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody(self): "", ), ( - "zzzzzzzzzzzzzzzzzzzz", - "
zzzzzzzzzzzzzzzzzzzz
", + "AAAAAAAAAAAAAAAAAAAA zzzzzzzzzzzzzzzzzzzz", + "
AAAAAAAAAAAAAAAAAAAAzzzzzzzzzzzzzzzzzzzz
", ), ] - def and_it_clips_a_positive_rowspan_even_when_a_huge_window_would_otherwise_merge_sections( - self, - ): - """The row-group boundary must hold for a CLIPPED positive `rowspan` too, not only - `rowspan="0"` -- and must hold even when the character budget alone would happily pack - the clipped group and the next row-group's rows into one chunk. A tight budget (as in - `and_it_clips_a_positive_rowspan_at_the_end_of_its_own_tbody`) would pass even without - this protection, purely by accident; this uses a window large enough that only the - clipped-group tracking itself can be responsible for keeping the groups apart.""" + def and_it_merges_tbody_and_tfoot_into_one_chunk_when_a_huge_window_allows_it(self): + """A positive `rowspan`'s reach across a ``/`` boundary is not an artifact + of a tight character budget -- with a window large enough to hold every row, the whole + table (tbody + tfoot) is emitted as a single chunk, and the overdeclared "5" is rewritten + to "4", the table's actual total row count.""" opts = ChunkingOptions(max_characters=100_000) html_table = HtmlTable.from_html_text( """ @@ -3282,28 +3356,24 @@ def and_it_clips_a_positive_rowspan_even_when_a_huge_window_would_otherwise_merg chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- rowspan is clipped to the tbody's row count; the tfoot rows stay a separate chunk -- assert chunks == [ ( - "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy", - "" - '' - "" - "
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
", - ), - ( + "AAAAAAAAAAAAAAAAAAAA xxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyy " "zzzzzzzzzzzzzzzzzzzz wwwwwwwwwwwwwwwwwwww", "" + '' + "" "" "" "
AAAAAAAAAAAAAAAAAAAAxxxxxxxxxxxxxxxxxxxx
yyyyyyyyyyyyyyyyyyyy
zzzzzzzzzzzzzzzzzzzz
wwwwwwwwwwwwwwwwwwww
", ), ] - def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self): - """Same clipped-positive-rowspan protection, exercised through the public - `chunk_by_title()` entry point: tbody and tfoot content must never land in the same - chunk, and the rewritten rowspan must match the tbody's own row count.""" + def and_it_splits_a_tbody_tfoot_spanning_group_through_the_public_chunk_by_title_path(self): + """Same tbody-into-tfoot reach, exercised through the public `chunk_by_title()` entry + point with a window too small to hold the whole merged group: every emitted chunk must + still respect `max_characters`, and no cell text may be lost across the whole set.""" + pd = pytest.importorskip("pandas") html = ( "" "" @@ -3327,18 +3397,22 @@ def and_it_clips_a_positive_rowspan_through_the_public_chunk_by_title_path(self) assert len(chunks) > 1 for chunk in chunks: assert isinstance(chunk, TableChunk) - # -- every emitted chunk is well-formed, parseable HTML -- - html = chunk.metadata.text_as_html - assert html.startswith("
") - assert html.endswith("
") - # -- tbody and tfoot content must never share a chunk, or the tbody's rowspan="5" - # -- would reach into the tfoot rows and shift them a column over -- - has_tbody_content = "golf" in html - has_tfoot_content = "juliet" in html or "mike" in html - assert not (has_tbody_content and has_tfoot_content) - # -- no cell text lost or duplicated across the whole set of chunks -- + assert len(chunk.text) <= 75 + html_out = chunk.metadata.text_as_html + assert html_out is not None + assert html_out.startswith("") + assert html_out.endswith("
") + # -- the covering cell ("alpha bravo charlie") is re-materialized into whichever + # -- fragment doesn't hold its origin row, so every row -- tbody or tfoot -- keeps + # -- its correct column 0 value in every chunk that contains it -- + grid = pd.read_html(io.StringIO(html_out))[0].to_numpy().tolist() + for row in grid: + if row[1] in ("golf hotel india", "juliet kilo lima", "mike november oscar"): + assert row[0] == "alpha bravo charlie" + # -- no cell text lost across the whole set of chunks -- the covering cell's own text is + # -- allowed to repeat, once per fragment it spans, per the class docstring -- combined_text = " ".join(chunk.text for chunk in chunks) - for word in ("alpha", "delta", "golf", "juliet", "mike"): + for word in ("delta", "golf", "juliet", "mike"): assert combined_text.count(word) == 1 def and_an_exactly_fitting_positive_rowspan_is_emitted_unchanged(self): @@ -3362,13 +3436,14 @@ def and_an_exactly_fitting_positive_rowspan_is_emitted_unchanged(self): ), ] - def and_it_clips_an_overdeclared_thead_rowspan_when_header_repetition_is_not_active(self): - """A `` row's overdeclared positive `rowspan` is only exempt from clipping when - header repetition is actually configured and active for it -- `_prepend_repeated_headers` - is what makes reaching past the header's own section safe, and that mechanism plays no - part when `repeat_table_headers=False`. Without it, nothing else protects an overreaching - `` span from binding body rows it was never meant to cover, so it must be clipped - exactly like a ``/`` span would be.""" + def and_it_lets_a_positive_thead_rowspan_reach_into_the_tbody_regardless_of_repetition(self): + """A `` row's positive `rowspan` reaching into the `` is valid HTML whether + or not header repetition is configured -- `repeat_table_headers` only governs a + *repeated copy* injected on a continuation chunk (see `_prepend_repeated_headers`), never + the header's own original occurrence, which is bound by its true reach like any other + row. Here that reach ("3") exactly matches the table's total row count, so nothing is + clipped at all.""" + pd = pytest.importorskip("pandas") opts = ChunkingOptions(max_characters=200) html_table = HtmlTable.from_html_text( """ @@ -3382,36 +3457,71 @@ def and_it_clips_an_overdeclared_thead_rowspan_when_header_repetition_is_not_act """ ) - # -- header_row_count=0 (the default) means no repeat-header exemption applies here -- + # -- header_row_count=0 (the default) means no repeat-header injection applies here -- chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - assert len(chunks) == 2 - for _, html in chunks: - reparsed = HtmlTable.from_html_text(html) - rows = list(reparsed.iter_rows()) - for idx, row in enumerate(rows): - if row.max_rowspan is not None: - assert idx + row.max_rowspan - 1 < len(rows) - # -- the header's declared "3" is clipped to "1" (its own thead has one row) -- - assert chunks[0][1] == "
HeaderHX
" - assert ( - chunks[1][1] - == "
AB
CD
" + assert chunks == [ + ( + "Header HX A B C D", + "" + '' + "" + "" + "
HeaderHX
AB
CD
", + ), + ] + grid = pd.read_html(io.StringIO(chunks[0][1]))[0].to_numpy().tolist() + assert [row[0] for row in grid] == ["Header"] * 3 + + def and_it_lets_a_positive_span_from_a_tbody_bind_a_later_direct_row_of_the_same_key(self): + """A `rowspan` declared inside an explicit `` may legitimately reach into a + following direct (sectionless) row -- valid HTML, and no different from reaching into + another `` or ``. Here the declared "2" already matches its true reach + (the `` row plus the one direct row after it), so nothing is clipped, and the two + rows are packed into the same chunk as the unrelated leading "Lead" row since there's + room for all three.""" + pd = pytest.importorskip("pandas") + opts = ChunkingOptions(max_characters=200) + html_table = HtmlTable.from_html_text( + """ + + + + +
LeadX
ScopedInside
AfterY
+ """ ) + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert chunks == [ + ( + "Lead X Scoped Inside After Y", + "" + "" + '' + "" + "
LeadX
ScopedInside
AfterY
", + ), + ] + grid = pd.read_html(io.StringIO(chunks[0][1]))[0].to_numpy().tolist() + assert grid[1][0] == "Scoped" + assert grid[2][:2] == ["Scoped", "After"] + def and_it_does_not_let_a_clipped_span_bind_a_later_direct_row_of_the_same_key(self): """`_RowAccumulator` must compare a candidate group's row-group identity against the MOST RECENTLY accumulated row-group, not the first one ever added to this accumulator -- a direct (sectionless) row before and after an explicit `` both key by the same `` element, so comparing against the first accumulated row can mistake a real transition (leaving the clipped tbody group) for "no change", letting the clipped span - bind the trailing direct row it was never meant to cover.""" + bind the trailing direct row it was never meant to cover. `rowspan="0"` is used here + since it is the one span variety that still clips to its own row-group's end.""" opts = ChunkingOptions(max_characters=200) html_table = HtmlTable.from_html_text( """
- +
LeadX
ScopedInside
ScopedInside
AfterY
""" @@ -3426,7 +3536,7 @@ def and_it_does_not_let_a_clipped_span_bind_a_later_direct_row_of_the_same_key(s for idx, row in enumerate(rows): if row.max_rowspan is not None: assert idx + row.max_rowspan - 1 < len(rows) - # -- "Scoped"'s declared "2" is clipped to "1" (its own tbody has one row) -- + # -- "Scoped"'s rowspan="0" is clipped to "1" (its own tbody has one row) -- assert ( chunks[0][1] == "" "
LeadX
ScopedInside
" @@ -3472,16 +3582,16 @@ def and_it_preserves_non_text_cell_content_when_correcting_a_clipped_rowspan(sel ) ] - def and_it_bounds_the_theads_own_original_occurrence_even_when_repetition_is_configured( + def and_it_lets_the_theads_own_occurrence_bind_body_rows_it_legitimately_reaches( self, ): - """The carried-header exemption must apply only to a REPEATED copy - `_prepend_repeated_headers` injects onto a continuation chunk (a separate artifact built - from `row.source_html`/`row.html`, wrapped in its own real ``) -- never to the - header row's own ORIGINAL, wrapper-less occurrence. Before this fix, - `repeat_table_headers=True` alone was enough to exempt that original occurrence too, so - its overreaching `rowspan` could merge with body rows from a different row-group in the - same wrapper-less chunk and shift their columns. + """`Region`'s `rowspan="3"` legitimately reaches from its own `` into the first + two `` rows -- valid HTML, and no different from any other cross-row-group reach. + Those two rows share a chunk with it; the third (`Midwest Territory`) lands in its own + continuation chunk with a REPEATED header copy that `_prepend_repeated_headers` injects + (a separate artifact built from `row.source_html`/`row.html`, wrapped in its own real + ``) -- and since only one header row is ever carried, that copy's `rowspan` is + clipped away rather than reaching into `Midwest Territory`'s row. Verified by reparsing each chunk's own emitted HTML with `pandas.read_html` (which correctly honors `rowspan`/`colspan` when building a grid) and checking that no body @@ -3512,12 +3622,16 @@ def and_it_bounds_the_theads_own_original_occurrence_even_when_repetition_is_con grid = pd.read_html(io.StringIO(html_out))[0].to_numpy().tolist() for row in grid: if "NW" in row: - assert row[0] == "NW" - assert row[1] == "Q1" + assert row[0] == "Region" + assert row[1] == "NW" + assert row[2] == "Q1" if "Southwest Territory" in row: - assert row[0] == "Southwest Territory" - assert row[1] == "Q2" + assert row[0] == "Region" + assert row[1] == "Southwest Territory" + assert row[2] == "Q2" if "Midwest Territory" in row: + # -- the repeated header's own rowspan was clipped away, so this row is not + # -- shifted by a stray "Region" column -- assert row[0] == "Midwest Territory" assert row[1] == "Q3" # -- no cell text lost or duplicated across the whole set of chunks -- @@ -3525,11 +3639,13 @@ def and_it_bounds_the_theads_own_original_occurrence_even_when_repetition_is_con for word in ("NW", "Southwest", "Midwest"): assert combined_text.count(word) == 1 - def and_it_bounds_a_positive_rowspan_in_a_multi_cell_oversized_singleton_row(self): - """A singleton rowspan-bound group whose ROW as a whole is too big for the chunking - window, but whose FIRST cell fits on its own, is emitted via `_CellAccumulator`, which - serializes the cell's real `.html` including its original `rowspan` -- that span must be - bounded before reaching the row splitter, or it survives uncorrected into the sub-chunk.""" + def and_it_bounds_a_positive_rowspan_whose_own_row_is_oversized_even_alone(self): + """`Region`'s `rowspan="3"` exactly reaches the table's last row, so it opens a 3-row + group (thead + both tbody rows), not a singleton -- but its own row alone (with the huge + second cell) is too big to fit even by itself, so the group degrades all the way to + `_CellAccumulator`, which serializes a cell's real `.html` including its original + `rowspan`. That span must be bounded before reaching the row splitter, or it survives + uncorrected into the sub-chunk even though `Region` is now emitted alone.""" opts = ChunkingOptions(max_characters=20) html_table = HtmlTable.from_html_text( """ @@ -3549,8 +3665,7 @@ def and_it_bounds_a_positive_rowspan_in_a_multi_cell_oversized_singleton_row(sel chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) - # -- the one-row thead's own bound is "1" (no other row in its group), so the header's - # -- first sub-chunk must carry no rowspan claim at all -- + # -- "Region"'s own cell-level sub-chunk carries no rowspan claim at all -- assert chunks[0] == ("Region", "
Region
") def and_it_bounds_a_rowspan_0_in_a_multi_cell_oversized_singleton_row(self): diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index 7484fa0add..db8d965b89 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -234,8 +234,10 @@ def test_partition_docx_table_with_merged_cell_in_nested_table_does_not_duplicat def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geometry(tmp_path): """A DOCX table with a real vertical merge, partitioned then chunked with a small window, - must never split between rows an active `rowspan` still covers -- doing so would leave a - continuation `TableChunk` with cells shifted into the wrong column.""" + must never split between rows an active `rowspan` still covers without correcting for it -- + doing so would leave a continuation `TableChunk` with cells shifted into the wrong column. + Splitting is fine as long as the covering cell's `rowspan` is rewritten to match, and + re-materialized in whichever fragment doesn't hold the row that originally declared it.""" document = docx.Document() table = document.add_table(rows=4, cols=2) table.cell(0, 0).merge(table.cell(1, 0)).merge(table.cell(2, 0)).text = "REGIONWIDE TOTAL" @@ -253,18 +255,24 @@ def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geom chunks = chunk_by_title([table_element], max_characters=60) - assert len(chunks) == 2, "fixture should be oversized enough to actually require a split" + assert len(chunks) == 3, "fixture should be oversized enough to actually require a split" assert all(isinstance(chunk, TableChunk) for chunk in chunks) - # -- the rowspan-3 group is never split -- it lands whole in the first chunk -- + assert all(len(chunk.text) <= 60 for chunk in chunks) + # -- the rowspan-3 group is itself too big for the window, so it's split on a row boundary; + # -- the covering cell's rowspan is rewritten to the rows present in the first fragment -- assert chunks[0].metadata.text_as_html == ( "" - '' + '' "" - "" "
REGIONWIDE TOTALalpha bravo charlie
REGIONWIDE TOTALalpha bravo charlie
delta echo foxtrot
golf hotel india
" ) - # -- the final row lands in its own chunk with both its cells correctly positioned -- + # -- the covering cell is re-materialized (rowspan="1") in the next fragment, which doesn't + # -- include the row that originally declared the span, keeping columns aligned -- assert chunks[1].metadata.text_as_html == ( + "
REGIONWIDE TOTALgolf hotel india
" + ) + # -- the final row lands in its own chunk with both its cells correctly positioned -- + assert chunks[2].metadata.text_as_html == ( "
julietkilo lima mike
" ) diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index cc25360cd7..b5f500b20f 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -6,14 +6,14 @@ import copy import uuid from functools import cached_property -from typing import Any, Callable, DefaultDict, Iterable, Iterator, Sequence, cast +from typing import Any, Callable, DefaultDict, Iterable, Iterator, NamedTuple, Sequence, cast import regex from lxml.etree import ParserError, tostring from lxml.html import fragment_fromstring from typing_extensions import Self, TypeAlias -from unstructured.common.html_table import HtmlCell, HtmlRow, HtmlTable +from unstructured.common.html_table import HtmlCell, HtmlRow, HtmlTable, _format_td from unstructured.documents.elements import ( CodeSnippet, CompositeElement, @@ -1211,6 +1211,16 @@ def _text_with_overlap(self) -> str: # ================================================================================================ +class _OpenSpan(NamedTuple): + """A `rowspan` still active at some row past the one that declared it.""" + + col: int + colspan: int + text: str + reach_idx: int + """Last row-index (relative to the containing rowspan-bound group) this span still covers.""" + + class _HtmlTableSplitter: """Produces (text, html) pairs for a `` HtmlElement. @@ -1286,11 +1296,12 @@ def _iter_subtables(self) -> Iterator[TextAndHtml]: maxlen=self._maxlen(is_first_chunk), measure=self._opts.measure ) else: - # -- A rowspan-bound group doesn't fit even in an empty chunking window. Splitting - # -- it would corrupt the span it exists to protect, so it's emitted whole, same - # -- tolerance the codebase already grants a single oversized row/cell. - accum.add_rows(group, group_bounds, is_clipped=group_is_clipped) - for text, html in accum.flush(): + # -- A rowspan-bound group doesn't fit even in an empty chunking window; split it + # -- like an ordinary oversized row, re-materializing any covered column a + # -- fragment boundary separates from the row whose rowspan declares it. + for text, html in self._iter_oversized_group_splits( + group, maxlen=self._maxlen(is_first_chunk) + ): yield self._prepend_repeated_headers(text, html, is_first_chunk) is_first_chunk = False accum = _RowAccumulator( @@ -1307,10 +1318,14 @@ def _iter_rowspan_bound_row_groups( """Group consecutive rows that a `rowspan` binds together. A row whose cell declares `rowspan=N` binds the next `N-1` rows to it, since splitting - them across chunks would misplace their cells or overclaim the span's row count. A group's - far edge is the max reach of every span opened within it (standard overlapping-interval - merge); a span reaching past its own row-group's last row, or `rowspan="0"`, clips to that - row-group's end (see `_group_last_idx`) and is always yielded rather than dropped. + them across chunks would misplace their cells or overclaim the span's row count. A + positive `rowspan` is bound by its true reach, clamped only to the table's last row -- + HTML allows a span to cross a ``/``/`` boundary, with the + continuation rows in the next row-group omitting the covered column. `rowspan="0"` (HTML's + "span every remaining row") has no literal count to reach with, so it clips to its own + row-group's last row instead (see `_group_last_idx`). A group's far edge is the max reach + of every span opened within it (standard overlapping-interval merge); a clipped span is + always yielded rather than dropped. Yields, per group: the rows, a same-length tuple of per-row safe rowspan bounds (consumed by `HtmlRow.html_clipped_to_rows()` so an emitted rowspan can never claim a row that isn't @@ -1323,26 +1338,24 @@ def _iter_rowspan_bound_row_groups( group_last_idx = self._group_last_idx(rows) reach = [0] * n clipped = [False] * n - bound: list[int] = [0] * n for idx, row in enumerate(rows): - own_group_last = group_last_idx[idx] - bound[idx] = own_group_last - idx + 1 if row.max_rowspan is None: - reach[idx] = own_group_last + reach[idx] = group_last_idx[idx] clipped[idx] = True else: declared_reach = idx + row.max_rowspan - 1 - reach[idx] = min(declared_reach, own_group_last) - clipped[idx] = declared_reach > own_group_last + reach[idx] = min(declared_reach, n - 1) + clipped[idx] = declared_reach > n - 1 group_start = 0 group_end = -1 # -- index of the furthest row any span opened so far reaches -- for idx in range(n): group_end = max(group_end, reach[idx]) if idx == group_end: + bound = tuple(group_end - i + 1 for i in range(group_start, idx + 1)) yield ( tuple(rows[group_start : idx + 1]), - tuple(bound[group_start : idx + 1]), + bound, any(clipped[group_start : idx + 1]), ) group_start = idx + 1 @@ -1351,8 +1364,9 @@ def _iter_rowspan_bound_row_groups( def _group_last_idx(rows: Sequence[HtmlRow]) -> list[int]: """For each row-index in `rows`, the index of the last row sharing its row-group. - Rows are grouped by identity of `HtmlRow.row_group_key`, so a `rowspan` can never bind - rows across a real ``/``/`` boundary. + Rows are grouped by identity of `HtmlRow.row_group_key`. Used only to bound a + `rowspan="0"` cell, the one span variety that is scoped to its own row-group rather than + bound by a literal count. """ n = len(rows) last_idx = [0] * n @@ -1367,6 +1381,114 @@ def _group_last_idx(rows: Sequence[HtmlRow]) -> list[int]: i = j + 1 return last_idx + def _iter_oversized_group_splits( + self, group: tuple[HtmlRow, ...], maxlen: int + ) -> Iterator[TextAndHtml]: + """Split a rowspan-bound `group` too big to fit even an empty chunking window. + + Rows are packed in order like an ordinary sequence, falling back to `_iter_row_splits` + for a single row still too big alone. A column covered only by an earlier row's + `rowspan` -- and so absent from a later row's own `` -- is re-materialized as a fresh + cell repeating the covering cell's text whenever a fragment boundary separates that row + from the row declaring the span, with the copy's `rowspan` set to only the rows of that + span actually present in the fragment. This unavoidably repeats the covering cell's text + across fragments, the accepted trade-off for honoring the hard size limit. + """ + n = len(group) + active: list[_OpenSpan] = [] + fragment_cells: list[list[str]] = [] + fragment_texts: list[list[str]] = [] + + def build_row( + row: HtmlRow, idx: int, active: list[_OpenSpan], materialize: bool + ) -> tuple[list[str], list[str], list[_OpenSpan]]: + cells: list[str] = [] + texts: list[str] = [] + new_active: list[_OpenSpan] = [] + spans = iter(sorted((s for s in active if s.reach_idx >= idx), key=lambda s: s.col)) + next_span = next(spans, None) + own_cells = list(row.iter_cells()) + own_idx = 0 + col = 0 + while True: + if next_span is not None and next_span.col == col: + if materialize: + remaining = next_span.reach_idx - idx + 1 + cells.append(_format_td(next_span.text, next_span.colspan, remaining)) + if next_span.text: + texts.append(next_span.text) + if next_span.reach_idx > idx: + new_active.append(next_span) + col += next_span.colspan + next_span = next(spans, None) + continue + if own_idx < len(own_cells): + cell = own_cells[own_idx] + own_idx += 1 + cells.append(cell.html) + if cell.text: + texts.append(cell.text) + cell_reach = ( + n - 1 if cell.rowspan is None else min(idx + cell.rowspan - 1, n - 1) + ) + if cell_reach > idx: + new_active.append(_OpenSpan(col, cell.colspan, cell.text, cell_reach)) + col += cell.colspan + continue + break + return cells, texts, new_active + + def fits(texts: Sequence[str]) -> bool: + candidate = fragment_texts + [list(texts)] + joined = " ".join(t for row_texts in candidate for t in row_texts) + return self._opts.measure(joined) <= maxlen + + def flush_fragment() -> Iterator[TextAndHtml]: + nonlocal fragment_cells, fragment_texts + if not fragment_cells: + return + m = len(fragment_cells) + trs: list[str] = [] + for k, cells in enumerate(fragment_cells): + bound = m - k + tr = _HtmlTableSplitter._parse_row_fragment(f"{''.join(cells)}") + row = HtmlRow(tr) + trs.append( + row.html_clipped_to_rows(bound) + if row.max_rowspan is None or row.max_rowspan > bound + else row.html + ) + text = " ".join(t for row_texts in fragment_texts for t in row_texts) + html = f"
{''.join(trs)}
" + fragment_cells, fragment_texts = [], [] + yield text, html + + for idx, row in enumerate(group): + active = [s for s in active if s.reach_idx >= idx] + cells, texts, next_active = build_row(row, idx, active, materialize=False) + if fragment_cells and fits(texts): + fragment_cells.append(cells) + fragment_texts.append(texts) + active = next_active + continue + + yield from flush_fragment() + + mat_cells, mat_texts, mat_active = build_row(row, idx, active, materialize=True) + if self._opts.measure(" ".join(mat_texts)) <= maxlen: + fragment_cells, fragment_texts = [mat_cells], [mat_texts] + active = mat_active + else: + # -- even this single row, with its covered columns materialized, is too big to + # -- fit alone; fall back to cell-level splitting, the same tolerance granted an + # -- ordinary oversized row -- it can't span beyond itself, so bound it to 1 -- + tr = _HtmlTableSplitter._parse_row_fragment(f"{''.join(mat_cells)}") + bounded_row = HtmlRow(tr).row_clipped_to_rows(1) + yield from self._iter_row_splits(bounded_row, maxlen=maxlen) + active = [] + + yield from flush_fragment() + def _iter_row_splits(self, row: HtmlRow, maxlen: int) -> Iterator[TextAndHtml]: """Split oversized row into (text, html) pairs containing as many cells as will fit.""" accum = _CellAccumulator(maxlen=maxlen) @@ -1431,7 +1553,11 @@ def _header_rows_html(self) -> str: if not self._header_rows: return "" - rows_html = "".join(self._as_header_row_html(row) for row in self._header_rows) + n = len(self._header_rows) + rows_html = "".join( + self._as_header_row_html(row, max_rowspan=n - i) + for i, row in enumerate(self._header_rows) + ) return f"{rows_html}" @cached_property @@ -1482,16 +1608,27 @@ def _prepend_repeated_headers(self, text: str, html: str, is_first_chunk: bool) return chunk_text, chunk_html @staticmethod - def _as_header_row_html(row: HtmlRow) -> str: - """Serialize `row` preserving source HTML while converting direct-child `` to ``.""" + def _as_header_row_html(row: HtmlRow, max_rowspan: int) -> str: + """Serialize `row` preserving source HTML while converting direct-child `` to ``. + + Clips any cell's `rowspan` down to `max_rowspan` -- the number of header rows actually + being prepended -- so a repeated header can never claim rows beyond its own synthetic + `` and reach into the continuation chunk's body. + """ row_html = row.source_html or row.html tr = _HtmlTableSplitter._parse_row_fragment(row_html) if tr is None and row.source_html: tr = _HtmlTableSplitter._parse_row_fragment(row.html) if tr is None: - return row.html + return row.html_clipped_to_rows(max_rowspan) for cell in tr: + rowspan = HtmlCell(cell).rowspan + if rowspan is None or rowspan > max_rowspan: + if max_rowspan <= 1: + cell.attrib.pop("rowspan", None) + else: + cell.attrib["rowspan"] = str(max_rowspan) if getattr(cell, "tag", None) == "td": cell.tag = "th" From 8309692c80705fda7625ecb068af6aa62f727303 Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Tue, 8 Sep 2026 10:35:44 -0500 Subject: [PATCH 17/19] Preserve colspan/rowspan when splitting oversized cells and serializing empty spanned cells Oversized-cell text splitting now keeps the cell's colspan on each emitted fragment, and HtmlCell.html now preserves colspan/rowspan for empty cells instead of collapsing them to a bare . --- test_unstructured/chunking/test_base.py | 13 +++++++++++++ test_unstructured/common/test_html_table.py | 4 ++++ unstructured/chunking/base.py | 4 ++-- unstructured/common/html_table.py | 4 +++- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index bc2e4afeb3..2eb4908775 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3009,6 +3009,19 @@ def and_it_splits_an_oversized_cell_on_an_even_word_boundary(self): ), ] + def and_it_preserves_colspan_when_splitting_an_oversized_cell(self): + opts = ChunkingOptions(max_characters=50) + words = " ".join(["word"] * 30) + html_table = HtmlTable.from_html_text( + f'
{words}
' + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert len(chunks) > 1 + for _, html in chunks: + assert html.startswith('' ) + def and_it_preserves_colspan_and_rowspan_on_an_empty_cell(self): + cell = HtmlCell(fragment_fromstring('')) + assert cell.html == '".""" - return etree.tostring(self._td, encoding=str) if self.text else "
') + def and_it_uses_the_configured_measurement_units_for_row_fitting( self, monkeypatch: pytest.MonkeyPatch ): diff --git a/test_unstructured/common/test_html_table.py b/test_unstructured/common/test_html_table.py index 0b790118b9..4537596aef 100644 --- a/test_unstructured/common/test_html_table.py +++ b/test_unstructured/common/test_html_table.py @@ -397,6 +397,10 @@ def and_it_preserves_nested_markup_when_serializing_nonempty_cells(self): 'Category Link' + @pytest.mark.parametrize( ("cell_html", "expected_value"), [ diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index b5f500b20f..5109547fde 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1522,12 +1522,12 @@ def _iter_cell_splits(self, cell: HtmlCell, maxlen: int) -> Iterator[TextAndHtml split = _TextSplitter(opts) text, remainder = split(cell.text) - yield text, f"
{text}
" + yield text, f"{_format_td(text, cell.colspan, rowspan=1)}
" # -- an oversized cell will have a remainder, split that up into additional chunks. while remainder: text, remainder = split(remainder) - yield text, f"
{text}
" + yield text, f"{_format_td(text, cell.colspan, rowspan=1)}
" @cached_property def _header_text(self) -> str: diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index d91628b02f..0b37c5b8ab 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -370,7 +370,9 @@ def __init__(self, td: HtmlElement): @cached_property def html(self) -> str: """Like "		foo bar baz" + if self.text: + return etree.tostring(self._td, encoding=str) + return _format_td("", self.colspan, self.rowspan or 1) @cached_property def text(self) -> str: From 89a2159a5a6e1cfc08a3a92e1906a6991c51d44a Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Tue, 8 Sep 2026 10:52:13 -0500 Subject: [PATCH 18/19] Account for colspan and HTML escaping when splitting oversized table cells The fixed-overhead assumption for an oversized cell's split fragments didn't account for a colspan attribute's own characters, or for the fact that escaping cell text (&, <, >) can make the formatted fragment longer than the raw text budgeted for it. Overhead is now derived from the cell's actual colspan, and each split verifies the formatted fragment against the limit, shrinking and re-splitting (falling back to a raw truncation in pathological cases) until it fits. --- test_unstructured/chunking/test_base.py | 49 +++++++++++++ unstructured/chunking/base.py | 91 +++++++++++++++++++++---- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 2eb4908775..275c51c73d 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3021,6 +3021,55 @@ def and_it_preserves_colspan_when_splitting_an_oversized_cell(self): assert len(chunks) > 1 for _, html in chunks: assert html.startswith('` overhead a fixed constant would assume -- a cell with a large `colspan` must + reserve more room for it, not just for the text content.""" + opts = ChunkingOptions(max_characters=50) + words = " ".join(["word"] * 30) + html_table = HtmlTable.from_html_text( + f'
') + assert len(html) <= 50 + + def and_it_accounts_for_a_large_colspan_attributes_own_overhead_when_splitting(self): + """A `colspan` attribute is real characters ("` colspan="100""), on top of the plain + `
{words}
' + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert len(chunks) > 1 + for _, html in chunks: + assert html.startswith('
') + assert len(html) <= 50 + + def and_it_accounts_for_html_escaping_when_splitting_an_oversized_cell(self): + """Cell text is HTML-escaped (`&` -> `&`, etc.) when formatted, which can make the + formatted fragment longer than the raw text a word-boundary split was budgeted for.""" + opts = ChunkingOptions(max_characters=50) + text = " & ".join(["x"] * 20) + html_table = HtmlTable.from_html_text(f"
{text}
") + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert len(chunks) > 1 + for _, html in chunks: + assert len(html) <= 50 + # -- no text lost or duplicated across the split -- + assert " ".join(text for text, _ in chunks).replace(" & ", " ").split() == ["x"] * 20 + + def and_it_accounts_for_colspan_and_escaping_together_when_splitting_an_oversized_cell(self): + """A large `colspan` and heavy escaping both eat into a cell's usable content budget at + once -- neither can be handled in isolation from the other.""" + opts = ChunkingOptions(max_characters=80) + text = " & ".join(["word"] * 20) + html_table = HtmlTable.from_html_text( + f'
{text}
' + ) + + chunks = list(_HtmlTableSplitter.iter_subtables(html_table, opts)) + + assert len(chunks) > 1 + for _, html in chunks: + assert html.startswith('
') + assert len(html) <= 80 def and_it_uses_the_configured_measurement_units_for_row_fitting( self, monkeypatch: pytest.MonkeyPatch diff --git a/unstructured/chunking/base.py b/unstructured/chunking/base.py index 5109547fde..3efb5cb146 100644 --- a/unstructured/chunking/base.py +++ b/unstructured/chunking/base.py @@ -1507,9 +1507,6 @@ def _iter_row_splits(self, row: HtmlRow, maxlen: int) -> Iterator[TextAndHtml]: def _iter_cell_splits(self, cell: HtmlCell, maxlen: int) -> Iterator[TextAndHtml]: """Split a single oversized cell into sub-sub-sub-table HTML fragments.""" - # -- 33 is len("
"), HTML overhead beyond text content -- - # -- For token-based chunking, we subtract 33 chars worth of overhead but still use tokens - # -- for the actual content limit. For character-based, we use the reduced character limit. if self._opts.use_token_counting: # -- In token mode, keep token limit but account for HTML overhead in char terms -- # -- The HTML tags themselves are usually ~10-15 tokens, so we reduce by a small amount @@ -1517,18 +1514,88 @@ def _iter_cell_splits(self, cell: HtmlCell, maxlen: int) -> Iterator[TextAndHtml max_tokens=max(1, maxlen - 10), tokenizer=self._opts._kwargs.get("tokenizer"), ) - else: - opts = ChunkingOptions(max_characters=max(1, maxlen - 33)) - split = _TextSplitter(opts) - - text, remainder = split(cell.text) - yield text, f"{_format_td(text, cell.colspan, rowspan=1)}
" + split = _TextSplitter(opts) - # -- an oversized cell will have a remainder, split that up into additional chunks. - while remainder: - text, remainder = split(remainder) + text, remainder = split(cell.text) yield text, f"{_format_td(text, cell.colspan, rowspan=1)}
" + while remainder: + text, remainder = split(remainder) + yield text, f"{_format_td(text, cell.colspan, rowspan=1)}
" + return + + # -- overhead depends on this cell's own colspan, so derive it from an empty wrapper -- + empty_td = _format_td("", cell.colspan, rowspan=1) + empty_fragment_len = len(f"{empty_td}
") + budget = max(1, maxlen - empty_fragment_len) + + remaining = cell.text + while True: + text, html, remaining = self._split_cell_fragment( + remaining, cell.colspan, maxlen, budget + ) + yield text, html + if not remaining: + return + + @staticmethod + def _split_cell_fragment( + text: str, colspan: int, maxlen: int, budget: int + ) -> tuple[str, str, str]: + """Split `text` at a word boundary and format it as a `		` fragment, guaranteeing + `len(html) <= maxlen`. + + `budget` is only an upper bound on the raw-text length handed to the word-boundary + splitter -- escaping (`&`, `<`, `>`) can expand a raw character into several, so the + actual formatted length isn't known until after splitting. Binary-searches `budget` for + the largest word-boundary split whose formatted fragment still fits (valid because a + larger budget only ever grows the selected raw text, and escaping never shrinks it, so + formatted length is non-decreasing in `budget`). Falls back to a raw, non-word-boundary + truncation if even a single raw character can't fit (e.g. a lone `&` whose escaped form + alone is longer than the room left). Returns `(split_text, html, remainder)`. + """ + lo, hi, best = 1, budget, None + while lo <= hi: + mid = (lo + hi) // 2 + split = _TextSplitter(ChunkingOptions(max_characters=mid)) + split_text, remainder = split(text) + html = f"{_format_td(split_text, colspan, rowspan=1)}
" + if len(html) <= maxlen: + best = (split_text, html, remainder) + lo = mid + 1 + else: + hi = mid - 1 + + return ( + best + if best is not None + else _HtmlTableSplitter._truncate_cell_fragment(text, colspan, maxlen) + ) + + @staticmethod + def _truncate_cell_fragment(text: str, colspan: int, maxlen: int) -> tuple[str, str, str]: + """Binary-search the longest raw-text prefix of `text` whose escaped, formatted `		` + fragment fits within `maxlen`, ignoring word boundaries. + + `html.escape()` never shrinks a character, so formatted length is non-decreasing in + prefix length, making the search valid. A one-character prefix is used even if it still + overflows (a fixed `colspan` attribute makes the wrapper itself too large for `maxlen`) + so the caller always makes forward progress on `text`. + """ + lo, hi, best = 0, len(text), 0 + while lo <= hi: + mid = (lo + hi) // 2 + candidate = f"{_format_td(text[:mid], colspan, rowspan=1)}
" + if len(candidate) <= maxlen: + best, lo = mid, mid + 1 + else: + hi = mid - 1 + best = max(best, 1) if text else 0 + + split_text = text[:best] + html = f"{_format_td(split_text, colspan, rowspan=1)}
" + return split_text, html, text[best:] + @cached_property def _header_text(self) -> str: """Concatenated text for leading header rows identified by caller.""" From 4f3f81de7d82fdeca19bfb4601ec504da661e10b Mon Sep 17 00:00:00 2001 From: Alan Bertl Date: Tue, 8 Sep 2026 11:04:29 -0500 Subject: [PATCH 19/19] Assert the escaped HTML directly in the cell-escaping split test The test compared only the parallel plain-text tuple element across fragments, never inspecting the emitted HTML itself, so a regression that broke escaping in the HTML specifically (while leaving the plain text correct) would have passed unnoticed. --- test_unstructured/chunking/test_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_unstructured/chunking/test_base.py b/test_unstructured/chunking/test_base.py index 275c51c73d..1615161487 100644 --- a/test_unstructured/chunking/test_base.py +++ b/test_unstructured/chunking/test_base.py @@ -3054,6 +3054,8 @@ def and_it_accounts_for_html_escaping_when_splitting_an_oversized_cell(self): assert len(html) <= 50 # -- no text lost or duplicated across the split -- assert " ".join(text for text, _ in chunks).replace(" & ", " ").split() == ["x"] * 20 + # -- the escape itself actually landed in the emitted HTML, not just the parallel text -- + assert any("&" in html for _, html in chunks) def and_it_accounts_for_colspan_and_escaping_together_when_splitting_an_oversized_cell(self): """A large `colspan` and heavy escaping both eat into a cell's usable content budget at