Fix DOCX text_as_html duplicating merged-cell text instead of colspan/rowspan

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.27.6
+
+### Fixes
+
+- **Stop duplicating merged-cell text in DOCX `text_as_html`.** A merged cell (`gridSpan`/`vMerge`) was repeated into every `<td>` its merge visually covered, with no `colspan`/`rowspan` attribute marking the merge; merged cells are now emitted once, with `colspan`/`rowspan` reflecting the true geometry. Since DOCX tables can now carry real spans, table chunking was also made rowspan-aware, so a chunk boundary can no longer split a table in a way that misattributes a spanned cell's rows to the wrong columns.
+
 ## 0.27.5
 
 ### Fixes

test_unstructured/common/test_html_table.py

@@ -11,7 +11,9 @@
     HtmlCell,
     HtmlRow,
     HtmlTable,
+    collapse_matrix_of_keyed_cells_to_spans,
     htmlify_matrix_of_cell_texts,
+    htmlify_matrix_of_spanned_cell_texts,
 )
 
 
@@ -43,6 +45,112 @@ def test_htmlify_matrix_handles_empty_matrix(self):
         assert htmlify_matrix_of_cell_texts([]) == ""
 
 
+class Describe_htmlify_matrix_of_spanned_cell_texts:
+    """Unit-test suite for `html_table.htmlify_matrix_of_spanned_cell_texts()`."""
+
+    def it_emits_plain_cells_when_no_span_is_greater_than_1(self):
+        assert htmlify_matrix_of_spanned_cell_texts(
+            [[("a", 1, 1), ("b", 1, 1)], [("c", 1, 1), ("d", 1, 1)]]
+        ) == ("<table><tr><td>a</td><td>b</td></tr><tr><td>c</td><td>d</td></tr></table>")
+
+    def it_emits_colspan_and_rowspan_attributes_only_when_greater_than_1(self):
+        assert htmlify_matrix_of_spanned_cell_texts([[("a", 2, 3)], [("b", 1, 1)]]) == (
+            '<table><tr><td colspan="2" rowspan="3">a</td></tr><tr><td>b</td></tr></table>'
+        )
+
+    def it_emits_a_void_td_for_an_empty_spanned_cell(self):
+        assert htmlify_matrix_of_spanned_cell_texts([[("", 2, 1)]]) == (
+            '<table><tr><td colspan="2"/></tr></table>'
+        )
+
+    def it_emits_an_empty_tr_for_a_row_entirely_consumed_by_a_rowspan(self):
+        """A row with no originating cells still needs its own `<tr>`.
+
+        HTML `rowspan` counts actual `<tr>` elements, not "rows that happened to have content";
+        dropping this row would shift the column-placement of every row after it.
+        """
+        assert htmlify_matrix_of_spanned_cell_texts([[("a", 1, 2)], [], [("b", 1, 1)]]) == (
+            '<table><tr><td rowspan="2">a</td></tr><tr></tr><tr><td>b</td></tr></table>'
+        )
+
+    def it_handles_an_empty_matrix(self):
+        assert htmlify_matrix_of_spanned_cell_texts([]) == ""
+
+    def it_renders_a_full_width_vertical_merge_via_the_full_collapse_and_render_pipeline(self):
+        """Regression: `collapse_matrix_of_keyed_cells_to_spans()` and
+        `htmlify_matrix_of_spanned_cell_texts()` were each unit-tested individually, but their
+        interaction was not -- the collapse step legitimately produces an empty row for a grid-row
+        entirely covered by a multi-column `rowspan`, and the render step must still emit a `<tr>`
+        for it rather than suppressing it.
+        """
+        keyed_matrix = [
+            [("a", "M"), ("a", "M")],
+            [("a", "M"), ("a", "M")],
+            [("c", "C1"), ("d", "C2")],
+        ]
+        spanned_matrix = collapse_matrix_of_keyed_cells_to_spans(keyed_matrix)
+        assert spanned_matrix == [[("a", 2, 2)], [], [("c", 1, 1), ("d", 1, 1)]]
+        assert htmlify_matrix_of_spanned_cell_texts(spanned_matrix) == (
+            '<table><tr><td colspan="2" rowspan="2">a</td></tr>'
+            "<tr></tr>"
+            "<tr><td>c</td><td>d</td></tr></table>"
+        )
+
+
+class Describe_collapse_matrix_of_keyed_cells_to_spans:
+    """Unit-test suite for `html_table.collapse_matrix_of_keyed_cells_to_spans()`."""
+
+    def it_leaves_a_matrix_with_no_repeated_keys_unchanged(self):
+        matrix = [[("a", 1), ("b", 2)], [("c", 3), ("d", 4)]]
+        assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [
+            [("a", 1, 1), ("b", 1, 1)],
+            [("c", 1, 1), ("d", 1, 1)],
+        ]
+
+    def it_collapses_a_horizontal_run_of_matching_keys_into_a_colspan(self):
+        matrix = [[("a", 1), ("a", 1), ("b", 2)]]
+        assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [[("a", 2, 1), ("b", 1, 1)]]
+
+    def it_collapses_a_vertical_run_of_matching_keys_into_a_rowspan(self):
+        matrix = [[("a", 1)], [("a", 1)], [("b", 2)]]
+        assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [
+            [("a", 1, 2)],
+            [],
+            [("b", 1, 1)],
+        ]
+
+    def it_collapses_a_rectangular_region_into_a_single_cell_with_colspan_and_rowspan(self):
+        """Reproduces the docx-tables.docx merged-cell fixture geometry.
+
+        +---+-------+
+        | a | b     |
+        |   +---+---+
+        |   | c | d |
+        +---+---+   |
+        | e     |   |
+        +-------+---+
+        """
+        matrix = [
+            [("a", 1), ("b", 2), ("b", 2)],
+            [("a", 1), ("c", 3), ("d", 4)],
+            [("e", 5), ("e", 5), ("d", 4)],
+        ]
+        assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [
+            [("a", 1, 2), ("b", 2, 1)],
+            [("c", 1, 1), ("d", 1, 2)],
+            [("e", 2, 1)],
+        ]
+
+    def it_treats_distinct_keys_as_never_merged_even_when_their_text_matches(self):
+        matrix = [[("", 1), ("", 2)]]
+        assert collapse_matrix_of_keyed_cells_to_spans(matrix) == [
+            [("", 1, 1), ("", 1, 1)],
+        ]
+
+    def it_handles_an_empty_matrix(self):
+        assert collapse_matrix_of_keyed_cells_to_spans([]) == []
+
+
 class DescribeHtmlTable:
     """Unit-test suite for `unstructured.common.html_table.HtmlTable`."""
 
@@ -289,6 +397,10 @@ def and_it_preserves_nested_markup_when_serializing_nonempty_cells(self):
             '<td><a href="#">Category Link</a></td>'
         )
 
+    def and_it_preserves_colspan_and_rowspan_on_an_empty_cell(self):
+        cell = HtmlCell(fragment_fromstring('<td colspan="2" rowspan="3"></td>'))
+        assert cell.html == '<td colspan="2" rowspan="3"/>'
+
     @pytest.mark.parametrize(
         ("cell_html", "expected_value"),
         [

test_unstructured/partition/test_docx.py

@@ -149,6 +149,134 @@ def test_partition_docx_processes_table():
     assert elements[0].metadata.filename == "fake_table.docx"
 
 
+def test_partition_docx_table_with_merged_cells_reports_spans_instead_of_duplicating_text():
+    """A merged cell's text must appear in exactly one `<td>`, marked with colspan/rowspan.
+
+    Fixture table is:
+
+        +---+-------+
+        | a | b     |
+        |   +---+---+
+        |   | c | d |
+        +---+---+   |
+        | e     |   |
+        +-------+---+
+    """
+    elements = partition_docx(example_doc_path("docx-tables.docx"), infer_table_structure=True)
+    tables = [e for e in elements if isinstance(e, Table)]
+    table = next(t for t in tables if t.text == "a b c d e")
+
+    assert table.metadata.text_as_html == (
+        "<table>"
+        '<tr><td rowspan="2">a</td><td colspan="2">b</td></tr>'
+        '<tr><td>c</td><td rowspan="2">d</td></tr>'
+        '<tr><td colspan="2">e</td></tr>'
+        "</table>"
+    )
+    # -- no cell's text is duplicated across more than one `<td>` --
+    for letter in "abcde":
+        assert table.metadata.text_as_html.count(f">{letter}<") == 1
+
+
+def test_partition_docx_table_with_full_width_vertical_merge_reports_a_tr_for_every_row(tmp_path):
+    """A grid-row entirely covered by a `rowspan` (no originating cells of its own) still needs
+    its own `<tr>` in the output. HTML `rowspan` counts actual `<tr>` elements, not "rows that
+    happened to have content" -- suppressing this row would shift the column-placement of every
+    row after it.
+
+        +---+
+        | A |
+        |   |
+        +---+
+        | B |
+        +---+
+    """
+    document = docx.Document()
+    table = document.add_table(rows=3, cols=1)
+    table.cell(0, 0).merge(table.cell(1, 0)).text = "A"
+    table.cell(2, 0).text = "B"
+    docx_path = tmp_path / "vertical-merge.docx"
+    document.save(str(docx_path))
+
+    elements = partition_docx(str(docx_path), infer_table_structure=True)
+    table_element = next(e for e in elements if isinstance(e, Table))
+
+    assert table_element.metadata.text_as_html == (
+        '<table><tr><td rowspan="2">A</td></tr><tr></tr><tr><td>B</td></tr></table>'
+    )
+
+
+def test_partition_docx_table_with_merged_cell_in_nested_table_does_not_duplicate_its_text(
+    tmp_path,
+):
+    """A merged cell in a table nested inside another table's cell contributes its text once.
+
+    Nested tables are flattened to plain text rather than nested `<table>` HTML, but that
+    flattening must still collapse a merged cell to a single occurrence of its text.
+    """
+    document = docx.Document()
+    outer_table = document.add_table(rows=1, cols=1)
+    nested_table = outer_table.cell(0, 0).add_table(rows=2, cols=2)
+    nested_table.cell(0, 0).merge(nested_table.cell(0, 1)).text = "MERGEDNESTED"
+    nested_table.cell(1, 0).text = "foo"
+    nested_table.cell(1, 1).text = "bar"
+    docx_path = tmp_path / "nested-merged-cell.docx"
+    document.save(str(docx_path))
+
+    elements = partition_docx(str(docx_path), infer_table_structure=True)
+    table_element = next(e for e in elements if isinstance(e, Table))
+
+    assert table_element.text == "MERGEDNESTED foo bar"
+    assert table_element.metadata.text_as_html == (
+        "<table><tr><td>MERGEDNESTED foo bar</td></tr></table>"
+    )
+
+
+def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geometry(tmp_path):
+    """A DOCX table with a real vertical merge, partitioned then chunked with a small window,
+    must never split between rows an active `rowspan` still covers without correcting for it --
+    doing so would leave a continuation `TableChunk` with cells shifted into the wrong column.
+    Splitting is fine as long as the covering cell's `rowspan` is rewritten to match, and
+    re-materialized in whichever fragment doesn't hold the row that originally declared it."""
+    document = docx.Document()
+    table = document.add_table(rows=4, cols=2)
+    table.cell(0, 0).merge(table.cell(1, 0)).merge(table.cell(2, 0)).text = "REGIONWIDE TOTAL"
+    table.cell(0, 1).text = "alpha bravo charlie"
+    table.cell(1, 1).text = "delta echo foxtrot"
+    table.cell(2, 1).text = "golf hotel india"
+    table.cell(3, 0).text = "juliet"
+    table.cell(3, 1).text = "kilo lima mike"
+    docx_path = tmp_path / "merged-cell-chunking.docx"
+    document.save(str(docx_path))
+
+    elements = partition_docx(str(docx_path), infer_table_structure=True)
+    table_element = next(e for e in elements if isinstance(e, Table))
+    assert 'rowspan="3"' in table_element.metadata.text_as_html
+
+    chunks = chunk_by_title([table_element], max_characters=60)
+
+    assert len(chunks) == 3, "fixture should be oversized enough to actually require a split"
+    assert all(isinstance(chunk, TableChunk) for chunk in chunks)
+    assert all(len(chunk.text) <= 60 for chunk in chunks)
+    # -- the rowspan-3 group is itself too big for the window, so it's split on a row boundary;
+    # -- the covering cell's rowspan is rewritten to the rows present in the first fragment --
+    assert chunks[0].metadata.text_as_html == (
+        "<table>"
+        '<tr><td rowspan="2">REGIONWIDE TOTAL</td><td>alpha bravo charlie</td></tr>'
+        "<tr><td>delta echo foxtrot</td></tr>"
+        "</table>"
+    )
+    # -- the covering cell is re-materialized (rowspan="1") in the next fragment, which doesn't
+    # -- include the row that originally declared the span, keeping columns aligned --
+    assert chunks[1].metadata.text_as_html == (
+        "<table><tr><td>REGIONWIDE TOTAL</td><td>golf hotel india</td></tr></table>"
+    )
+    # -- the final row lands in its own chunk with both its cells correctly positioned --
+    assert chunks[2].metadata.text_as_html == (
+        "<table><tr><td>juliet</td><td>kilo lima mike</td></tr></table>"
+    )
+
+
 def test_partition_docx_grabs_header_and_footer():
     elements = partition_docx(example_doc_path("handbook-1p.docx"))
 
@@ -1072,6 +1200,31 @@ def but_the_text_of_a_merged_cell_appears_only_once(self, opts_args: dict[str, A
         table = docx.Document(example_doc_path("docx-tables.docx")).tables[2]
         assert " ".join(_DocxPartitioner(opts)._iter_table_texts(table)) == "a b c d e"
 
+    def and_the_html_of_a_merged_cell_carries_colspan_and_rowspan_instead_of_repeating_text(
+        self, opts_args: dict[str, Any]
+    ):
+        """
+        Fixture table is:
+
+            +---+-------+
+            | a | b     |
+            |   +---+---+
+            |   | c | d |
+            +---+---+   |
+            | e     |   |
+            +-------+---+
+        """
+        opts = DocxPartitionerOptions(**opts_args)
+        table = docx.Document(example_doc_path("docx-tables.docx")).tables[2]
+
+        assert _DocxPartitioner(opts)._convert_table_to_html(table) == (
+            "<table>"
+            '<tr><td rowspan="2">a</td><td colspan="2">b</td></tr>'
+            '<tr><td>c</td><td rowspan="2">d</td></tr>'
+            '<tr><td colspan="2">e</td></tr>'
+            "</table>"
+        )
+
     def it_can_partition_tables_with_incomplete_rows(self):
         """DOCX permits table rows to start late and end early.
 
@@ -1128,7 +1281,7 @@ def it_can_partition_tables_with_incomplete_rows(self):
         assert e.text == "a b c d", f"actual {e.text=}"
         assert e.metadata.text_as_html == (
             "<table>"
-            "<tr><td>a</td><td>a</td><td/></tr>"
+            '<tr><td colspan="2">a</td><td/></tr>'
             "<tr><td>b</td><td>c</td><td>d</td></tr>"
             "</table>"
         ), f"actual {e.metadata.text_as_html=}"
@@ -1143,8 +1296,8 @@ def it_can_partition_tables_with_incomplete_rows(self):
         assert e.text == "a b c d", f"actual {e.text=}"
         assert e.metadata.text_as_html == (
             "<table>"
-            "<tr><td>a</td><td>b</td><td/></tr>"
-            "<tr><td>a</td><td>c</td><td>d</td></tr>"
+            '<tr><td rowspan="2">a</td><td>b</td><td/></tr>'
+            "<tr><td>c</td><td>d</td></tr>"
             "</table>"
         ), f"actual {e.metadata.text_as_html=}"
         # -- late-start, early-end, and >2 rows vertical span --
@@ -1162,10 +1315,10 @@ def it_can_partition_tables_with_incomplete_rows(self):
         assert e.text == "a b c d e f", f"actual {e.text=}"
         assert e.metadata.text_as_html == (
             "<table>"
-            "<tr><td>a</td><td>a</td><td>b</td><td>c</td></tr>"
-            "<tr><td/><td>d</td><td>d</td><td/></tr>"
-            "<tr><td>e</td><td>d</td><td>d</td><td>f</td></tr>"
-            "<tr><td/><td>d</td><td>d</td><td/></tr>"
+            '<tr><td colspan="2">a</td><td>b</td><td>c</td></tr>'
+            '<tr><td/><td colspan="2" rowspan="3">d</td><td/></tr>'
+            "<tr><td>e</td><td>f</td></tr>"
+            "<tr><td/><td/></tr>"
             "</table>"
         ), f"actual {e.metadata.text_as_html=}"
         # --
@@ -1175,14 +1328,14 @@ def it_can_partition_tables_with_incomplete_rows(self):
         assert e.text == "Data More Dato WTF? Strange Format", f"actual {e.text=}"
         assert e.metadata.text_as_html == (
             "<table>"
-            "<tr><td>Data</td><td>Data</td><td/></tr>"
-            "<tr><td>Data</td><td>Data</td><td/></tr>"
-            "<tr><td>Data</td><td>Data</td><td/></tr>"
+            '<tr><td colspan="2" rowspan="3">Data</td><td/></tr>'
+            "<tr><td/></tr>"
+            "<tr><td/></tr>"
             "<tr><td/><td>More</td><td/></tr>"
             "<tr><td>Dato</td><td/></tr>"
-            "<tr><td>WTF?</td><td>WTF?</td><td/></tr>"
-            "<tr><td>Strange</td><td>Strange</td><td/></tr>"
-            "<tr><td/><td>Format</td><td>Format</td></tr>"
+            '<tr><td colspan="2">WTF?</td><td/></tr>'
+            '<tr><td colspan="2">Strange</td><td/></tr>'
+            '<tr><td/><td colspan="2">Format</td></tr>'
             "</table>"
         ), f"actual {e.metadata.text_as_html=}"
 

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.27.5"  # pragma: no cover
+__version__ = "0.27.6"  # pragma: no cover