ENH: allow concat of Styler objects (#46105)

Author: attack68

doc/source/_static/style/footer_extended.png

@@ -42,6 +42,7 @@ Style application
    Styler.format
    Styler.format_index
    Styler.hide
+   Styler.concat
    Styler.set_td_classes
    Styler.set_table_styles
    Styler.set_table_attributes

doc/source/_static/style/footer_simple.png

@@ -21,6 +21,7 @@ Styler
 
   - New method :meth:`.Styler.to_string` for alternative customisable output methods (:issue:`44502`)
   - Added the ability to render ``border`` and ``border-{side}`` CSS properties in Excel (:issue:`42276`)
+  - Added a new method :meth:`.Styler.concat` which allows adding customised footer rows to visualise additional calculations on the data, e.g. totals and counts etc. (:issue:`43875`)
 
 .. _whatsnew_150.enhancements.enhancement2:
 

doc/source/reference/style.rst

@@ -271,6 +271,87 @@ def __init__(
             thousands=thousands,
         )
 
+    def concat(self, other: Styler) -> Styler:
+        """
+        Append another Styler to combine the output into a single table.
+
+        .. versionadded:: 1.5.0
+
+        Parameters
+        ----------
+        other : Styler
+            The other Styler object which has already been styled and formatted. The
+            data for this Styler must have the same columns as the original.
+
+        Returns
+        -------
+        self : Styler
+
+        Notes
+        -----
+        The purpose of this method is to extend existing styled dataframes with other
+        metrics that may be useful but may not conform to the original's structure.
+        For example adding a sub total row, or displaying metrics such as means,
+        variance or counts.
+
+        Styles that are applied using the ``apply``, ``applymap``, ``apply_index``
+        and ``applymap_index``, and formatting applied with ``format`` and
+        ``format_index`` will be preserved.
+
+        .. warning::
+            Only the output methods ``to_html`` and ``to_string`` currently work with
+            concatenated Stylers.
+
+            The output methods ``to_latex`` and ``to_excel`` **do not** work with
+            concatenated Stylers.
+
+        The following should be noted:
+
+          - ``table_styles``, ``table_attributes``, ``caption`` and ``uuid`` are all
+            inherited from the original Styler and not ``other``.
+          - hidden columns and hidden index levels will be inherited from the
+            original Styler
+
+        A common use case is to concatenate user defined functions with
+        ``DataFrame.agg`` or with described statistics via ``DataFrame.describe``.
+        See examples.
+
+        Examples
+        --------
+        A common use case is adding totals rows, or otherwise, via methods calculated
+        in ``DataFrame.agg``.
+
+        >>> df = DataFrame([[4, 6], [1, 9], [3, 4], [5, 5], [9,6]],
+        ...                columns=["Mike", "Jim"],
+        ...                index=["Mon", "Tue", "Wed", "Thurs", "Fri"])
+        >>> styler = df.style.concat(df.agg(["sum"]).style)  # doctest: +SKIP
+
+        .. figure:: ../../_static/style/footer_simple.png
+
+        Since the concatenated object is a Styler the existing functionality can be
+        used to conditionally format it as well as the original.
+
+        >>> descriptors = df.agg(["sum", "mean", lambda s: s.dtype])
+        >>> descriptors.index = ["Total", "Average", "dtype"]
+        >>> other = (descriptors.style
+        ...          .highlight_max(axis=1, subset=(["Total", "Average"], slice(None)))
+        ...          .format(subset=("Average", slice(None)), precision=2, decimal=",")
+        ...          .applymap(lambda v: "font-weight: bold;"))
+        >>> styler = (df.style
+        ...             .highlight_max(color="salmon")
+        ...             .set_table_styles([{"selector": ".foot_row0",
+        ...                                 "props": "border-top: 1px solid black;"}]))
+        >>> styler.concat(other)  # doctest: +SKIP
+
+        .. figure:: ../../_static/style/footer_extended.png
+        """
+        if not isinstance(other, Styler):
+            raise TypeError("`other` must be of type `Styler`")
+        if not self.data.columns.equals(other.data.columns):
+            raise ValueError("`other.data` must have same columns as `Styler.data`")
+        self.concatenated = other
+        return self
+
     def _repr_html_(self) -> str | None:
         """
         Hooks into Jupyter notebook rich display system, which calls _repr_html_ by
@@ -1405,6 +1486,7 @@ def _copy(self, deepcopy: bool = False) -> Styler:
           - cell_context (cell css classes)
           - ctx (cell css styles)
           - caption
+          - concatenated stylers
 
         Non-data dependent attributes [copied and exported]:
           - css
@@ -1435,6 +1517,7 @@ def _copy(self, deepcopy: bool = False) -> Styler:
         ]
         deep = [  # nested lists or dicts
             "css",
+            "concatenated",
             "_display_funcs",
             "_display_funcs_index",
             "_display_funcs_columns",
@@ -2348,11 +2431,13 @@ def set_table_styles(
                                "col_heading": "col_heading",
                                "index_name": "index_name",
                                "col": "col",
+                               "row": "row",
                                "col_trim": "col_trim",
                                "row_trim": "row_trim",
                                "level": "level",
                                "data": "data",
-                               "blank": "blank}
+                               "blank": "blank",
+                               "foot": "foot"}
 
         Examples
         --------

doc/source/whatsnew/v1.5.0.rst

@@ -120,8 +120,9 @@ def __init__(
             "level": "level",
             "data": "data",
             "blank": "blank",
+            "foot": "foot",
         }
-
+        self.concatenated: StylerRenderer | None = None
         # add rendering variables
         self.hide_index_names: bool = False
         self.hide_column_names: bool = False
@@ -148,6 +149,35 @@ def __init__(
             tuple[int, int], Callable[[Any], str]
         ] = defaultdict(lambda: partial(_default_formatter, precision=precision))
 
+    def _render(
+        self,
+        sparse_index: bool,
+        sparse_columns: bool,
+        max_rows: int | None = None,
+        max_cols: int | None = None,
+        blank: str = "",
+    ):
+        """
+        Computes and applies styles and then generates the general render dicts
+        """
+        self._compute()
+        dx = None
+        if self.concatenated is not None:
+            self.concatenated.hide_index_ = self.hide_index_
+            self.concatenated.hidden_columns = self.hidden_columns
+            self.concatenated.css = {
+                **self.css,
+                "data": f"{self.css['foot']}_{self.css['data']}",
+                "row_heading": f"{self.css['foot']}_{self.css['row_heading']}",
+                "row": f"{self.css['foot']}_{self.css['row']}",
+                "foot": self.css["foot"],
+            }
+            dx, _ = self.concatenated._render(
+                sparse_index, sparse_columns, max_rows, max_cols, blank
+            )
+        d = self._translate(sparse_index, sparse_columns, max_rows, max_cols, blank, dx)
+        return d, dx
+
     def _render_html(
         self,
         sparse_index: bool,
@@ -160,9 +190,7 @@ def _render_html(
         Renders the ``Styler`` including all applied styles to HTML.
         Generates a dict with necessary kwargs passed to jinja2 template.
         """
-        self._compute()
-        # TODO: namespace all the pandas keys
-        d = self._translate(sparse_index, sparse_columns, max_rows, max_cols)
+        d, _ = self._render(sparse_index, sparse_columns, max_rows, max_cols, " ")
         d.update(kwargs)
         return self.template_html.render(
             **d,
@@ -176,16 +204,12 @@ def _render_latex(
         """
         Render a Styler in latex format
         """
-        self._compute()
-
-        d = self._translate(sparse_index, sparse_columns, blank="")
+        d, _ = self._render(sparse_index, sparse_columns, None, None)
         self._translate_latex(d, clines=clines)
-
         self.template_latex.globals["parse_wrap"] = _parse_latex_table_wrapping
         self.template_latex.globals["parse_table"] = _parse_latex_table_styles
         self.template_latex.globals["parse_cell"] = _parse_latex_cell_styles
         self.template_latex.globals["parse_header"] = _parse_latex_header_span
-
         d.update(kwargs)
         return self.template_latex.render(**d)
 
@@ -200,10 +224,7 @@ def _render_string(
         """
         Render a Styler in string format
         """
-        self._compute()
-
-        d = self._translate(sparse_index, sparse_columns, max_rows, max_cols, blank="")
-
+        d, _ = self._render(sparse_index, sparse_columns, max_rows, max_cols)
         d.update(kwargs)
         return self.template_string.render(**d)
 
@@ -231,6 +252,7 @@ def _translate(
         max_rows: int | None = None,
         max_cols: int | None = None,
         blank: str = " ",
+        dx: dict | None = None,
     ):
         """
         Process Styler data and settings into a dict for template rendering.
@@ -246,10 +268,12 @@ def _translate(
         sparse_cols : bool
             Whether to sparsify the columns or print all hierarchical column elements.
             Upstream defaults are typically to `pandas.options.styler.sparse.columns`.
-        blank : str
-            Entry to top-left blank cells.
         max_rows, max_cols : int, optional
             Specific max rows and cols. max_elements always take precedence in render.
+        blank : str
+            Entry to top-left blank cells.
+        dx : dict
+            The render dict of the concatenated Styler.
 
         Returns
         -------
@@ -295,7 +319,7 @@ def _translate(
         self.cellstyle_map_index: DefaultDict[
             tuple[CSSPair, ...], list[str]
         ] = defaultdict(list)
-        body = self._translate_body(idx_lengths, max_rows, max_cols)
+        body: list = self._translate_body(idx_lengths, max_rows, max_cols)
         d.update({"body": body})
 
         ctx_maps = {
@@ -310,6 +334,11 @@ def _translate(
             ]
             d.update({k: map})
 
+        if dx is not None:  # self.concatenated is not None
+            d["body"].extend(dx["body"])  # type: ignore[union-attr]
+            d["cellstyle"].extend(dx["cellstyle"])  # type: ignore[union-attr]
+            d["cellstyle_index"].extend(dx["cellstyle"])  # type: ignore[union-attr]
+
         table_attr = self.table_attributes
         if not get_option("styler.html.mathjax"):
             table_attr = table_attr or ""

pandas/io/formats/style.py

@@ -0,0 +1,33 @@
+import pytest
+
+jinja2 = pytest.importorskip("jinja2")
+
+from pandas import DataFrame
+
+from pandas.io.formats.style import Styler
+
+
+@pytest.fixture
+def df():
+    return DataFrame(
+        data=[[0, -0.609], [1, -1.228]],
+        columns=["A", "B"],
+        index=["x", "y"],
+    )
+
+
+@pytest.fixture
+def styler(df):
+    return Styler(df, uuid_len=0)
+
+
+def test_concat_bad_columns(styler):
+    msg = "`other.data` must have same columns as `Styler.data"
+    with pytest.raises(ValueError, match=msg):
+        styler.concat(DataFrame([[1, 2]]).style)
+
+
+def test_concat_bad_type(styler):
+    msg = "`other` must be of type `Styler`"
+    with pytest.raises(TypeError, match=msg):
+        styler.concat(DataFrame([[1, 2]]))

pandas/io/formats/style_render.py

@@ -804,3 +804,24 @@ def test_multiple_rendered_links():
     for link in links:
         assert href.format(link) in result
     assert href.format("text") not in result
+
+
+def test_concat(styler):
+    other = styler.data.agg(["mean"]).style
+    styler.concat(other).set_uuid("X")
+    result = styler.to_html()
+    expected = dedent(
+        """\
+    <tr>
+      <th id="T_X_level0_row1" class="row_heading level0 row1" >b</th>
+      <td id="T_X_row1_col0" class="data row1 col0" >2.690000</td>
+    </tr>
+    <tr>
+      <th id="T_X_level0_foot_row0" class="foot_row_heading level0 foot_row0" >mean</th>
+      <td id="T_X_foot_row0_col0" class="foot_data foot_row0 col0" >2.650000</td>
+    </tr>
+  </tbody>
+</table>
+    """
+    )
+    assert expected in result

pandas/tests/io/formats/style/test_exceptions.py

@@ -56,6 +56,7 @@ def mi_styler_comp(mi_styler):
     mi_styler.hide(axis="index")
     mi_styler.hide([("i0", "i1_a")], axis="index", names=True)
     mi_styler.set_table_attributes('class="box"')
+    mi_styler.concat(mi_styler.data.agg(["mean"]).style)
     mi_styler.format(na_rep="MISSING", precision=3)
     mi_styler.format_index(precision=2, axis=0)
     mi_styler.format_index(precision=4, axis=1)

pandas/tests/io/formats/style/test_html.py

@@ -997,3 +997,19 @@ def test_col_format_len(styler):
     result = styler.to_latex(environment="longtable", column_format="lrr{10cm}")
     expected = r"\multicolumn{4}{r}{Continued on next page} \\"
     assert expected in result
+
+
+@pytest.mark.xfail  # concat not yet implemented for to_latex
+def test_concat(styler):
+    result = styler.concat(styler.data.agg(["sum"]).style).to_latex()
+    expected = dedent(
+        """\
+    \\begin{tabular}{lrrl}
+     & A & B & C \\\\
+    0 & 0 & -0.61 & ab \\\\
+    1 & 1 & -1.22 & cd \\\\
+    sum & 1 & -1.830000 & abcd \\\\
+    \\end{tabular}
+    """
+    )
+    assert result == expected

pandas/tests/io/formats/style/test_style.py

@@ -40,3 +40,16 @@ def test_string_delimiter(styler):
     """
     )
     assert result == expected
+
+
+def test_concat(styler):
+    result = styler.concat(styler.data.agg(["sum"]).style).to_string()
+    expected = dedent(
+        """\
+     A B C
+    0 0 -0.61 ab
+    1 1 -1.22 cd
+    sum 1 -1.830000 abcd
+    """
+    )
+    assert result == expected