ENH: Be able to add lines for all index levels, not just visible ones [fix #59877]

doc/source/whatsnew/v3.0.0.rst

@@ -65,6 +65,8 @@ Other enhancements
 - Support passing a :class:`Iterable[Hashable]` input to :meth:`DataFrame.drop_duplicates` (:issue:`59237`)
 - Support reading Stata 102-format (Stata 1) dta files (:issue:`58978`)
 - Support reading Stata 110-format (Stata 7) dta files (:issue:`47176`)
+- The ``clines`` parameter to :meth:`Styler.to_latex` now accepts a tuple of option strings. The previous interface of passing a single string for multiple options is deprecated.
+- :meth:`Styler.to_latex` accepts additional options to the ``clines`` parameter, allowing lines to be drawn between hidden index levels.
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.notable_bug_fixes:

pandas/core/generic.py

@@ -3582,7 +3582,7 @@ def _wrap(x, alt_format_):
             "label": label,
             "position": position,
             "column_format": column_format,
-            "clines": "skip-last;data"
+            "clines": ("skip-last", "rule-data")
             if (multirow and isinstance(self.index, MultiIndex))
             else None,
             "bold_rows": bold_rows,

pandas/io/formats/style.py

@@ -620,7 +620,7 @@ def to_latex(
         position: str | None = ...,
         position_float: str | None = ...,
         hrules: bool | None = ...,
-        clines: str | None = ...,
+        clines: str | tuple | None = ...,
         label: str | None = ...,
         caption: str | tuple | None = ...,
         sparse_index: bool | None = ...,
@@ -642,7 +642,7 @@ def to_latex(
         position: str | None = ...,
         position_float: str | None = ...,
         hrules: bool | None = ...,
-        clines: str | None = ...,
+        clines: str | tuple | None = ...,
         label: str | None = ...,
         caption: str | tuple | None = ...,
         sparse_index: bool | None = ...,
@@ -663,7 +663,7 @@ def to_latex(
         position: str | None = None,
         position_float: str | None = None,
         hrules: bool | None = None,
-        clines: str | None = None,
+        clines: str | tuple | None = None,
         label: str | None = None,
         caption: str | tuple | None = None,
         sparse_index: bool | None = None,
@@ -712,22 +712,36 @@ def to_latex(
             Defaults to ``pandas.options.styler.latex.hrules``, which is `False`.
 
             .. versionchanged:: 1.4.0
-        clines : str, optional
+        clines : str, tuple, optional
             Use to control adding \\cline commands for the index labels separation.
-            Possible values are:
+            If `None`, then no cline commands are added (default).
+
+            If a tuple is passed, the following elements are recognised:
+              - `"rule-data"`: if present, clines are added for both the index and data.
+                Otherwise, they are only drawn for the index.
+              - `"skip-last"`: if present, the last index level is omitted when deciding
+                where to add clines. Otherwise, all index levels are used.
+              - `"include-hidden"`: if present, hidden index levels are included when
+                deciding where to add clines. Otherwise, only visible index levels are
+                used.
+
+            If a str is passed, possible values are:
 
               - `None`: no cline commands are added (default).
-              - `"all;data"`: a cline is added for every index value extending the
-                width of the table, including data entries.
+              - `"all;data"`: a cline is added for every visible index value extending
+                the width of the table, including data entries.
               - `"all;index"`: as above with lines extending only the width of the
                 index entries.
-              - `"skip-last;data"`: a cline is added for each index value except the
-                last level (which is never sparsified), extending the widtn of the
+              - `"skip-last;data"`: a cline is added for each visible index value except
+                the last level (which is never sparsified), extending the widtn of the
                 table.
               - `"skip-last;index"`: as above with lines extending only the width of the
                 index entries.
 
-            .. versionadded:: 1.4.0
+            .. versionchanged:: 3.0.0
+            .. deprecated:: 3.0.0
+                Passing a str is deprecated and will be removed in a future version,
+                please use a tuple instead.
         label : str, optional
             The LaTeX label included as: \\label{<label>}.
             This is used with \\ref{<label>} in the main .tex file.
@@ -1080,7 +1094,7 @@ def to_latex(
 
         >>> styler.to_latex(
         ...     caption="Selected stock correlation and simple statistics.",
-        ...     clines="skip-last;data",
+        ...     clines=("skip-last", "across-data"),
         ...     convert_css=True,
         ...     position_float="centering",
         ...     multicol_align="|c|",

pandas/io/formats/style_render.py

@@ -16,13 +16,15 @@
     Union,
 )
 from uuid import uuid4
+import warnings
 
 import numpy as np
 
 from pandas._config import get_option
 
 from pandas._libs import lib
 from pandas.compat._optional import import_optional_dependency
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
     is_complex,
@@ -218,7 +220,11 @@ def _render_html(
         )
 
     def _render_latex(
-        self, sparse_index: bool, sparse_columns: bool, clines: str | None, **kwargs
+        self,
+        sparse_index: bool,
+        sparse_columns: bool,
+        clines: str | tuple | None,
+        **kwargs,
     ) -> str:
         """
         Render a Styler in latex format
@@ -857,7 +863,33 @@ def _generate_body_row(
 
         return index_headers + data
 
-    def _translate_latex(self, d: dict, clines: str | None) -> None:
+    def _convert_clines_to_tuple(self, clines: str | tuple | None) -> tuple | None:
+        if not isinstance(clines, str):
+            return clines
+
+        msg = (
+            "Passing a string argument to the clines parameter is deprecated and will "
+            "be removed in a future version, please use a tuple instead, "
+            "e.g. ('rule-data', 'skip-last') instead of 'skip-last;data'."
+        )
+        warnings.warn(msg, FutureWarning, stacklevel=find_stack_level())
+
+        if clines not in ["all;data", "all;index", "skip-last;data", "skip-last;index"]:
+            raise ValueError(
+                f"`clines` value of {clines} is invalid. Should either be None, "
+                "a tuple, or one of 'all;data', 'all;index', 'skip-last;data', "
+                "'skip-last;index'."
+            )
+
+        result: tuple[str, ...] = ()
+        if "data" in clines:
+            result += ("rule-data",)
+        if "skip-last" in clines:
+            result += ("skip-last",)
+
+        return result
+
+    def _translate_latex(self, d: dict, clines: str | tuple | None) -> None:
         r"""
         Post-process the default render dict for the LaTeX template format.
 
@@ -867,6 +899,7 @@ def _translate_latex(self, d: dict, clines: str | None) -> None:
           - Remove hidden indexes or reinsert missing th elements if part of multiindex
             or multirow sparsification (so that \multirow and \multicol work correctly).
         """
+        clines = self._convert_clines_to_tuple(clines)
         index_levels = self.index.nlevels
         # GH 52218
         visible_index_level_n = max(1, index_levels - sum(self.hide_index_))
@@ -929,19 +962,15 @@ def concatenated_visible_rows(obj):
 
         # clines are determined from info on index_lengths and hidden_rows and input
         # to a dict defining which row clines should be added in the template.
-        if clines not in [
-            None,
-            "all;data",
-            "all;index",
-            "skip-last;data",
-            "skip-last;index",
-        ]:
-            raise ValueError(
-                f"`clines` value of {clines} is invalid. Should either be None or one "
-                f"of 'all;data', 'all;index', 'skip-last;data', 'skip-last;index'."
-            )
         if clines is not None:
-            data_len = len(row_body_cells) if "data" in clines and d["body"] else 0
+            valid_clines_options = ["skip-last", "rule-data", "include-hidden"]
+            for option in clines:
+                if option not in valid_clines_options:
+                    raise ValueError(
+                        f"`clines` option of {option} is invalid. "
+                        f"Choose one of {valid_clines_options}"
+                    )
+            data_len = len(row_body_cells) if "rule-data" in clines and d["body"] else 0
 
             d["clines"] = defaultdict(list)
             visible_row_indexes: list[int] = [
@@ -951,14 +980,22 @@ def concatenated_visible_rows(obj):
                 i for i in range(index_levels) if not self.hide_index_[i]
             ]
             for rn, r in enumerate(visible_row_indexes):
-                for lvln, lvl in enumerate(visible_index_levels):
+                lvln = 0
+                for lvl in range(index_levels):
+                    if self.hide_index_[lvl] and "include-hidden" not in clines:
+                        continue
                     if lvl == index_levels - 1 and "skip-last" in clines:
                         continue
                     idx_len = d["index_lengths"].get((lvl, r), None)
                     if idx_len is not None:  # i.e. not a sparsified entry
-                        d["clines"][rn + idx_len].append(
-                            f"\\cline{{{lvln+1}-{len(visible_index_levels)+data_len}}}"
-                        )
+                        cline_start_col = lvln + 1
+                        cline_end_col = len(visible_index_levels) + data_len
+                        if cline_end_col >= cline_start_col:
+                            d["clines"][rn + idx_len].append(
+                                f"\\cline{{{cline_start_col}-{cline_end_col}}}"
+                            )
+                    if lvl in visible_index_levels:
+                        lvln += 1
 
     def format(
         self,