ENH: GH9746 DataFrame.unstack and Series.unstack now take fill_value kw for filling NaN when unstack results in a sparse DataFrame

Author: amcpherson

doc/source/reshaping.rst

@@ -228,6 +228,22 @@ which level in the columns to stack:
    df2.stack('exp')
    df2.stack('animal')
 
+Unstacking can result in missing values if subgroups do not have the same
+set of labels.  By default, missing values will be replaced with NaN.
+
+.. ipython:: python
+   
+   df3 = df.ix[[0, 1, 4, 7], [1, 2]]
+   df3
+   df3.unstack()
+
+Alternatively, unstack takes an optional ``fill_value`` argument, for specifying
+the value of missing data.
+
+.. ipython:: python
+   
+   df3.unstack(fill_value=-1e9)
+
 With a MultiIndex
 ~~~~~~~~~~~~~~~~~
 

doc/source/whatsnew/v0.17.0.txt

@@ -39,6 +39,7 @@ Other API Changes
 ^^^^^^^^^^^^^^^^^
 
 - ``Holiday`` now raises ``NotImplementedError`` if both ``offset`` and ``observance`` are used in constructor. (:issue:`102171`)
+- ``DataFrame.unstack`` and ``Series.unstack`` now take ``fill_value`` keyword to allow direct replacement of missing values when an unstack results in missing values in the resulting ``DataFrame``.  As an added benefit, specifying ``fill_value`` will preserve the data type of the original stacked data.
 
 .. _whatsnew_0170.deprecations:
 

pandas/core/frame.py

@@ -3545,7 +3545,7 @@ def stack(self, level=-1, dropna=True):
         else:
             return stack(self, level, dropna=dropna)
 
-    def unstack(self, level=-1):
+    def unstack(self, level=-1, fill_value=None):
         """
         Pivot a level of the (necessarily hierarchical) index labels, returning
         a DataFrame having a new level of column labels whose inner-most level
@@ -3558,6 +3558,8 @@ def unstack(self, level=-1):
         ----------
         level : int, string, or list of these, default -1 (last level)
             Level(s) of index to unstack, can pass level name
+        fill_value : replace NaN with this value if the unstack produces
+            missing values
 
         See also
         --------
@@ -3599,7 +3601,7 @@ def unstack(self, level=-1):
         unstacked : DataFrame or Series
         """
         from pandas.core.reshape import unstack
-        return unstack(self, level)
+        return unstack(self, level, fill_value)
 
     #----------------------------------------------------------------------
     # Time series-related

pandas/core/reshape.py

@@ -61,7 +61,7 @@ class _Unstacker(object):
     unstacked : DataFrame
     """
 
-    def __init__(self, values, index, level=-1, value_columns=None):
+    def __init__(self, values, index, level=-1, value_columns=None, fill_value=None):
 
         self.is_categorical = None
         if values.ndim == 1:
@@ -71,6 +71,7 @@ def __init__(self, values, index, level=-1, value_columns=None):
             values = values[:, np.newaxis]
         self.values = values
         self.value_columns = value_columns
+        self.fill_value = fill_value
 
         if value_columns is None and values.shape[1] != 1:  # pragma: no cover
             raise ValueError('must pass column labels for multi-column data')
@@ -179,6 +180,10 @@ def get_new_values(self):
         if self.mask.all():
             dtype = values.dtype
             new_values = np.empty(result_shape, dtype=dtype)
+        elif self.fill_value is not None:
+            dtype = values.dtype
+            new_values = np.empty(result_shape, dtype=dtype)
+            new_values.fill(self.fill_value)
         else:
             dtype, fill_value = _maybe_promote(values.dtype)
             new_values = np.empty(result_shape, dtype=dtype)
@@ -389,21 +394,21 @@ def _slow_pivot(index, columns, values):
     return DataFrame(tree)
 
 
-def unstack(obj, level):
+def unstack(obj, level, fill_value=None):
     if isinstance(level, (tuple, list)):
         return _unstack_multiple(obj, level)
 
     if isinstance(obj, DataFrame):
         if isinstance(obj.index, MultiIndex):
-            return _unstack_frame(obj, level)
+            return _unstack_frame(obj, level, fill_value=fill_value)
         else:
             return obj.T.stack(dropna=False)
     else:
-        unstacker = _Unstacker(obj.values, obj.index, level=level)
+        unstacker = _Unstacker(obj.values, obj.index, level=level, fill_value=fill_value)
         return unstacker.get_result()
 
 
-def _unstack_frame(obj, level):
+def _unstack_frame(obj, level, fill_value=None):
     from pandas.core.internals import BlockManager, make_block
 
     if obj._is_mixed_type:
@@ -419,7 +424,7 @@ def _unstack_frame(obj, level):
         for blk in obj._data.blocks:
             blk_items = obj._data.items[blk.mgr_locs.indexer]
             bunstacker = _Unstacker(blk.values.T, obj.index, level=level,
-                                    value_columns=blk_items)
+                                    value_columns=blk_items, fill_value=fill_value)
             new_items = bunstacker.get_new_columns()
             new_placement = new_columns.get_indexer(new_items)
             new_values, mask = bunstacker.get_new_values()
@@ -435,7 +440,7 @@ def _unstack_frame(obj, level):
         return result.ix[:, mask_frame.sum(0) > 0]
     else:
         unstacker = _Unstacker(obj.values, obj.index, level=level,
-                               value_columns=obj.columns)
+                               value_columns=obj.columns, fill_value=fill_value)
         return unstacker.get_result()
 
 

pandas/core/series.py

@@ -1914,7 +1914,7 @@ def reorder_levels(self, order):
         result.index = result.index.reorder_levels(order)
         return result
 
-    def unstack(self, level=-1):
+    def unstack(self, level=-1, fill_value=None):
         """
         Unstack, a.k.a. pivot, Series with MultiIndex to produce DataFrame.
         The level involved will automatically get sorted.
@@ -1923,6 +1923,8 @@ def unstack(self, level=-1):
         ----------
         level : int, string, or list of these, default last level
             Level(s) to unstack, can pass level name
+        fill_value : replace NaN with this value if the unstack produces
+            missing values
 
         Examples
         --------
@@ -1947,7 +1949,7 @@ def unstack(self, level=-1):
         unstacked : DataFrame
         """
         from pandas.core.reshape import unstack
-        return unstack(self, level)
+        return unstack(self, level, fill_value)
 
     #----------------------------------------------------------------------
     # function application

pandas/tests/test_frame.py

@@ -12485,6 +12485,34 @@ def test_stack_unstack(self):
         assert_frame_equal(unstacked_cols.T, self.frame)
         assert_frame_equal(unstacked_cols_df['bar'].T, self.frame)
 
+    def test_unstack_fill(self):
+
+        # GH #9746: fill_value keyword argument for Series
+        # and DataFrame unstack
+
+        # From a series
+        data = Series([1, 2, 4, 5], dtype=np.int16)
+        data.index = MultiIndex.from_tuples(
+            [('x', 'a'), ('x', 'b'), ('y', 'b'), ('z', 'a')])
+
+        result = data.unstack(fill_value=-1)
+        expected = DataFrame({'a':[1, -1, 5], 'b':[2, 4, -1]}, index=['x', 'y', 'z'], dtype=np.int16)
+        assert_frame_equal(result, expected)
+
+        # From a dataframe
+        rows = [[1, 2], [3, 4], [5, 6], [7, 8]]
+        df = DataFrame(rows, columns=list('AB'), dtype=np.int32)
+        df.index = MultiIndex.from_tuples(
+            [('x', 'a'), ('x', 'b'), ('y', 'b'), ('z', 'a')])
+        
+        result = df.unstack(fill_value=-1)
+
+        rows = [[1, 3, 2, 4], [-1, 5, -1, 6], [7, -1, 8, -1]]
+        expected = DataFrame(rows, index=list('xyz'), dtype=np.int32)
+        expected.columns = MultiIndex.from_tuples(
+            [('A', 'a'), ('A', 'b'), ('B', 'a'), ('B', 'b')])
+        assert_frame_equal(result, expected)        
+
     def test_stack_ints(self):
         df = DataFrame(
              np.random.randn(30, 27),