Merge pull request #6388 from jreback/dups_dropna

BUG: Bug in DataFrame.dropna with duplicate indices (GH6355)

Author: jreback

doc/source/release.rst

@@ -126,6 +126,7 @@ Bug Fixes
 - Bug in interpolate changing dtypes (:issue:`6290`)
 - Bug in Series.get, was using a buggy access method (:issue:`6383`)
 - Bug in hdfstore queries of the form ``where=[('date', '>=', datetime(2013,1,1)), ('date', '<=', datetime(2014,1,1))]`` (:issue:`6313`)
+- Bug in DataFrame.dropna with duplicate indices (:issue:`6355`)
 
 pandas 0.13.1
 -------------

pandas/core/frame.py

@@ -2414,8 +2414,8 @@ def dropna(self, axis=0, how='any', thresh=None, subset=None,
 
             agg_obj = self
             if subset is not None:
-                agg_axis_name = self._get_axis_name(agg_axis)
-                agg_obj = self.reindex(**{agg_axis_name: subset})
+                ax = self._get_axis(agg_axis)
+                agg_obj = self.take(ax.get_indexer_for(subset),axis=agg_axis)
 
             count = agg_obj.count(axis=agg_axis)
 

pandas/core/index.py

@@ -1210,6 +1210,12 @@ def get_indexer_non_unique(self, target, **kwargs):
         indexer, missing = self._engine.get_indexer_non_unique(tgt_values)
         return Index(indexer), missing
 
+    def get_indexer_for(self, target, **kwargs):
+        """ guaranteed return of an indexer even when non-unique """
+        if self.is_unique:
+            return self.get_indexer(target, **kwargs)
+        return self.get_indexer_non_unique(target, **kwargs)[0]
+
     def _possibly_promote(self, other):
         # A hack, but it works
         from pandas.tseries.index import DatetimeIndex

pandas/tests/test_frame.py

@@ -3256,6 +3256,23 @@ def test_column_dups2(self):
         result = df2.drop('C',axis=1)
         assert_frame_equal(result, expected)
 
+        # dropna
+        df = DataFrame({'A' : np.random.randn(5),
+                        'B' : np.random.randn(5),
+                        'C' : np.random.randn(5),
+                        'D' : ['a','b','c','d','e'] })
+        df.iloc[2,[0,1,2]] = np.nan
+        df.iloc[0,0] = np.nan
+        df.iloc[1,1] = np.nan
+        df.iloc[:,3] = np.nan
+        expected = df.dropna(subset=['A','B','C'],how='all')
+        expected.columns = ['A','A','B','C']
+
+        df.columns = ['A','A','B','C']
+
+        result = df.dropna(subset=['A','C'],how='all')
+        assert_frame_equal(result, expected)
+
     def test_column_dups_indexing(self):
 
         def check(result, expected=None):