Merge pull request #8634 from jreback/numeric_index

BUG: bug in Float/Int index with ndarray for add/sub numeric ops (GH8608)

Author: jreback

doc/source/whatsnew/v0.15.1.txt

@@ -49,5 +49,5 @@ Bug Fixes
 ~~~~~~~~~
 
 - Bug in ``cut``/``qcut`` when using ``Series`` and ``retbins=True`` (:issue:`8589`)
-
+- Bug in numeric index operations of add/sub with Float/Index Index with numpy arrays (:issue:`8608`)
 - Fix ``shape`` attribute for ``MultiIndex`` (:issue:`8609`)

pandas/core/index.py

@@ -1147,21 +1147,21 @@ def __add__(self, other):
                           "use '|' or .union()",FutureWarning)
             return self.union(other)
         return Index(np.array(self) + other)
-
     __iadd__ = __add__
-    __eq__ = _indexOp('__eq__')
-    __ne__ = _indexOp('__ne__')
-    __lt__ = _indexOp('__lt__')
-    __gt__ = _indexOp('__gt__')
-    __le__ = _indexOp('__le__')
-    __ge__ = _indexOp('__ge__')
 
     def __sub__(self, other):
         if isinstance(other, Index):
             warnings.warn("using '-' to provide set differences with Indexes is deprecated, "
                           "use .difference()",FutureWarning)
         return self.difference(other)
 
+    __eq__ = _indexOp('__eq__')
+    __ne__ = _indexOp('__ne__')
+    __lt__ = _indexOp('__lt__')
+    __gt__ = _indexOp('__gt__')
+    __le__ = _indexOp('__le__')
+    __ge__ = _indexOp('__ge__')
+
     def __and__(self, other):
         return self.intersection(other)
 
@@ -2098,7 +2098,7 @@ def _invalid_op(self, other=None):
     def _add_numeric_methods(cls):
         """ add in numeric methods """
 
-        def _make_evaluate_binop(op, opstr):
+        def _make_evaluate_binop(op, opstr, reversed=False):
 
             def _evaluate_numeric_binop(self, other):
                 import pandas.tseries.offsets as offsets
@@ -2128,7 +2128,13 @@ def _evaluate_numeric_binop(self, other):
                 else:
                     if not (com.is_float(other) or com.is_integer(other)):
                         raise TypeError("can only perform ops with scalar values")
-                return self._shallow_copy(op(self.values, other))
+
+                # if we are a reversed non-communative op
+                values = self.values
+                if reversed:
+                    values, other = other, values
+
+                return self._shallow_copy(op(values, other))
 
             return _evaluate_numeric_binop
 
@@ -2145,16 +2151,23 @@ def _evaluate_numeric_unary(self):
 
             return _evaluate_numeric_unary
 
+        cls.__add__ = cls.__radd__ = _make_evaluate_binop(operator.add,'__add__')
+        cls.__sub__ = _make_evaluate_binop(operator.sub,'__sub__')
+        cls.__rsub__ = _make_evaluate_binop(operator.sub,'__sub__',reversed=True)
         cls.__mul__ = cls.__rmul__ = _make_evaluate_binop(operator.mul,'__mul__')
-        cls.__floordiv__ = cls.__rfloordiv__ = _make_evaluate_binop(operator.floordiv,'__floordiv__')
-        cls.__truediv__ = cls.__rtruediv__ = _make_evaluate_binop(operator.truediv,'__truediv__')
+        cls.__floordiv__ = _make_evaluate_binop(operator.floordiv,'__floordiv__')
+        cls.__rfloordiv__ = _make_evaluate_binop(operator.floordiv,'__floordiv__',reversed=True)
+        cls.__truediv__ = _make_evaluate_binop(operator.truediv,'__truediv__')
+        cls.__rtruediv__ = _make_evaluate_binop(operator.truediv,'__truediv__',reversed=True)
         if not compat.PY3:
-            cls.__div__ = cls.__rdiv__ = _make_evaluate_binop(operator.div,'__div__')
+            cls.__div__ = _make_evaluate_binop(operator.div,'__div__')
+            cls.__rdiv__ = _make_evaluate_binop(operator.div,'__div__',reversed=True)
         cls.__neg__ = _make_evaluate_unary(lambda x: -x,'__neg__')
         cls.__pos__ = _make_evaluate_unary(lambda x: x,'__pos__')
         cls.__abs__ = _make_evaluate_unary(lambda x: np.abs(x),'__abs__')
         cls.__inv__ = _make_evaluate_unary(lambda x: -x,'__inv__')
 
+
 Index._add_numeric_methods_disabled()
 
 class NumericIndex(Index):
@@ -2166,7 +2179,6 @@ class NumericIndex(Index):
     """
     _is_numeric_dtype = True
 
-
 class Int64Index(NumericIndex):
 
     """

pandas/stats/plm.py

@@ -240,7 +240,7 @@ def _add_entity_effects(self, panel):
 
             self.log('-- Excluding dummy for entity: %s' % to_exclude)
 
-            dummies = dummies.filter(dummies.columns - [to_exclude])
+            dummies = dummies.filter(dummies.columns.difference([to_exclude]))
 
         dummies = dummies.add_prefix('FE_')
         panel = panel.join(dummies)
@@ -286,7 +286,7 @@ def _add_categorical_dummies(self, panel, cat_mappings):
                 self.log(
                     '-- Excluding dummy for %s: %s' % (effect, to_exclude))
 
-                dummies = dummies.filter(dummies.columns - [mapped_name])
+                dummies = dummies.filter(dummies.columns.difference([mapped_name]))
                 dropped_dummy = True
 
             dummies = _convertDummies(dummies, cat_mappings.get(effect))

pandas/tests/test_index.py

@@ -595,7 +595,7 @@ def test_add(self):
 
         # - API change GH 8226
         with tm.assert_produces_warning():
-            self.strIndex + self.dateIndex
+            self.strIndex + self.strIndex
 
         firstCat = self.strIndex.union(self.dateIndex)
         secondCat = self.strIndex.union(self.strIndex)
@@ -729,7 +729,7 @@ def test_symmetric_diff(self):
 
         # other isn't iterable
         with tm.assertRaises(TypeError):
-            idx1 - 1
+            Index(idx1,dtype='object') - 1
 
     def test_pickle(self):
 
@@ -1132,12 +1132,37 @@ def test_numeric_compat(self):
         tm.assert_index_equal(result,
                               Float64Index(np.arange(5,dtype='float64')*(np.arange(5,dtype='float64')+0.1)))
 
-
         # invalid
         self.assertRaises(TypeError, lambda : idx * date_range('20130101',periods=5))
         self.assertRaises(ValueError, lambda : idx * self._holder(np.arange(3)))
         self.assertRaises(ValueError, lambda : idx * np.array([1,2]))
 
+
+    def test_explicit_conversions(self):
+
+        # GH 8608
+        # add/sub are overriden explicity for Float/Int Index
+        idx = self._holder(np.arange(5,dtype='int64'))
+
+        # float conversions
+        arr = np.arange(5,dtype='int64')*3.2
+        expected = Float64Index(arr)
+        fidx = idx * 3.2
+        tm.assert_index_equal(fidx,expected)
+        fidx = 3.2 * idx
+        tm.assert_index_equal(fidx,expected)
+
+        # interops with numpy arrays
+        expected = Float64Index(arr)
+        a = np.zeros(5,dtype='float64')
+        result = fidx - a
+        tm.assert_index_equal(result,expected)
+
+        expected = Float64Index(-arr)
+        a = np.zeros(5,dtype='float64')
+        result = a - fidx
+        tm.assert_index_equal(result,expected)
+
     def test_ufunc_compat(self):
         idx = self._holder(np.arange(5,dtype='int64'))
         result = np.sin(idx)

pandas/tseries/base.py

@@ -317,50 +317,56 @@ def _add_datelike(self, other):
     def _sub_datelike(self, other):
         raise NotImplementedError
 
-    def __add__(self, other):
-        from pandas.core.index import Index
-        from pandas.tseries.tdi import TimedeltaIndex
-        from pandas.tseries.offsets import DateOffset
-        if isinstance(other, TimedeltaIndex):
-            return self._add_delta(other)
-        elif isinstance(self, TimedeltaIndex) and isinstance(other, Index):
-            if hasattr(other,'_add_delta'):
-                return other._add_delta(self)
-            raise TypeError("cannot add TimedeltaIndex and {typ}".format(typ=type(other)))
-        elif isinstance(other, Index):
-            return self.union(other)
-        elif isinstance(other, (DateOffset, timedelta, np.timedelta64, tslib.Timedelta)):
-            return self._add_delta(other)
-        elif com.is_integer(other):
-            return self.shift(other)
-        elif isinstance(other, (tslib.Timestamp, datetime)):
-            return self._add_datelike(other)
-        else:  # pragma: no cover
-            return NotImplemented
-
-    def __sub__(self, other):
-        from pandas.core.index import Index
-        from pandas.tseries.tdi import TimedeltaIndex
-        from pandas.tseries.offsets import DateOffset
-        if isinstance(other, TimedeltaIndex):
-            return self._add_delta(-other)
-        elif isinstance(self, TimedeltaIndex) and isinstance(other, Index):
-            if not isinstance(other, TimedeltaIndex):
-                raise TypeError("cannot subtract TimedeltaIndex and {typ}".format(typ=type(other)))
-            return self._add_delta(-other)
-        elif isinstance(other, Index):
-            return self.difference(other)
-        elif isinstance(other, (DateOffset, timedelta, np.timedelta64, tslib.Timedelta)):
-            return self._add_delta(-other)
-        elif com.is_integer(other):
-            return self.shift(-other)
-        elif isinstance(other, (tslib.Timestamp, datetime)):
-            return self._sub_datelike(other)
-        else:  # pragma: no cover
-            return NotImplemented
-
-    __iadd__ = __add__
-    __isub__ = __sub__
+    @classmethod
+    def _add_datetimelike_methods(cls):
+        """ add in the datetimelike methods (as we may have to override the superclass) """
+
+        def __add__(self, other):
+            from pandas.core.index import Index
+            from pandas.tseries.tdi import TimedeltaIndex
+            from pandas.tseries.offsets import DateOffset
+            if isinstance(other, TimedeltaIndex):
+                return self._add_delta(other)
+            elif isinstance(self, TimedeltaIndex) and isinstance(other, Index):
+                if hasattr(other,'_add_delta'):
+                    return other._add_delta(self)
+                raise TypeError("cannot add TimedeltaIndex and {typ}".format(typ=type(other)))
+            elif isinstance(other, Index):
+                return self.union(other)
+            elif isinstance(other, (DateOffset, timedelta, np.timedelta64, tslib.Timedelta)):
+                return self._add_delta(other)
+            elif com.is_integer(other):
+                return self.shift(other)
+            elif isinstance(other, (tslib.Timestamp, datetime)):
+                return self._add_datelike(other)
+            else:  # pragma: no cover
+                return NotImplemented
+        cls.__add__ = __add__
+
+        def __sub__(self, other):
+            from pandas.core.index import Index
+            from pandas.tseries.tdi import TimedeltaIndex
+            from pandas.tseries.offsets import DateOffset
+            if isinstance(other, TimedeltaIndex):
+                return self._add_delta(-other)
+            elif isinstance(self, TimedeltaIndex) and isinstance(other, Index):
+                if not isinstance(other, TimedeltaIndex):
+                    raise TypeError("cannot subtract TimedeltaIndex and {typ}".format(typ=type(other)))
+                return self._add_delta(-other)
+            elif isinstance(other, Index):
+                return self.difference(other)
+            elif isinstance(other, (DateOffset, timedelta, np.timedelta64, tslib.Timedelta)):
+                return self._add_delta(-other)
+            elif com.is_integer(other):
+                return self.shift(-other)
+            elif isinstance(other, (tslib.Timestamp, datetime)):
+                return self._sub_datelike(other)
+            else:  # pragma: no cover
+                return NotImplemented
+        cls.__sub__ = __sub__
+
+        cls.__iadd__ = __add__
+        cls.__isub__ = __sub__
 
     def _add_delta(self, other):
         return NotImplemented
@@ -469,5 +475,3 @@ def repeat(self, repeats, axis=None):
         """
         return self._simple_new(self.values.repeat(repeats),
                                 name=self.name)
-
-

pandas/tseries/index.py

@@ -1665,6 +1665,7 @@ def to_julian_date(self):
                              self.nanosecond/3600.0/1e+9
                             )/24.0)
 DatetimeIndex._add_numeric_methods_disabled()
+DatetimeIndex._add_datetimelike_methods()
 
 def _generate_regular_range(start, end, periods, offset):
     if isinstance(offset, Tick):

pandas/tseries/period.py

@@ -1263,6 +1263,7 @@ def tz_localize(self, tz, infer_dst=False):
         raise NotImplementedError("Not yet implemented for PeriodIndex")
 
 PeriodIndex._add_numeric_methods_disabled()
+PeriodIndex._add_datetimelike_methods()
 
 def _get_ordinal_range(start, end, periods, freq):
     if com._count_not_none(start, end, periods) < 2:

pandas/tseries/tdi.py

@@ -898,6 +898,7 @@ def delete(self, loc):
         return TimedeltaIndex(new_tds, name=self.name, freq=freq)
 
 TimedeltaIndex._add_numeric_methods()
+TimedeltaIndex._add_datetimelike_methods()
 
 def _is_convertible_to_index(other):
     """ return a boolean whether I can attempt conversion to a TimedeltaIndex """
@@ -978,5 +979,3 @@ def timedelta_range(start=None, end=None, periods=None, freq='D',
     return TimedeltaIndex(start=start, end=end, periods=periods,
                           freq=freq, name=name,
                           closed=closed)
-
-