BUG: Index._searchsorted_monotonic(..., side='right') returns the left side position for monotonic decreasing indexes (#17272)

Author: jschendel

doc/source/whatsnew/v0.21.0.txt

@@ -400,6 +400,7 @@ Indexing
 - Bug in ``.isin()`` in which checking membership in empty ``Series`` objects raised an error (:issue:`16991`)
 - Bug in ``CategoricalIndex`` reindexing in which specified indices containing duplicates were not being respected (:issue:`17323`)
 - Bug in intersection of ``RangeIndex`` with negative step (:issue:`17296`)
+- Bug in ``IntervalIndex`` where performing a scalar lookup fails for included right endpoints of non-overlapping monotonic decreasing indexes (:issue:`16417`, :issue:`17271`)
 
 I/O
 ^^^

pandas/core/indexes/base.py

@@ -3465,7 +3465,7 @@ def _searchsorted_monotonic(self, label, side='left'):
             # everything for it to work (element ordering, search side and
             # resulting value).
             pos = self[::-1].searchsorted(label, side='right' if side == 'left'
-                                          else 'right')
+                                          else 'left')
             return len(self) - pos
 
         raise ValueError('index must be monotonic increasing or decreasing')

pandas/tests/indexes/common.py

@@ -11,6 +11,7 @@
                     RangeIndex, MultiIndex, CategoricalIndex, DatetimeIndex,
                     TimedeltaIndex, PeriodIndex, IntervalIndex,
                     notna, isna)
+from pandas.core.indexes.base import InvalidIndexError
 from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin
 from pandas.core.dtypes.common import needs_i8_conversion
 from pandas._libs.tslib import iNaT
@@ -138,9 +139,14 @@ def test_get_indexer_consistency(self):
             if isinstance(index, IntervalIndex):
                 continue
 
-            indexer = index.get_indexer(index[0:2])
-            assert isinstance(indexer, np.ndarray)
-            assert indexer.dtype == np.intp
+            if index.is_unique or isinstance(index, CategoricalIndex):
+                indexer = index.get_indexer(index[0:2])
+                assert isinstance(indexer, np.ndarray)
+                assert indexer.dtype == np.intp
+            else:
+                e = "Reindexing only valid with uniquely valued Index objects"
+                with tm.assert_raises_regex(InvalidIndexError, e):
+                    indexer = index.get_indexer(index[0:2])
 
             indexer, _ = index.get_indexer_non_unique(index[0:2])
             assert isinstance(indexer, np.ndarray)
@@ -632,7 +638,8 @@ def test_difference_base(self):
                     pass
                 elif isinstance(idx, (DatetimeIndex, TimedeltaIndex)):
                     assert result.__class__ == answer.__class__
-                    tm.assert_numpy_array_equal(result.asi8, answer.asi8)
+                    tm.assert_numpy_array_equal(result.sort_values().asi8,
+                                                answer.sort_values().asi8)
                 else:
                     result = first.difference(case)
                     assert tm.equalContents(result, answer)
@@ -954,3 +961,47 @@ def test_join_self_unique(self, how):
         if index.is_unique:
             joined = index.join(index, how=how)
             assert (index == joined).all()
+
+    def test_searchsorted_monotonic(self):
+        # GH17271
+        for index in self.indices.values():
+            # not implemented for tuple searches in MultiIndex
+            # or Intervals searches in IntervalIndex
+            if isinstance(index, (MultiIndex, IntervalIndex)):
+                continue
+
+            # nothing to test if the index is empty
+            if index.empty:
+                continue
+            value = index[0]
+
+            # determine the expected results (handle dupes for 'right')
+            expected_left, expected_right = 0, (index == value).argmin()
+            if expected_right == 0:
+                # all values are the same, expected_right should be length
+                expected_right = len(index)
+
+            # test _searchsorted_monotonic in all cases
+            # test searchsorted only for increasing
+            if index.is_monotonic_increasing:
+                ssm_left = index._searchsorted_monotonic(value, side='left')
+                assert expected_left == ssm_left
+
+                ssm_right = index._searchsorted_monotonic(value, side='right')
+                assert expected_right == ssm_right
+
+                ss_left = index.searchsorted(value, side='left')
+                assert expected_left == ss_left
+
+                ss_right = index.searchsorted(value, side='right')
+                assert expected_right == ss_right
+            elif index.is_monotonic_decreasing:
+                ssm_left = index._searchsorted_monotonic(value, side='left')
+                assert expected_left == ssm_left
+
+                ssm_right = index._searchsorted_monotonic(value, side='right')
+                assert expected_right == ssm_right
+            else:
+                # non-monotonic should raise.
+                with pytest.raises(ValueError):
+                    index._searchsorted_monotonic(value, side='left')

pandas/tests/indexes/datetimes/test_datetimelike.py

@@ -12,7 +12,9 @@ class TestDatetimeIndex(DatetimeLike):
     _holder = DatetimeIndex
 
     def setup_method(self, method):
-        self.indices = dict(index=tm.makeDateIndex(10))
+        self.indices = dict(index=tm.makeDateIndex(10),
+                            index_dec=date_range('20130110', periods=10,
+                                                 freq='-1D'))
         self.setup_indices()
 
     def create_index(self):

pandas/tests/indexes/period/test_period.py

@@ -18,7 +18,9 @@ class TestPeriodIndex(DatetimeLike):
     _multiprocess_can_split_ = True
 
     def setup_method(self, method):
-        self.indices = dict(index=tm.makePeriodIndex(10))
+        self.indices = dict(index=tm.makePeriodIndex(10),
+                            index_dec=period_range('20130101', periods=10,
+                                                   freq='D')[::-1])
         self.setup_indices()
 
     def create_index(self):

pandas/tests/indexes/test_base.py

@@ -46,7 +46,8 @@ def setup_method(self, method):
                             catIndex=tm.makeCategoricalIndex(100),
                             empty=Index([]),
                             tuples=MultiIndex.from_tuples(lzip(
-                                ['foo', 'bar', 'baz'], [1, 2, 3])))
+                                ['foo', 'bar', 'baz'], [1, 2, 3])),
+                            repeats=Index([0, 0, 1, 1, 2, 2]))
         self.setup_indices()
 
     def create_index(self):

pandas/tests/indexes/test_numeric.py

@@ -181,7 +181,9 @@ class TestFloat64Index(Numeric):
 
     def setup_method(self, method):
         self.indices = dict(mixed=Float64Index([1.5, 2, 3, 4, 5]),
-                            float=Float64Index(np.arange(5) * 2.5))
+                            float=Float64Index(np.arange(5) * 2.5),
+                            mixed_dec=Float64Index([5, 4, 3, 2, 1.5]),
+                            float_dec=Float64Index(np.arange(4, -1, -1) * 2.5))
         self.setup_indices()
 
     def create_index(self):
@@ -654,7 +656,8 @@ class TestInt64Index(NumericInt):
     _holder = Int64Index
 
     def setup_method(self, method):
-        self.indices = dict(index=Int64Index(np.arange(0, 20, 2)))
+        self.indices = dict(index=Int64Index(np.arange(0, 20, 2)),
+                            index_dec=Int64Index(np.arange(19, -1, -1)))
         self.setup_indices()
 
     def create_index(self):
@@ -949,8 +952,9 @@ class TestUInt64Index(NumericInt):
     _holder = UInt64Index
 
     def setup_method(self, method):
-        self.indices = dict(index=UInt64Index([2**63, 2**63 + 10, 2**63 + 15,
-                                               2**63 + 20, 2**63 + 25]))
+        vals = [2**63, 2**63 + 10, 2**63 + 15, 2**63 + 20, 2**63 + 25]
+        self.indices = dict(index=UInt64Index(vals),
+                            index_dec=UInt64Index(reversed(vals)))
         self.setup_indices()
 
     def create_index(self):

pandas/tests/indexes/test_range.py

@@ -25,7 +25,8 @@ class TestRangeIndex(Numeric):
     _compat_props = ['shape', 'ndim', 'size', 'itemsize']
 
     def setup_method(self, method):
-        self.indices = dict(index=RangeIndex(0, 20, 2, name='foo'))
+        self.indices = dict(index=RangeIndex(0, 20, 2, name='foo'),
+                            index_dec=RangeIndex(18, -1, -2, name='bar'))
         self.setup_indices()
 
     def create_index(self):

pandas/tests/indexing/test_interval.py

@@ -3,6 +3,7 @@
 import pandas as pd
 
 from pandas import Series, DataFrame, IntervalIndex, Interval
+from pandas.compat import product
 import pandas.util.testing as tm
 
 
@@ -14,16 +15,6 @@ def setup_method(self, method):
     def test_loc_with_scalar(self):
 
         s = self.s
-        expected = 0
-
-        result = s.loc[0.5]
-        assert result == expected
-
-        result = s.loc[1]
-        assert result == expected
-
-        with pytest.raises(KeyError):
-            s.loc[0]
 
         expected = s.iloc[:3]
         tm.assert_series_equal(expected, s.loc[:3])
@@ -42,16 +33,6 @@ def test_loc_with_scalar(self):
     def test_getitem_with_scalar(self):
 
         s = self.s
-        expected = 0
-
-        result = s[0.5]
-        assert result == expected
-
-        result = s[1]
-        assert result == expected
-
-        with pytest.raises(KeyError):
-            s[0]
 
         expected = s.iloc[:3]
         tm.assert_series_equal(expected, s[:3])
@@ -67,6 +48,41 @@ def test_getitem_with_scalar(self):
         expected = s.iloc[2:5]
         tm.assert_series_equal(expected, s[s >= 2])
 
+    @pytest.mark.parametrize('direction, closed',
+                             product(('increasing', 'decreasing'),
+                                     ('left', 'right', 'neither', 'both')))
+    def test_nonoverlapping_monotonic(self, direction, closed):
+        tpls = [(0, 1), (2, 3), (4, 5)]
+        if direction == 'decreasing':
+            tpls = reversed(tpls)
+
+        idx = IntervalIndex.from_tuples(tpls, closed=closed)
+        s = Series(list('abc'), idx)
+
+        for key, expected in zip(idx.left, s):
+            if idx.closed_left:
+                assert s[key] == expected
+                assert s.loc[key] == expected
+            else:
+                with pytest.raises(KeyError):
+                    s[key]
+                with pytest.raises(KeyError):
+                    s.loc[key]
+
+        for key, expected in zip(idx.right, s):
+            if idx.closed_right:
+                assert s[key] == expected
+                assert s.loc[key] == expected
+            else:
+                with pytest.raises(KeyError):
+                    s[key]
+                with pytest.raises(KeyError):
+                    s.loc[key]
+
+        for key, expected in zip(idx.mid, s):
+            assert s[key] == expected
+            assert s.loc[key] == expected
+
     def test_with_interval(self):
 
         s = self.s