BUG: Index.union() inconsistent with non-unique Indexes

doc/source/whatsnew/v1.2.0.rst

@@ -272,6 +272,7 @@ Indexing
 
 - Bug in :meth:`PeriodIndex.get_loc` incorrectly raising ``ValueError`` on non-datelike strings instead of ``KeyError``, causing similar errors in :meth:`Series.__geitem__`, :meth:`Series.__contains__`, and :meth:`Series.loc.__getitem__` (:issue:`34240`)
 - Bug in :meth:`Index.sort_values` where, when empty values were passed, the method would break by trying to compare missing values instead of pushing them to the end of the sort order. (:issue:`35584`)
+- Bug in :meth:`Index.union` dropped duplicate Index values when Index was not monotonic or ``sort`` was set to ``False`` (:issue:`36289`, :issue:`31326`)
 -
 
 Missing

pandas/core/indexes/base.py

@@ -2622,41 +2622,42 @@ def _union(self, other, sort):
         lvals = self._values
         rvals = other._values
 
-        if sort is None and self.is_monotonic and other.is_monotonic:
-            try:
+        # if sort is None and self.is_monotonic and other.is_monotonic:
+        try:
+            if sort is None and self.is_monotonic and other.is_monotonic:
                 result = self._outer_indexer(lvals, rvals)[0]
-            except TypeError:
-                # incomparable objects
-                result = list(lvals)
-
-                # worth making this faster? a very unusual case
-                value_set = set(lvals)
-                result.extend([x for x in rvals if x not in value_set])
-                result = Index(result)._values  # do type inference here
-        else:
-            # find indexes of things in "other" that are not in "self"
-            if self.is_unique:
-                indexer = self.get_indexer(other)
-                indexer = (indexer == -1).nonzero()[0]
             else:
-                indexer = algos.unique1d(self.get_indexer_non_unique(other)[1])
-
-            if len(indexer) > 0:
-                other_diff = algos.take_nd(rvals, indexer, allow_fill=False)
-                result = concat_compat((lvals, other_diff))
-
-            else:
-                result = lvals
-
-            if sort is None:
-                try:
-                    result = algos.safe_sort(result)
-                except TypeError as err:
-                    warnings.warn(
-                        f"{err}, sort order is undefined for incomparable objects",
-                        RuntimeWarning,
-                        stacklevel=3,
+                # We calculate the sorted union and resort it afterwards
+                new_index_sorted = self._outer_indexer(np.sort(lvals), np.sort(rvals))
+                l_reindexer = self.unique().reindex(new_index_sorted[0])[1]
+                if l_reindexer is None:
+                    result = lvals
+                else:
+                    l_result = self.unique().take(
+                        np.sort(l_reindexer[l_reindexer != -1])
+                    )
+                    r_reindexer = other.unique().reindex(new_index_sorted[0])[1]
+                    r_result = other.unique().take(
+                        np.sort(r_reindexer[new_index_sorted[1] == -1])
                     )
+                    result = np.append(l_result, r_result)
+        except TypeError:
+            # incomparable objects
+            result = list(lvals)
+
+            # worth making this faster? a very unusual case
+            value_set = set(lvals)
+            result.extend([x for x in rvals if x not in value_set])
+            result = Index(result)._values  # do type inference here
+        if sort is None and (not self.is_monotonic or not other.is_monotonic):
+            try:
+                result = algos.safe_sort(result)
+            except TypeError as err:
+                warnings.warn(
+                    f"{err}, sort order is undefined for incomparable objects",
+                    RuntimeWarning,
+                    stacklevel=3,
+                )
 
         # for subclasses
         return self._wrap_setop_result(other, result)

pandas/tests/indexes/test_setops.py

@@ -8,7 +8,15 @@
 from pandas.core.dtypes.common import is_dtype_equal
 
 import pandas as pd
-from pandas import Float64Index, Int64Index, RangeIndex, UInt64Index
+from pandas import (
+    CategoricalIndex,
+    DatetimeIndex,
+    Float64Index,
+    Int64Index,
+    RangeIndex,
+    TimedeltaIndex,
+    UInt64Index,
+)
 import pandas._testing as tm
 from pandas.api.types import pandas_dtype
 
@@ -95,3 +103,38 @@ def test_union_dtypes(left, right, expected):
     b = pd.Index([], dtype=right)
     result = (a | b).dtype
     assert result == expected
+
+
+def test_union_duplicate_index_subsets_of_each_other():
+    # GH: 31326
+    a = pd.Index([1, 2, 2, 3])
+    b = pd.Index([3, 3, 4])
+    expected = pd.Index([1, 2, 2, 3, 3, 4])
+
+    result = a.union(b)
+    tm.assert_index_equal(result, expected)
+    result = a.union(b, sort=False)
+    tm.assert_index_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+    "func",
+    [
+        (Int64Index),
+        (Float64Index),
+        (DatetimeIndex),
+        (CategoricalIndex),
+        (TimedeltaIndex),
+    ],
+)
+def test_union_with_duplicate_index(func):
+    # GH: 36289
+    idx1 = func([1, 0, 0])
+    idx2 = func([0, 1])
+    expected = func([0, 0, 1])
+
+    result = idx1.union(idx2)
+    tm.assert_index_equal(result, expected)
+
+    result = idx2.union(idx1)
+    tm.assert_index_equal(result, expected)