Merge remote-tracking branch 'upstream/master' into extensiontyping

Author: simonjayhawkins

pandas/_libs/parsers.pyx

@@ -337,7 +337,7 @@ cdef class TextReader:
         object skiprows
         object dtype
         object usecols
-        list dtype_cast_order
+        list dtype_cast_order  # list[np.dtype]
         set unnamed_cols
         set noconvert
 

pandas/_testing/asserters.py

@@ -980,8 +980,8 @@ def assert_series_equal(
         left_values = left._values
         right_values = right._values
         # Only check exact if dtype is numeric
-        if is_extension_array_dtype(left_values) and is_extension_array_dtype(
-            right_values
+        if isinstance(left_values, ExtensionArray) and isinstance(
+            right_values, ExtensionArray
         ):
             assert_extension_array_equal(
                 left_values,

pandas/core/algorithms.py

@@ -237,38 +237,26 @@ def _reconstruct_data(
         # Catch DatetimeArray/TimedeltaArray
         return values
 
-    if is_extension_array_dtype(dtype):
-        # error: Item "dtype[Any]" of "Union[dtype[Any], ExtensionDtype]" has no
-        # attribute "construct_array_type"
-        cls = dtype.construct_array_type()  # type: ignore[union-attr]
+    if not isinstance(dtype, np.dtype):
+        # i.e. ExtensionDtype
+        cls = dtype.construct_array_type()
         if isinstance(values, cls) and values.dtype == dtype:
             return values
 
         values = cls._from_sequence(values)
     elif is_bool_dtype(dtype):
-        # error: Argument 1 to "astype" of "_ArrayOrScalarCommon" has incompatible
-        # type "Union[ExtensionDtype, Union[str, dtype[Any]], Type[str], Type[float],
-        # Type[int], Type[complex], Type[bool], Type[object]]"; expected
-        # "Union[dtype[Any], None, type, _SupportsDType, str, Union[Tuple[Any, int],
-        # Tuple[Any, Union[int, Sequence[int]]], List[Any], _DTypeDict,
-        # Tuple[Any, Any]]]"
-        values = values.astype(dtype, copy=False)  # type: ignore[arg-type]
+        values = values.astype(dtype, copy=False)
 
         # we only support object dtypes bool Index
         if isinstance(original, ABCIndex):
             values = values.astype(object, copy=False)
     elif dtype is not None:
         if is_datetime64_dtype(dtype):
-            dtype = "datetime64[ns]"
+            dtype = np.dtype("datetime64[ns]")
         elif is_timedelta64_dtype(dtype):
-            dtype = "timedelta64[ns]"
-        # error: Argument 1 to "astype" of "_ArrayOrScalarCommon" has incompatible
-        # type "Union[ExtensionDtype, Union[str, dtype[Any]], Type[str], Type[float],
-        # Type[int], Type[complex], Type[bool], Type[object]]"; expected
-        # "Union[dtype[Any], None, type, _SupportsDType, str, Union[Tuple[Any, int],
-        # Tuple[Any, Union[int, Sequence[int]]], List[Any], _DTypeDict,
-        # Tuple[Any, Any]]]"
-        values = values.astype(dtype, copy=False)  # type: ignore[arg-type]
+            dtype = np.dtype("timedelta64[ns]")
+
+        values = values.astype(dtype, copy=False)
 
     return values
 
@@ -773,7 +761,8 @@ def factorize(
             uniques = Index(uniques)
         return codes, uniques
 
-    if is_extension_array_dtype(values.dtype):
+    if not isinstance(values.dtype, np.dtype):
+        # i.e. ExtensionDtype
         codes, uniques = values.factorize(na_sentinel=na_sentinel)
         dtype = original.dtype
     else:
@@ -1663,7 +1652,8 @@ def diff(arr, n: int, axis: int = 0, stacklevel=3):
         arr = arr.to_numpy()
         dtype = arr.dtype
 
-    if is_extension_array_dtype(dtype):
+    if not isinstance(dtype, np.dtype):
+        # i.e ExtensionDtype
         if hasattr(arr, f"__{op.__name__}__"):
             if axis != 0:
                 raise ValueError(f"cannot diff {type(arr).__name__} on axis={axis}")

pandas/core/arrays/categorical.py

@@ -67,7 +67,10 @@
     needs_i8_conversion,
     pandas_dtype,
 )
-from pandas.core.dtypes.dtypes import CategoricalDtype
+from pandas.core.dtypes.dtypes import (
+    CategoricalDtype,
+    ExtensionDtype,
+)
 from pandas.core.dtypes.generic import (
     ABCIndex,
     ABCSeries,
@@ -513,7 +516,7 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
             result = self._set_dtype(dtype)
 
         # TODO: consolidate with ndarray case?
-        elif is_extension_array_dtype(dtype):
+        elif isinstance(dtype, ExtensionDtype):
             result = pd_array(self, dtype=dtype, copy=copy)
 
         elif is_integer_dtype(dtype) and self.isna().any():
@@ -524,28 +527,15 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
             # variable has type "Categorical")
             result = np.array(  # type: ignore[assignment]
                 self,
-                # error: Argument "dtype" to "array" has incompatible type
-                # "Union[ExtensionDtype, str, dtype[Any], Type[str], Type[float],
-                # Type[int], Type[complex], Type[bool], Type[object]]"; expected
-                # "Union[dtype[Any], None, type, _SupportsDType, str, Union[Tuple[Any,
-                # int], Tuple[Any, Union[int, Sequence[int]]], List[Any], _DTypeDict,
-                # Tuple[Any, Any]]]"
-                dtype=dtype,  # type: ignore[arg-type]
+                dtype=dtype,
                 copy=copy,
             )
 
         else:
             # GH8628 (PERF): astype category codes instead of astyping array
             try:
                 new_cats = np.asarray(self.categories)
-                # error: Argument "dtype" to "astype" of "_ArrayOrScalarCommon" has
-                # incompatible type "Union[ExtensionDtype, dtype[Any]]"; expected
-                # "Union[dtype[Any], None, type, _SupportsDType, str, Union[Tuple[Any,
-                # int], Tuple[Any, Union[int, Sequence[int]]], List[Any], _DTypeDict,
-                # Tuple[Any, Any]]]"
-                new_cats = new_cats.astype(
-                    dtype=dtype, copy=copy  # type: ignore[arg-type]
-                )
+                new_cats = new_cats.astype(dtype=dtype, copy=copy)
             except (
                 TypeError,  # downstream error msg for CategoricalIndex is misleading
                 ValueError,

pandas/core/arrays/datetimelike.py

@@ -877,12 +877,11 @@ def _isnan(self) -> np.ndarray:
         return self.asi8 == iNaT
 
     @property  # NB: override with cache_readonly in immutable subclasses
-    def _hasnans(self) -> np.ndarray:
+    def _hasnans(self) -> bool:
         """
         return if I have any nans; enables various perf speedups
         """
-        # error: Incompatible return value type (got "bool", expected "ndarray")
-        return bool(self._isnan.any())  # type: ignore[return-value]
+        return bool(self._isnan.any())
 
     def _maybe_mask_results(
         self, result: np.ndarray, fill_value=iNaT, convert=None

pandas/core/indexes/base.py

@@ -191,8 +191,7 @@
 str_t = str
 
 
-# error: Value of type variable "_DTypeScalar" of "dtype" cannot be "object"
-_o_dtype = np.dtype(object)  # type: ignore[type-var]
+_o_dtype = np.dtype("object")
 
 
 _Identity = NewType("_Identity", object)
@@ -417,11 +416,7 @@ def __new__(
                 # maybe coerce to a sub-class
                 arr = data
             else:
-                # error: Argument "dtype" to "asarray_tuplesafe" has incompatible type
-                # "Type[object]"; expected "Union[str, dtype[Any], None]"
-                arr = com.asarray_tuplesafe(
-                    data, dtype=object  # type: ignore[arg-type]
-                )
+                arr = com.asarray_tuplesafe(data, dtype=np.dtype("object"))
 
                 if dtype is None:
                     arr = _maybe_cast_data_without_dtype(arr)
@@ -456,9 +451,7 @@ def __new__(
                     )
             # other iterable of some kind
 
-            # error: Argument "dtype" to "asarray_tuplesafe" has incompatible type
-            # "Type[object]"; expected "Union[str, dtype[Any], None]"
-            subarr = com.asarray_tuplesafe(data, dtype=object)  # type: ignore[arg-type]
+            subarr = com.asarray_tuplesafe(data, dtype=np.dtype("object"))
             return Index(subarr, dtype=dtype, copy=copy, name=name, **kwargs)
 
     @classmethod
@@ -2902,16 +2895,10 @@ def union(self, other, sort=None):
                 # <T>   | <T>    -> T
                 # <T>   | <U>    -> object
                 if not (is_integer_dtype(self.dtype) and is_integer_dtype(other.dtype)):
-                    # error: Incompatible types in assignment (expression has type
-                    # "str", variable has type "Union[dtype[Any], ExtensionDtype]")
-                    dtype = "float64"  # type: ignore[assignment]
+                    dtype = np.dtype("float64")
                 else:
                     # one is int64 other is uint64
-
-                    # error: Incompatible types in assignment (expression has type
-                    # "Type[object]", variable has type "Union[dtype[Any],
-                    # ExtensionDtype]")
-                    dtype = object  # type: ignore[assignment]
+                    dtype = np.dtype("object")
 
             left = self.astype(dtype, copy=False)
             right = other.astype(dtype, copy=False)
@@ -3906,6 +3893,9 @@ def join(
         self_is_mi = isinstance(self, ABCMultiIndex)
         other_is_mi = isinstance(other, ABCMultiIndex)
 
+        lindexer: Optional[np.ndarray]
+        rindexer: Optional[np.ndarray]
+
         # try to figure out the join level
         # GH3662
         if level is None and (self_is_mi or other_is_mi):
@@ -4003,15 +3993,11 @@ def join(
 
         if return_indexers:
             if join_index is self:
-                # error: Incompatible types in assignment (expression has type "None",
-                # variable has type "ndarray")
-                lindexer = None  # type: ignore[assignment]
+                lindexer = None
             else:
                 lindexer = self.get_indexer(join_index)
             if join_index is other:
-                # error: Incompatible types in assignment (expression has type "None",
-                # variable has type "ndarray")
-                rindexer = None  # type: ignore[assignment]
+                rindexer = None
             else:
                 rindexer = other.get_indexer(join_index)
             return join_index, lindexer, rindexer
@@ -4114,15 +4100,11 @@ def _join_non_unique(self, other, how="left", return_indexers=False):
         left_idx = ensure_platform_int(left_idx)
         right_idx = ensure_platform_int(right_idx)
 
-        join_index = np.asarray(lvalues.take(left_idx))
+        join_array = np.asarray(lvalues.take(left_idx))
         mask = left_idx == -1
-        np.putmask(join_index, mask, rvalues.take(right_idx))
+        np.putmask(join_array, mask, rvalues.take(right_idx))
 
-        # error: Incompatible types in assignment (expression has type "Index", variable
-        # has type "ndarray")
-        join_index = self._wrap_joined_index(
-            join_index, other  # type: ignore[assignment]
-        )
+        join_index = self._wrap_joined_index(join_array, other)
 
         if return_indexers:
             return join_index, left_idx, right_idx
@@ -4286,6 +4268,9 @@ def _join_monotonic(self, other, how="left", return_indexers=False):
         sv = self._get_engine_target()
         ov = other._get_engine_target()
 
+        ridx: Optional[np.ndarray]
+        lidx: Optional[np.ndarray]
+
         if self.is_unique and other.is_unique:
             # We can perform much better than the general case
             if how == "left":
@@ -4295,61 +4280,24 @@ def _join_monotonic(self, other, how="left", return_indexers=False):
             elif how == "right":
                 join_index = other
                 lidx = self._left_indexer_unique(ov, sv)
-                # error: Incompatible types in assignment (expression has type "None",
-                # variable has type "ndarray")
-                ridx = None  # type: ignore[assignment]
+                ridx = None
             elif how == "inner":
-                # error: Incompatible types in assignment (expression has type
-                # "ndarray", variable has type "Index")
-                join_index, lidx, ridx = self._inner_indexer(  # type:ignore[assignment]
-                    sv, ov
-                )
-                # error: Argument 1 to "_wrap_joined_index" of "Index" has incompatible
-                # type "Index"; expected "ndarray"
-                join_index = self._wrap_joined_index(
-                    join_index, other  # type: ignore[arg-type]
-                )
+                join_array, lidx, ridx = self._inner_indexer(sv, ov)
+                join_index = self._wrap_joined_index(join_array, other)
             elif how == "outer":
-                # error: Incompatible types in assignment (expression has type
-                # "ndarray", variable has type "Index")
-                join_index, lidx, ridx = self._outer_indexer(  # type:ignore[assignment]
-                    sv, ov
-                )
-                # error: Argument 1 to "_wrap_joined_index" of "Index" has incompatible
-                # type "Index"; expected "ndarray"
-                join_index = self._wrap_joined_index(
-                    join_index, other  # type: ignore[arg-type]
-                )
+                join_array, lidx, ridx = self._outer_indexer(sv, ov)
+                join_index = self._wrap_joined_index(join_array, other)
         else:
             if how == "left":
-                # error: Incompatible types in assignment (expression has type
-                # "ndarray", variable has type "Index")
-                join_index, lidx, ridx = self._left_indexer(  # type: ignore[assignment]
-                    sv, ov
-                )
+                join_array, lidx, ridx = self._left_indexer(sv, ov)
             elif how == "right":
-                # error: Incompatible types in assignment (expression has type
-                # "ndarray", variable has type "Index")
-                join_index, ridx, lidx = self._left_indexer(  # type: ignore[assignment]
-                    ov, sv
-                )
+                join_array, ridx, lidx = self._left_indexer(ov, sv)
             elif how == "inner":
-                # error: Incompatible types in assignment (expression has type
-                # "ndarray", variable has type "Index")
-                join_index, lidx, ridx = self._inner_indexer(  # type:ignore[assignment]
-                    sv, ov
-                )
+                join_array, lidx, ridx = self._inner_indexer(sv, ov)
             elif how == "outer":
-                # error: Incompatible types in assignment (expression has type
-                # "ndarray", variable has type "Index")
-                join_index, lidx, ridx = self._outer_indexer(  # type:ignore[assignment]
-                    sv, ov
-                )
-            # error: Argument 1 to "_wrap_joined_index" of "Index" has incompatible type
-            # "Index"; expected "ndarray"
-            join_index = self._wrap_joined_index(
-                join_index, other  # type: ignore[arg-type]
-            )
+                join_array, lidx, ridx = self._outer_indexer(sv, ov)
+
+            join_index = self._wrap_joined_index(join_array, other)
 
         if return_indexers:
             lidx = None if lidx is None else ensure_platform_int(lidx)
@@ -6477,12 +6425,8 @@ def _maybe_cast_data_without_dtype(subarr):
                 pass
 
         elif inferred.startswith("timedelta"):
-            # error: Incompatible types in assignment (expression has type
-            # "TimedeltaArray", variable has type "ndarray")
-            data = TimedeltaArray._from_sequence(  # type: ignore[assignment]
-                subarr, copy=False
-            )
-            return data
+            tda = TimedeltaArray._from_sequence(subarr, copy=False)
+            return tda
         elif inferred == "period":
             try:
                 data = PeriodArray._from_sequence(subarr)

pandas/core/internals/blocks.py

@@ -118,9 +118,7 @@
     from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
 
 # comparison is faster than is_object_dtype
-
-# error: Value of type variable "_DTypeScalar" of "dtype" cannot be "object"
-_dtype_obj = np.dtype(object)  # type: ignore[type-var]
+_dtype_obj = np.dtype("object")
 
 
 class Block(PandasObject):
@@ -1598,14 +1596,9 @@ def to_native_types(self, na_rep="nan", quoting=None, **kwargs):
         values = self.values
         mask = isna(values)
 
-        # error: Incompatible types in assignment (expression has type "ndarray",
-        # variable has type "ExtensionArray")
-        values = np.asarray(values.astype(object))  # type: ignore[assignment]
-        values[mask] = na_rep
-
-        # TODO(EA2D): reshape not needed with 2D EAs
-        # we are expected to return a 2-d ndarray
-        return self.make_block(values)
+        new_values = np.asarray(values.astype(object))
+        new_values[mask] = na_rep
+        return self.make_block(new_values)
 
     def take_nd(
         self, indexer, axis: int = 0, new_mgr_locs=None, fill_value=lib.no_default