Merge remote-tracking branch 'upstream/main' into td64-tests

Author: patrickmckenna

asv_bench/benchmarks/tslibs/fields.py

@@ -66,9 +66,9 @@ class TimeGetStartEndField:
 
     def setup(self, size, side, period, freqstr, month_kw):
         arr = np.random.randint(0, 10, size=size, dtype="i8")
-        self.i8data = arr
+        self.dt64data = arr.view("M8[ns]")
 
         self.attrname = f"is_{period}_{side}"
 
     def time_get_start_end_field(self, size, side, period, freqstr, month_kw):
-        get_start_end_field(self.i8data, self.attrname, freqstr, month_kw=month_kw)
+        get_start_end_field(self.dt64data, self.attrname, freqstr, month_kw=month_kw)

doc/source/whatsnew/v1.5.0.rst

@@ -527,6 +527,7 @@ Other Deprecations
 - Deprecated passing arguments as positional in :meth:`DataFrame.any` and :meth:`Series.any` (:issue:`44802`)
 - Deprecated the ``closed`` argument in :meth:`interval_range` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the methods :meth:`DataFrame.mad`, :meth:`Series.mad`, and the corresponding groupby methods (:issue:`11787`)
+- Deprecated positional arguments to :meth:`Index.join` except for ``other``, use keyword-only arguments instead of positional arguments (:issue:`46518`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.performance:
@@ -708,6 +709,7 @@ Groupby/resample/rolling
 - Bug in :meth:`Rolling.var` and :meth:`Rolling.std` would give non-zero result with window of same values (:issue:`42064`)
 - Bug in :meth:`.Rolling.var` would segfault calculating weighted variance when window size was larger than data size (:issue:`46760`)
 - Bug in :meth:`Grouper.__repr__` where ``dropna`` was not included. Now it is (:issue:`46754`)
+- Bug in :meth:`DataFrame.rolling` gives ValueError when center=True, axis=1 and win_type is specified (:issue:`46135`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/groupby.pyx

@@ -1325,8 +1325,8 @@ def group_rank(
             mask=sub_mask,
         )
         for i in range(len(result)):
-            # TODO: why can't we do out[:, k] = result?
-            out[i, k] = result[i]
+            if labels[i] >= 0:
+                out[i, k] = result[i]
 
 
 # ----------------------------------------------------------------------

pandas/_libs/join.pyx

@@ -839,11 +839,16 @@ def asof_join_nearest_on_X_by_Y(numeric_t[:] left_values,
                                 by_t[:] left_by_values,
                                 by_t[:] right_by_values,
                                 bint allow_exact_matches=True,
-                                tolerance=None):
+                                tolerance=None,
+                                bint use_hashtable=True):
 
     cdef:
         ndarray[intp_t] bli, bri, fli, fri
 
+        ndarray[intp_t] left_indexer, right_indexer
+        Py_ssize_t left_size, i
+        numeric_t bdiff, fdiff
+
     # search both forward and backward
     bli, bri = asof_join_backward_on_X_by_Y(
         left_values,
@@ -852,6 +857,7 @@ def asof_join_nearest_on_X_by_Y(numeric_t[:] left_values,
         right_by_values,
         allow_exact_matches,
         tolerance,
+        use_hashtable
     )
     fli, fri = asof_join_forward_on_X_by_Y(
         left_values,
@@ -860,26 +866,11 @@ def asof_join_nearest_on_X_by_Y(numeric_t[:] left_values,
         right_by_values,
         allow_exact_matches,
         tolerance,
+        use_hashtable
     )
 
-    return _choose_smaller_timestamp(left_values, right_values, bli, bri, fli, fri)
-
-
-cdef _choose_smaller_timestamp(
-    numeric_t[:] left_values,
-    numeric_t[:] right_values,
-    ndarray[intp_t] bli,
-    ndarray[intp_t] bri,
-    ndarray[intp_t] fli,
-    ndarray[intp_t] fri,
-):
-    cdef:
-        ndarray[intp_t] left_indexer, right_indexer
-        Py_ssize_t left_size, i
-        numeric_t bdiff, fdiff
-
+    # choose the smaller timestamp
     left_size = len(left_values)
-
     left_indexer = np.empty(left_size, dtype=np.intp)
     right_indexer = np.empty(left_size, dtype=np.intp)
 
@@ -894,55 +885,3 @@ cdef _choose_smaller_timestamp(
         left_indexer[i] = bli[i]
 
     return left_indexer, right_indexer
-
-
-# ----------------------------------------------------------------------
-# asof_join
-# ----------------------------------------------------------------------
-
-def asof_join_backward(numeric_t[:] left_values,
-                       numeric_t[:] right_values,
-                       bint allow_exact_matches=True,
-                       tolerance=None):
-
-    return asof_join_backward_on_X_by_Y(
-        left_values,
-        right_values,
-        None,
-        None,
-        allow_exact_matches=allow_exact_matches,
-        tolerance=tolerance,
-        use_hashtable=False,
-    )
-
-
-def asof_join_forward(numeric_t[:] left_values,
-                      numeric_t[:] right_values,
-                      bint allow_exact_matches=True,
-                      tolerance=None):
-    return asof_join_forward_on_X_by_Y(
-        left_values,
-        right_values,
-        None,
-        None,
-        allow_exact_matches=allow_exact_matches,
-        tolerance=tolerance,
-        use_hashtable=False,
-    )
-
-
-def asof_join_nearest(numeric_t[:] left_values,
-                      numeric_t[:] right_values,
-                      bint allow_exact_matches=True,
-                      tolerance=None):
-
-    cdef:
-        ndarray[intp_t] bli, bri, fli, fri
-
-    # search both forward and backward
-    bli, bri = asof_join_backward(left_values, right_values,
-                                  allow_exact_matches, tolerance)
-    fli, fri = asof_join_forward(left_values, right_values,
-                                 allow_exact_matches, tolerance)
-
-    return _choose_smaller_timestamp(left_values, right_values, bli, bri, fli, fri)

pandas/_libs/tslibs/__init__.py

@@ -25,6 +25,7 @@
     "Tick",
     "BaseOffset",
     "tz_compare",
+    "is_unitless",
 ]
 
 from pandas._libs.tslibs import dtypes
@@ -39,6 +40,7 @@
 from pandas._libs.tslibs.np_datetime import (
     OutOfBoundsDatetime,
     OutOfBoundsTimedelta,
+    is_unitless,
 )
 from pandas._libs.tslibs.offsets import (
     BaseOffset,

pandas/_libs/tslibs/fields.pyi

@@ -12,7 +12,7 @@ def get_date_name_field(
     locale: str | None = ...,
 ) -> npt.NDArray[np.object_]: ...
 def get_start_end_field(
-    dtindex: npt.NDArray[np.int64],  # const int64_t[:]
+    dt64values: npt.NDArray[np.datetime64],
     field: str,
     freqstr: str | None = ...,
     month_kw: int = ...,

pandas/_libs/tslibs/fields.pyx

@@ -5,8 +5,10 @@ objects and arrays
 from locale import LC_TIME
 
 from _strptime import LocaleTime
+
 cimport cython
 from cython cimport Py_ssize_t
+
 import numpy as np
 
 cimport numpy as cnp
@@ -41,8 +43,11 @@ from pandas._libs.tslibs.ccalendar cimport (
 )
 from pandas._libs.tslibs.nattype cimport NPY_NAT
 from pandas._libs.tslibs.np_datetime cimport (
+    NPY_DATETIMEUNIT,
     dt64_to_dtstruct,
+    get_unit_from_dtype,
     npy_datetimestruct,
+    pandas_datetime_to_datetimestruct,
     pandas_timedeltastruct,
     td64_to_tdstruct,
 )
@@ -196,22 +201,35 @@ cdef inline bint _is_on_month(int month, int compare_month, int modby) nogil:
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def get_start_end_field(const int64_t[:] dtindex, str field,
+def get_start_end_field(ndarray dt64values, str field,
                         str freqstr=None, int month_kw=12):
     """
     Given an int64-based datetime index return array of indicators
     of whether timestamps are at the start/end of the month/quarter/year
     (defined by frequency).
+
+    Parameters
+    ----------
+    dt64values : ndarray[datetime64], any resolution
+    field : str
+    frestr : str or None, default None
+    month_kw : int, default 12
+
+    Returns
+    -------
+    ndarray[bool]
     """
     cdef:
         Py_ssize_t i
-        int count = len(dtindex)
+        int count = dt64values.size
         bint is_business = 0
         int end_month = 12
         int start_month = 1
         ndarray[int8_t] out
         npy_datetimestruct dts
         int compare_month, modby
+        ndarray dtindex = dt64values.view("i8")
+        NPY_DATETIMEUNIT reso = get_unit_from_dtype(dt64values.dtype)
 
     out = np.zeros(count, dtype='int8')
 
@@ -251,7 +269,7 @@ def get_start_end_field(const int64_t[:] dtindex, str field,
                     out[i] = 0
                     continue
 
-                dt64_to_dtstruct(dtindex[i], &dts)
+                pandas_datetime_to_datetimestruct(dtindex[i], reso, &dts)
 
                 if _is_on_month(dts.month, compare_month, modby) and (
                         dts.day == get_firstbday(dts.year, dts.month)):
@@ -263,7 +281,7 @@ def get_start_end_field(const int64_t[:] dtindex, str field,
                     out[i] = 0
                     continue
 
-                dt64_to_dtstruct(dtindex[i], &dts)
+                pandas_datetime_to_datetimestruct(dtindex[i], reso, &dts)
 
                 if _is_on_month(dts.month, compare_month, modby) and dts.day == 1:
                     out[i] = 1
@@ -275,7 +293,7 @@ def get_start_end_field(const int64_t[:] dtindex, str field,
                     out[i] = 0
                     continue
 
-                dt64_to_dtstruct(dtindex[i], &dts)
+                pandas_datetime_to_datetimestruct(dtindex[i], reso, &dts)
 
                 if _is_on_month(dts.month, compare_month, modby) and (
                         dts.day == get_lastbday(dts.year, dts.month)):
@@ -287,7 +305,7 @@ def get_start_end_field(const int64_t[:] dtindex, str field,
                     out[i] = 0
                     continue
 
-                dt64_to_dtstruct(dtindex[i], &dts)
+                pandas_datetime_to_datetimestruct(dtindex[i], reso, &dts)
 
                 if _is_on_month(dts.month, compare_month, modby) and (
                         dts.day == get_days_in_month(dts.year, dts.month)):

pandas/_libs/tslibs/np_datetime.pyi

@@ -9,3 +9,4 @@ def py_td64_to_tdstruct(td64: int, unit: int) -> dict: ...
 def astype_overflowsafe(
     arr: np.ndarray, dtype: np.dtype, copy: bool = ...
 ) -> np.ndarray: ...
+def is_unitless(dtype: np.dtype) -> bool: ...

pandas/_libs/tslibs/np_datetime.pyx

@@ -92,6 +92,18 @@ def py_get_unit_from_dtype(dtype):
     return get_unit_from_dtype(dtype)
 
 
+def is_unitless(dtype: cnp.dtype) -> bool:
+    """
+    Check if a datetime64 or timedelta64 dtype has no attached unit.
+    """
+    if dtype.type_num not in [cnp.NPY_DATETIME, cnp.NPY_TIMEDELTA]:
+        raise ValueError("is_unitless dtype must be datetime64 or timedelta64")
+    cdef:
+        NPY_DATETIMEUNIT unit = get_unit_from_dtype(dtype)
+
+    return unit == NPY_DATETIMEUNIT.NPY_FR_GENERIC
+
+
 # ----------------------------------------------------------------------
 # Comparison
 

pandas/_libs/tslibs/timedeltas.pyi

@@ -88,6 +88,8 @@ class Timedelta(timedelta):
     # GH 46171
     # While Timedelta can return pd.NaT, having the constructor return
     # a Union with NaTType makes things awkward for users of pandas
+    @classmethod
+    def _from_value_and_reso(cls, value: np.int64, reso: int) -> Timedelta: ...
     @property
     def days(self) -> int: ...
     @property

pandas/_libs/tslibs/timedeltas.pyx

@@ -347,6 +347,7 @@ def array_to_timedelta64(
                 raise ValueError(
                     "unit must not be specified if the input contains a str"
                 )
+            cnp.PyArray_ITER_NEXT(it)
 
     # Usually, we have all strings. If so, we hit the fast path.
     # If this path fails, we try conversion a different way, and

pandas/_libs/tslibs/timestamps.pyx

@@ -489,9 +489,7 @@ cdef class _Timestamp(ABCTimestamp):
             dict kwds
             ndarray[uint8_t, cast=True] out
             int month_kw
-
-        if self._reso != NPY_FR_ns:
-            raise NotImplementedError(self._reso)
+            str unit
 
         if freq:
             kwds = freq.kwds
@@ -502,7 +500,9 @@ cdef class _Timestamp(ABCTimestamp):
             freqstr = None
 
         val = self._maybe_convert_value_to_local()
-        out = get_start_end_field(np.array([val], dtype=np.int64),
+
+        unit = npy_unit_to_abbrev(self._reso)
+        out = get_start_end_field(np.array([val], dtype=f"M8[{unit}]"),
                                   field, freqstr, month_kw)
         return out[0]
 

pandas/_libs/tslibs/vectorized.pyi

@@ -11,24 +11,24 @@ from pandas._libs.tslibs.offsets import BaseOffset
 from pandas._typing import npt
 
 def dt64arr_to_periodarr(
-    stamps: npt.NDArray[np.int64],  # const int64_t[:]
+    stamps: npt.NDArray[np.int64],
     freq: int,
     tz: tzinfo | None,
-) -> npt.NDArray[np.int64]: ...  # np.ndarray[np.int64, ndim=1]
+) -> npt.NDArray[np.int64]: ...
 def is_date_array_normalized(
-    stamps: npt.NDArray[np.int64],  # const int64_t[:]
+    stamps: npt.NDArray[np.int64],
     tz: tzinfo | None = ...,
 ) -> bool: ...
 def normalize_i8_timestamps(
-    stamps: npt.NDArray[np.int64],  # const int64_t[:]
+    stamps: npt.NDArray[np.int64],
     tz: tzinfo | None,
 ) -> npt.NDArray[np.int64]: ...
 def get_resolution(
-    stamps: npt.NDArray[np.int64],  # const int64_t[:]
+    stamps: npt.NDArray[np.int64],
     tz: tzinfo | None = ...,
 ) -> Resolution: ...
 def ints_to_pydatetime(
-    arr: npt.NDArray[np.int64],  # const int64_t[:}]
+    arr: npt.NDArray[np.int64],
     tz: tzinfo | None = ...,
     freq: BaseOffset | None = ...,
     fold: bool = ...,