Merge branch 'pandas-dev:master' into groupby-mean-datetimelike

Author: AlexeyGy

doc/source/whatsnew/v1.4.0.rst

@@ -395,6 +395,7 @@ I/O
 - Column headers are dropped when constructing a :class:`DataFrame` from a sqlalchemy's ``Row`` object (:issue:`40682`)
 - Bug in unpickling a :class:`Index` with object dtype incorrectly inferring numeric dtypes (:issue:`43188`)
 - Bug in :func:`read_csv` where reading multi-header input with unequal lengths incorrectly raising uncontrolled ``IndexError`` (:issue:`43102`)
+- Bug in :func:`read_csv`, changed exception class when expecting a file path name or file-like object from ``OSError`` to ``TypeError`` (:issue:`43366`)
 
 Period
 ^^^^^^
@@ -420,6 +421,7 @@ Groupby/resample/rolling
 - Bug in :meth:`Series.rolling` when the :class:`Series` ``dtype`` was ``Int64`` (:issue:`43016`)
 - Bug in :meth:`DataFrame.rolling.corr` when the :class:`DataFrame` columns was a :class:`MultiIndex` (:issue:`21157`)
 - Bug in :meth:`DataFrame.groupby.rolling` when specifying ``on`` and calling ``__getitem__`` would subsequently return incorrect results (:issue:`43355`)
+- Bug in :meth:`GroupBy.apply` with time-based :class:`Grouper` objects incorrectly raising ``ValueError`` in corner cases where the grouping vector contains a ``NaT`` (:issue:`43500`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/groupby.pyi

@@ -32,7 +32,7 @@ def group_shift_indexer(
     periods: int,
 ) -> None: ...
 def group_fillna_indexer(
-    out: np.ndarray,  # ndarray[int64_t]
+    out: np.ndarray,  # ndarray[intp_t]
     labels: np.ndarray,  # ndarray[int64_t]
     mask: np.ndarray,  # ndarray[uint8_t]
     direction: Literal["ffill", "bfill"],

pandas/_libs/groupby.pyx

@@ -321,15 +321,15 @@ def group_shift_indexer(int64_t[::1] out, const intp_t[::1] labels,
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def group_fillna_indexer(ndarray[int64_t] out, ndarray[intp_t] labels,
+def group_fillna_indexer(ndarray[intp_t] out, ndarray[intp_t] labels,
                          ndarray[uint8_t] mask, str direction,
                          int64_t limit, bint dropna) -> None:
     """
     Indexes how to fill values forwards or backwards within a group.
 
     Parameters
     ----------
-    out : np.ndarray[np.int64]
+    out : np.ndarray[np.intp]
         Values into which this method will write its results.
     labels : np.ndarray[np.intp]
         Array containing unique label for each group, with its ordering

pandas/_libs/parsers.pyx

@@ -606,10 +606,6 @@ cdef class TextReader:
         cdef:
             void *ptr
 
-        if not hasattr(source, "read"):
-            raise IOError(f'Expected file path name or file-like object, '
-                          f'got {type(source)} type')
-
         ptr = new_rd_source(source)
         self.parser.source = ptr
         self.parser.cb_io = &buffer_rd_bytes

pandas/_libs/reduction.pyx

@@ -34,166 +34,6 @@ cpdef check_result_array(object obj, object dtype):
             raise ValueError("Must produce aggregated value")
 
 
-cdef class _BaseGrouper:
-    cdef _check_dummy(self, object dummy):
-        # both values and index must be an ndarray!
-
-        values = dummy.values
-        # GH 23683: datetimetz types are equivalent to datetime types here
-        if (dummy.dtype != self.arr.dtype
-                and values.dtype != self.arr.dtype):
-            raise ValueError('Dummy array must be same dtype')
-        if is_array(values) and not values.flags.contiguous:
-            # e.g. Categorical has no `flags` attribute
-            values = values.copy()
-        index = dummy.index.values
-        if not index.flags.contiguous:
-            index = index.copy()
-
-        return values, index
-
-    cdef _init_dummy_series_and_index(self, Slider islider, Slider vslider):
-        """
-        Create Series and Index objects that we will alter in-place while iterating.
-        """
-        cached_index = self.ityp(islider.buf, dtype=self.idtype)
-        cached_series = self.typ(
-            vslider.buf, dtype=vslider.buf.dtype, index=cached_index, name=self.name
-        )
-        return cached_index, cached_series
-
-    cdef inline _update_cached_objs(self, object cached_series, object cached_index,
-                                    Slider islider, Slider vslider):
-        cached_index._engine.clear_mapping()
-        cached_index._cache.clear()  # e.g. inferred_freq must go
-        cached_series._mgr.set_values(vslider.buf)
-
-    cdef inline object _apply_to_group(self,
-                                       object cached_series, object cached_index,
-                                       bint initialized):
-        """
-        Call self.f on our new group, then update to the next group.
-        """
-        cdef:
-            object res
-
-        # NB: we assume that _update_cached_objs has already cleared cleared
-        #  the cache and engine mapping
-        res = self.f(cached_series)
-        res = extract_result(res)
-        if not initialized:
-            # On the first pass, we check the output shape to see
-            #  if this looks like a reduction.
-            initialized = True
-            check_result_array(res, cached_series.dtype)
-
-        return res, initialized
-
-
-cdef class SeriesGrouper(_BaseGrouper):
-    """
-    Performs generic grouping operation while avoiding ndarray construction
-    overhead
-    """
-    cdef:
-        Py_ssize_t nresults, ngroups
-
-    cdef public:
-        ndarray arr, index, dummy_arr, dummy_index
-        object f, labels, values, typ, ityp, name, idtype
-
-    def __init__(self, object series, object f, ndarray[intp_t] labels,
-                 Py_ssize_t ngroups):
-
-        if len(series) == 0:
-            # get_result would never assign `result`
-            raise ValueError("SeriesGrouper requires non-empty `series`")
-
-        self.labels = labels
-        self.f = f
-
-        values = series.values
-        if is_array(values) and not values.flags.c_contiguous:
-            # e.g. Categorical has no `flags` attribute
-            values = values.copy('C')
-        self.arr = values
-        self.typ = series._constructor
-        self.ityp = series.index._constructor
-        self.idtype = series.index.dtype
-        self.index = series.index.values
-        self.name = series.name
-
-        dummy = series.iloc[:0]
-        self.dummy_arr, self.dummy_index = self._check_dummy(dummy)
-        self.ngroups = ngroups
-
-    def get_result(self):
-        cdef:
-            # Define result to avoid UnboundLocalError
-            ndarray arr, result = None
-            ndarray[intp_t] labels
-            ndarray[int64_t] counts
-            Py_ssize_t i, n, group_size, lab, start, end
-            object res
-            bint initialized = 0
-            Slider vslider, islider
-            object cached_series = None, cached_index = None
-
-        labels = self.labels
-        counts = np.zeros(self.ngroups, dtype=np.int64)
-        group_size = 0
-        n = len(self.arr)
-
-        vslider = Slider(self.arr, self.dummy_arr)
-        islider = Slider(self.index, self.dummy_index)
-
-        result = np.empty(self.ngroups, dtype='O')
-
-        cached_index, cached_series = self._init_dummy_series_and_index(
-            islider, vslider
-        )
-
-        start = 0
-        try:
-            for i in range(n):
-                group_size += 1
-
-                lab = labels[i]
-
-                if i == n - 1 or lab != labels[i + 1]:
-                    if lab == -1:
-                        start += group_size
-                        group_size = 0
-                        continue
-
-                    end = start + group_size
-                    islider.move(start, end)
-                    vslider.move(start, end)
-
-                    self._update_cached_objs(
-                        cached_series, cached_index, islider, vslider)
-
-                    res, initialized = self._apply_to_group(cached_series, cached_index,
-                                                            initialized)
-
-                    start += group_size
-
-                    result[lab] = res
-                    counts[lab] = group_size
-                    group_size = 0
-
-        finally:
-            # so we don't free the wrong memory
-            islider.reset()
-            vslider.reset()
-
-        # We check for empty series in the constructor, so should always
-        #  have result initialized by this point.
-        assert initialized, "`result` has not been initialized."
-
-        return result, counts
-
-
 cpdef inline extract_result(object res):
     """ extract the result object, it might be a 0-dim ndarray
         or a len-1 0-dim, or a scalar """
@@ -208,40 +48,3 @@ cpdef inline extract_result(object res):
             # see test_resampler_grouper.py::test_apply
             res = res[0]
     return res
-
-
-cdef class Slider:
-    """
-    Only handles contiguous data for now
-    """
-    cdef:
-        ndarray values, buf
-        Py_ssize_t stride
-        char *orig_data
-
-    def __init__(self, ndarray values, ndarray buf):
-        assert values.ndim == 1
-        assert values.dtype == buf.dtype
-
-        if not values.flags.contiguous:
-            values = values.copy()
-
-        self.values = values
-        self.buf = buf
-
-        self.stride = values.strides[0]
-        self.orig_data = self.buf.data
-
-        self.buf.data = self.values.data
-        self.buf.strides[0] = self.stride
-
-    cdef move(self, int start, int end):
-        """
-        For slicing
-        """
-        self.buf.data = self.values.data + self.stride * start
-        self.buf.shape[0] = end - start
-
-    cdef reset(self):
-        self.buf.data = self.orig_data
-        self.buf.shape[0] = 0

pandas/_testing/_io.py

@@ -70,7 +70,7 @@ def _get_default_network_errors():
     # Lazy import for http.client because it imports many things from the stdlib
     import http.client
 
-    return (IOError, http.client.HTTPException, TimeoutError)
+    return (OSError, http.client.HTTPException, TimeoutError)
 
 
 def optional_args(decorator):
@@ -135,7 +135,7 @@ def network(
         If True, checks connectivity before running the test case.
     error_classes : tuple or Exception
         error classes to ignore. If not in ``error_classes``, raises the error.
-        defaults to IOError. Be careful about changing the error classes here.
+        defaults to OSError. Be careful about changing the error classes here.
     skip_errnos : iterable of int
         Any exception that has .errno or .reason.erno set to one
         of these values will be skipped with an appropriate
@@ -165,19 +165,20 @@ def network(
       ... def test_network():
       ...     with pd.io.common.urlopen("rabbit://bonanza.com"):
       ...         pass
+      >>> test_network()
       Traceback
          ...
-      URLError: <urlopen error unknown url type: rabit>
+      URLError: <urlopen error unknown url type: rabbit>
 
       You can specify alternative URLs::
 
         >>> @ts.network("https://www.yahoo.com")
         ... def test_something_with_yahoo():
-        ...    raise IOError("Failure Message")
+        ...    raise OSError("Failure Message")
         >>> test_something_with_yahoo()
         Traceback (most recent call last):
             ...
-        IOError: Failure Message
+        OSError: Failure Message
 
     If you set check_before_test, it will check the url first and not run the
     test on failure::
@@ -241,7 +242,7 @@ def wrapper(*args, **kwargs):
 
 def can_connect(url, error_classes=None):
     """
-    Try to connect to the given url. True if succeeds, False if IOError
+    Try to connect to the given url. True if succeeds, False if OSError
     raised
 
     Parameters
@@ -252,7 +253,7 @@ def can_connect(url, error_classes=None):
     Returns
     -------
     connectable : bool
-        Return True if no IOError (unable to connect) or URLError (bad url) was
+        Return True if no OSError (unable to connect) or URLError (bad url) was
         raised
     """
     if error_classes is None:

pandas/core/groupby/generic.py

@@ -228,9 +228,10 @@ def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs)
         if maybe_use_numba(engine):
             with group_selection_context(self):
                 data = self._selected_obj
-            result, index = self._aggregate_with_numba(
+            result = self._aggregate_with_numba(
                 data.to_frame(), func, *args, engine_kwargs=engine_kwargs, **kwargs
             )
+            index = self._group_keys_index
             return self.obj._constructor(result.ravel(), index=index, name=data.name)
 
         relabeling = func is None
@@ -400,7 +401,7 @@ def _wrap_applied_output(
 
         if isinstance(values[0], dict):
             # GH #823 #24880
-            index = self._group_keys_index
+            index = self.grouper.result_index
             res_df = self.obj._constructor_expanddim(values, index=index)
             res_df = self._reindex_output(res_df)
             # if self.observed is False,
@@ -413,7 +414,7 @@ def _wrap_applied_output(
         else:
             # GH #6265 #24880
             result = self.obj._constructor(
-                data=values, index=self._group_keys_index, name=self.obj.name
+                data=values, index=self.grouper.result_index, name=self.obj.name
             )
             return self._reindex_output(result)
 
@@ -924,9 +925,10 @@ def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs)
         if maybe_use_numba(engine):
             with group_selection_context(self):
                 data = self._selected_obj
-            result, index = self._aggregate_with_numba(
+            result = self._aggregate_with_numba(
                 data, func, *args, engine_kwargs=engine_kwargs, **kwargs
             )
+            index = self._group_keys_index
             return self.obj._constructor(result, index=index, columns=data.columns)
 
         relabeling, func, columns, order = reconstruct_func(func, **kwargs)