Merge branch 'master' into file-manager-del

Author: shoyer

doc/whats-new.rst

@@ -33,6 +33,14 @@ v0.11.1 (unreleased)
 Breaking changes
 ~~~~~~~~~~~~~~~~
 
+- Time bounds variables are now also decoded according to CF conventions
+  (:issue:`2565`). The previous behavior was to decode them only if they
+  had specific time attributes, now these attributes are copied 
+  automatically from the corresponding time coordinate. This might 
+  brake downstream code that was relying on these variables to be
+  not decoded.
+  By `Fabien Maussion <https://github.com/fmaussion>`_.
+
 Enhancements
 ~~~~~~~~~~~~
 
@@ -50,6 +58,9 @@ Enhancements
   recommend enabling it in your test suites if you use xarray for IO.
   By `Stephan Hoyer <https://github.com/shoyer>`_
 - Support Dask ``HighLevelGraphs`` by `Matthew Rocklin <https://matthewrocklin.com>`_.
+- :py:meth:`DataArray.resample` and :py:meth:`Dataset.resample` now supports the
+  ``loffset`` kwarg just like Pandas.
+  By `Deepak Cherian <https://github.com/dcherian>`_
 
 Bug fixes
 ~~~~~~~~~

xarray/conventions.py

@@ -320,11 +320,39 @@ def decode_cf_variable(name, var, concat_characters=True, mask_and_scale=True,
     return Variable(dimensions, data, attributes, encoding=encoding)
 
 
+def _update_bounds_attributes(variables):
+    """Adds time attributes to time bounds variables.
+
+    Variables handling time bounds ("Cell boundaries" in the CF
+    conventions) do not necessarily carry the necessary attributes to be
+    decoded. This copies the attributes from the time variable to the
+    associated boundaries.
+
+    See Also:
+
+    http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/
+         cf-conventions.html#cell-boundaries
+
+    https://github.com/pydata/xarray/issues/2565
+    """
+
+    # For all time variables with bounds
+    for v in variables.values():
+        attrs = v.attrs
+        has_date_units = 'units' in attrs and 'since' in attrs['units']
+        if has_date_units and 'bounds' in attrs:
+            if attrs['bounds'] in variables:
+                bounds_attrs = variables[attrs['bounds']].attrs
+                bounds_attrs.setdefault('units', attrs['units'])
+                if 'calendar' in attrs:
+                    bounds_attrs.setdefault('calendar', attrs['calendar'])
+
+
 def decode_cf_variables(variables, attributes, concat_characters=True,
                         mask_and_scale=True, decode_times=True,
                         decode_coords=True, drop_variables=None):
     """
-    Decode a several CF encoded variables.
+    Decode several CF encoded variables.
 
     See: decode_cf_variable
     """
@@ -350,6 +378,10 @@ def stackable(dim):
         drop_variables = []
     drop_variables = set(drop_variables)
 
+    # Time bounds coordinates might miss the decoding attributes
+    if decode_times:
+        _update_bounds_attributes(variables)
+
     new_vars = OrderedDict()
     for k, v in iteritems(variables):
         if k in drop_variables:

xarray/core/common.py

@@ -592,7 +592,7 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs):
                                  center=center)
 
     def resample(self, indexer=None, skipna=None, closed=None, label=None,
-                 base=0, keep_attrs=None, **indexer_kwargs):
+                 base=0, keep_attrs=None, loffset=None, **indexer_kwargs):
         """Returns a Resample object for performing resampling operations.
 
         Handles both downsampling and upsampling. If any intervals contain no
@@ -612,6 +612,9 @@ def resample(self, indexer=None, skipna=None, closed=None, label=None,
             For frequencies that evenly subdivide 1 day, the "origin" of the
             aggregated intervals. For example, for '24H' frequency, base could
             range from 0 through 23.
+        loffset : timedelta or str, optional
+            Offset used to adjust the resampled time labels. Some pandas date
+            offset strings are supported.
         keep_attrs : bool, optional
             If True, the object's attributes (`attrs`) will be copied from
             the original object to the new one.  If False (default), the new
@@ -700,7 +703,9 @@ def resample(self, indexer=None, skipna=None, closed=None, label=None,
 
         group = DataArray(dim_coord, coords=dim_coord.coords,
                           dims=dim_coord.dims, name=RESAMPLE_DIM)
-        grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base)
+        # TODO: to_offset() call required for pandas==0.19.2
+        grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base,
+                             loffset=pd.tseries.frequencies.to_offset(loffset))
         resampler = self._resample_cls(self, group=group, dim=dim_name,
                                        grouper=grouper,
                                        resample_dim=RESAMPLE_DIM)

xarray/core/groupby.py

@@ -3,6 +3,7 @@
 import functools
 import warnings
 
+import datetime
 import numpy as np
 import pandas as pd
 
@@ -154,6 +155,32 @@ def _unique_and_monotonic(group):
         return index.is_unique and index.is_monotonic
 
 
+def _apply_loffset(grouper, result):
+    """
+    (copied from pandas)
+    if loffset is set, offset the result index
+
+    This is NOT an idempotent routine, it will be applied
+    exactly once to the result.
+
+    Parameters
+    ----------
+    result : Series or DataFrame
+        the result of resample
+    """
+
+    needs_offset = (
+        isinstance(grouper.loffset, (pd.DateOffset, datetime.timedelta))
+        and isinstance(result.index, pd.DatetimeIndex)
+        and len(result.index) > 0
+    )
+
+    if needs_offset:
+        result.index = result.index + grouper.loffset
+
+    grouper.loffset = None
+
+
 class GroupBy(SupportsArithmetic):
     """A object that implements the split-apply-combine pattern.
 
@@ -235,6 +262,7 @@ def __init__(self, obj, group, squeeze=False, grouper=None, bins=None,
                 raise ValueError('index must be monotonic for resampling')
             s = pd.Series(np.arange(index.size), index)
             first_items = s.groupby(grouper).first()
+            _apply_loffset(grouper, first_items)
             full_index = first_items.index
             if first_items.isnull().any():
                 first_items = first_items.dropna()

xarray/tests/test_coding_times.py

@@ -10,6 +10,7 @@
 from xarray import DataArray, Variable, coding, decode_cf
 from xarray.coding.times import (_import_cftime, cftime_to_nptime,
                                  decode_cf_datetime, encode_cf_datetime)
+from xarray.conventions import _update_bounds_attributes
 from xarray.core.common import contains_cftime_datetimes
 
 from . import (
@@ -624,6 +625,41 @@ def test_decode_cf(calendar):
             assert ds.test.dtype == np.dtype('M8[ns]')
 
 
+def test_decode_cf_time_bounds():
+
+    da = DataArray(np.arange(6, dtype='int64').reshape((3, 2)),
+                   coords={'time': [1, 2, 3]},
+                   dims=('time', 'nbnd'), name='time_bnds')
+
+    attrs = {'units': 'days since 2001-01',
+             'calendar': 'standard',
+             'bounds': 'time_bnds'}
+
+    ds = da.to_dataset()
+    ds['time'].attrs.update(attrs)
+    _update_bounds_attributes(ds.variables)
+    assert ds.variables['time_bnds'].attrs == {'units': 'days since 2001-01',
+                                               'calendar': 'standard'}
+    dsc = decode_cf(ds)
+    assert dsc.time_bnds.dtype == np.dtype('M8[ns]')
+    dsc = decode_cf(ds, decode_times=False)
+    assert dsc.time_bnds.dtype == np.dtype('int64')
+
+    # Do not overwrite existing attrs
+    ds = da.to_dataset()
+    ds['time'].attrs.update(attrs)
+    bnd_attr = {'units': 'hours since 2001-01', 'calendar': 'noleap'}
+    ds['time_bnds'].attrs.update(bnd_attr)
+    _update_bounds_attributes(ds.variables)
+    assert ds.variables['time_bnds'].attrs == bnd_attr
+
+    # If bounds variable not available do not complain
+    ds = da.to_dataset()
+    ds['time'].attrs.update(attrs)
+    ds['time'].attrs['bounds'] = 'fake_var'
+    _update_bounds_attributes(ds.variables)
+
+
 @pytest.fixture(params=_ALL_CALENDARS)
 def calendar(request):
     return request.param

xarray/tests/test_dataarray.py

@@ -2273,6 +2273,11 @@ def test_resample(self):
         actual = array.resample(time='24H').reduce(np.mean)
         assert_identical(expected, actual)
 
+        actual = array.resample(time='24H', loffset='-12H').mean()
+        expected = DataArray(array.to_series().resample('24H', loffset='-12H')
+                             .mean())
+        assert_identical(expected, actual)
+
         with raises_regex(ValueError, 'index must be monotonic'):
             array[[2, 0, 1]].resample(time='1D')
 

xarray/tests/test_dataset.py

@@ -2804,6 +2804,18 @@ def test_resample_by_mean_with_keep_attrs(self):
         expected = ds.attrs
         assert expected == actual
 
+    def test_resample_loffset(self):
+        times = pd.date_range('2000-01-01', freq='6H', periods=10)
+        ds = Dataset({'foo': (['time', 'x', 'y'], np.random.randn(10, 5, 3)),
+                      'bar': ('time', np.random.randn(10), {'meta': 'data'}),
+                      'time': times})
+        ds.attrs['dsmeta'] = 'dsdata'
+
+        actual = ds.resample(time='24H', loffset='-12H').mean('time').time
+        expected = xr.DataArray(ds.bar.to_series()
+                                .resample('24H', loffset='-12H').mean()).time
+        assert_identical(expected, actual)
+
     def test_resample_by_mean_discarding_attrs(self):
         times = pd.date_range('2000-01-01', freq='6H', periods=10)
         ds = Dataset({'foo': (['time', 'x', 'y'], np.random.randn(10, 5, 3)),

xarray/tests/test_ufuncs.py

@@ -172,6 +172,41 @@ def test_xarray_ufuncs_deprecation():
     with pytest.warns(PendingDeprecationWarning, match='xarray.ufuncs'):
         xu.cos(xr.DataArray([0, 1]))
 
+    with pytest.warns(None) as record:
+        xu.angle(xr.DataArray([0, 1]))
+    record = [el.message for el in record
+              if el.category == PendingDeprecationWarning]
+    assert len(record) == 0
+
+
+@requires_np113
+@pytest.mark.filterwarnings('ignore::RuntimeWarning')
+@pytest.mark.parametrize(
+    'name',
+    [name for name in dir(xu)
+        if (not name.startswith('_') and hasattr(np, name)
+            and name not in ['print_function', 'absolute_import', 'division'])]
+)
+def test_numpy_ufuncs(name, request):
+    x = xr.DataArray([1, 1])
+
+    np_func = getattr(np, name)
+    if hasattr(np_func, 'nin') and np_func.nin == 2:
+        args = (x, x)
+    else:
+        args = (x,)
+
+    y = np_func(*args)
+
+    if name in ['angle', 'iscomplex']:
+        # these functions need to be handled with __array_function__ protocol
+        assert isinstance(y, np.ndarray)
+    elif name in ['frexp']:
+        # np.frexp returns a tuple
+        assert not isinstance(y, xr.DataArray)
+    else:
+        assert isinstance(y, xr.DataArray)
+
 
 def test_xarray_ufuncs_pickle():
     a = 1.0

xarray/ufuncs.py

@@ -44,10 +44,12 @@ def __init__(self, name):
         self._name = name
 
     def __call__(self, *args, **kwargs):
-        _warnings.warn(
-            'xarray.ufuncs will be deprecated when xarray no longer supports '
-            'versions of numpy older than v1.13. Instead, use numpy ufuncs '
-            'directly.', PendingDeprecationWarning, stacklevel=2)
+        if self._name not in ['angle', 'iscomplex']:
+            _warnings.warn(
+                'xarray.ufuncs will be deprecated when xarray no longer '
+                'supports versions of numpy older than v1.17. Instead, use '
+                'numpy ufuncs directly.',
+                PendingDeprecationWarning, stacklevel=2)
 
         new_args = args
         f = _dask_or_eager_func(self._name, array_args=slice(len(args)))