Skip to content

Commit 63faec3

Browse files
authored
DEPR: datetime64tz cast mismatched timezones on setitemlike (#44243)
1 parent f1286a7 commit 63faec3

File tree

10 files changed

+109
-15
lines changed

10 files changed

+109
-15
lines changed

doc/source/whatsnew/v1.4.0.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,9 @@ Other Deprecations
395395
- Deprecated :meth:`.Rolling.validate`, :meth:`.Expanding.validate`, and :meth:`.ExponentialMovingWindow.validate` (:issue:`43665`)
396396
- Deprecated silent dropping of columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a dictionary (:issue:`43740`)
397397
- Deprecated silent dropping of columns that raised a ``TypeError``, ``DataError``, and some cases of ``ValueError`` in :meth:`Series.aggregate`, :meth:`DataFrame.aggregate`, :meth:`Series.groupby.aggregate`, and :meth:`DataFrame.groupby.aggregate` when used with a list (:issue:`43740`)
398+
- Deprecated casting behavior when setting timezone-aware value(s) into a timezone-aware :class:`Series` or :class:`DataFrame` column when the timezones do not match. Previously this cast to object dtype. In a future version, the values being inserted will be converted to the series or column's existing timezone (:issue:`37605`)
399+
- Deprecated casting behavior when passing an item with mismatched-timezone to :meth:`DatetimeIndex.insert`, :meth:`DatetimeIndex.putmask`, :meth:`DatetimeIndex.where` :meth:`DatetimeIndex.fillna`, :meth:`Series.mask`, :meth:`Series.where`, :meth:`Series.fillna`, :meth:`Series.shift`, :meth:`Series.replace`, :meth:`Series.reindex` (and :class:`DataFrame` column analogues). In the past this has cast to object dtype. In a future version, these will cast the passed item to the index or series's timezone (:issue:`37605`)
400+
-
398401

399402
.. ---------------------------------------------------------------------------
400403

pandas/core/arrays/datetimes.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
)
4040
from pandas._typing import npt
4141
from pandas.errors import PerformanceWarning
42+
from pandas.util._exceptions import find_stack_level
4243
from pandas.util._validators import validate_inclusive
4344

4445
from pandas.core.dtypes.cast import astype_dt64_to_dt64tz
@@ -509,6 +510,19 @@ def _check_compatible_with(self, other, setitem: bool = False):
509510
if setitem:
510511
# Stricter check for setitem vs comparison methods
511512
if not timezones.tz_compare(self.tz, other.tz):
513+
# TODO(2.0): remove this check. GH#37605
514+
warnings.warn(
515+
"Setitem-like behavior with mismatched timezones is deprecated "
516+
"and will change in a future version. Instead of raising "
517+
"(or for Index, Series, and DataFrame methods, coercing to "
518+
"object dtype), the value being set (or passed as a "
519+
"fill_value, or inserted) will be cast to the existing "
520+
"DatetimeArray/DatetimeIndex/Series/DataFrame column's "
521+
"timezone. To retain the old behavior, explicitly cast to "
522+
"object dtype before the operation.",
523+
FutureWarning,
524+
stacklevel=find_stack_level(),
525+
)
512526
raise ValueError(f"Timezones don't match. '{self.tz}' != '{other.tz}'")
513527

514528
# -----------------------------------------------------------------

pandas/tests/arrays/test_datetimelike.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -883,7 +883,15 @@ def test_take_fill_valid(self, arr1d):
883883
msg = "Timezones don't match. .* != 'Australia/Melbourne'"
884884
with pytest.raises(ValueError, match=msg):
885885
# require tz match, not just tzawareness match
886-
arr.take([-1, 1], allow_fill=True, fill_value=value)
886+
with tm.assert_produces_warning(
887+
FutureWarning, match="mismatched timezone"
888+
):
889+
result = arr.take([-1, 1], allow_fill=True, fill_value=value)
890+
891+
# once deprecation is enforced
892+
# expected = arr.take([-1, 1], allow_fill=True,
893+
# fill_value=value.tz_convert(arr.dtype.tz))
894+
# tm.assert_equal(result, expected)
887895

888896
def test_concat_same_type_invalid(self, arr1d):
889897
# different timezones

pandas/tests/arrays/test_datetimes.py

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,14 @@ def test_setitem_different_tz_raises(self):
128128
with pytest.raises(TypeError, match="Cannot compare tz-naive and tz-aware"):
129129
arr[0] = pd.Timestamp("2000")
130130

131+
ts = pd.Timestamp("2000", tz="US/Eastern")
131132
with pytest.raises(ValueError, match="US/Central"):
132-
arr[0] = pd.Timestamp("2000", tz="US/Eastern")
133+
with tm.assert_produces_warning(
134+
FutureWarning, match="mismatched timezones"
135+
):
136+
arr[0] = ts
137+
# once deprecation is enforced
138+
# assert arr[0] == ts.tz_convert("US/Central")
133139

134140
def test_setitem_clears_freq(self):
135141
a = DatetimeArray(pd.date_range("2000", periods=2, freq="D", tz="US/Central"))
@@ -385,7 +391,14 @@ def test_shift_requires_tzmatch(self):
385391

386392
msg = "Timezones don't match. 'UTC' != 'US/Pacific'"
387393
with pytest.raises(ValueError, match=msg):
388-
dta.shift(1, fill_value=fill_value)
394+
with tm.assert_produces_warning(
395+
FutureWarning, match="mismatched timezones"
396+
):
397+
dta.shift(1, fill_value=fill_value)
398+
399+
# once deprecation is enforced
400+
# expected = dta.shift(1, fill_value=fill_value.tz_convert("UTC"))
401+
# tm.assert_equal(result, expected)
389402

390403
def test_tz_localize_t2d(self):
391404
dti = pd.date_range("1994-05-12", periods=12, tz="US/Pacific")

pandas/tests/frame/methods/test_replace.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1109,12 +1109,17 @@ def test_replace_datetimetz(self):
11091109
# coerce to object
11101110
result = df.copy()
11111111
result.iloc[1, 0] = np.nan
1112-
result = result.replace({"A": pd.NaT}, Timestamp("20130104", tz="US/Pacific"))
1112+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
1113+
result = result.replace(
1114+
{"A": pd.NaT}, Timestamp("20130104", tz="US/Pacific")
1115+
)
11131116
expected = DataFrame(
11141117
{
11151118
"A": [
11161119
Timestamp("20130101", tz="US/Eastern"),
11171120
Timestamp("20130104", tz="US/Pacific"),
1121+
# once deprecation is enforced
1122+
# Timestamp("20130104", tz="US/Pacific").tz_convert("US/Eastern"),
11181123
Timestamp("20130103", tz="US/Eastern"),
11191124
],
11201125
"B": [0, np.nan, 2],

pandas/tests/indexes/datetimes/methods/test_insert.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,18 +197,32 @@ def test_insert_mismatched_tz(self):
197197

198198
# mismatched tz -> cast to object (could reasonably cast to same tz or UTC)
199199
item = Timestamp("2000-01-04", tz="US/Eastern")
200-
result = idx.insert(3, item)
200+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
201+
result = idx.insert(3, item)
201202
expected = Index(
202-
list(idx[:3]) + [item] + list(idx[3:]), dtype=object, name="idx"
203+
list(idx[:3]) + [item] + list(idx[3:]),
204+
dtype=object,
205+
# once deprecation is enforced
206+
# list(idx[:3]) + [item.tz_convert(idx.tz)] + list(idx[3:]),
207+
name="idx",
203208
)
209+
# once deprecation is enforced
210+
# assert expected.dtype == idx.dtype
204211
tm.assert_index_equal(result, expected)
205212

206213
# mismatched tz -> cast to object (could reasonably cast to same tz)
207214
item = datetime(2000, 1, 4, tzinfo=pytz.timezone("US/Eastern"))
208-
result = idx.insert(3, item)
215+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
216+
result = idx.insert(3, item)
209217
expected = Index(
210-
list(idx[:3]) + [item] + list(idx[3:]), dtype=object, name="idx"
218+
list(idx[:3]) + [item] + list(idx[3:]),
219+
dtype=object,
220+
# once deprecation is enforced
221+
# list(idx[:3]) + [item.astimezone(idx.tzinfo)] + list(idx[3:]),
222+
name="idx",
211223
)
224+
# once deprecation is enforced
225+
# assert expected.dtype == idx.dtype
212226
tm.assert_index_equal(result, expected)
213227

214228
@pytest.mark.parametrize(

pandas/tests/indexing/test_coercion.py

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -237,11 +237,17 @@ def test_setitem_series_datetime64tz(self, val, exp_dtype):
237237
[
238238
pd.Timestamp("2011-01-01", tz=tz),
239239
val,
240+
# once deprecation is enforced
241+
# val if getattr(val, "tz", None) is None else val.tz_convert(tz),
240242
pd.Timestamp("2011-01-03", tz=tz),
241243
pd.Timestamp("2011-01-04", tz=tz),
242244
]
243245
)
244-
self._assert_setitem_series_conversion(obj, val, exp, exp_dtype)
246+
warn = None
247+
if getattr(val, "tz", None) is not None and val.tz != obj[0].tz:
248+
warn = FutureWarning
249+
with tm.assert_produces_warning(warn, match="mismatched timezones"):
250+
self._assert_setitem_series_conversion(obj, val, exp, exp_dtype)
245251

246252
@pytest.mark.parametrize(
247253
"val,exp_dtype",
@@ -467,9 +473,12 @@ def test_insert_index_datetimes(self, request, fill_val, exp_dtype, insert_value
467473

468474
# mismatched tz --> cast to object (could reasonably cast to common tz)
469475
ts = pd.Timestamp("2012-01-01", tz="Asia/Tokyo")
470-
result = obj.insert(1, ts)
476+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
477+
result = obj.insert(1, ts)
478+
# once deprecation is enforced:
479+
# expected = obj.insert(1, ts.tz_convert(obj.dtype.tz))
480+
# assert expected.dtype == obj.dtype
471481
expected = obj.astype(object).insert(1, ts)
472-
assert expected.dtype == object
473482
tm.assert_index_equal(result, expected)
474483

475484
else:
@@ -990,11 +999,18 @@ def test_fillna_datetime64tz(self, index_or_series, fill_val, fill_dtype):
990999
[
9911000
pd.Timestamp("2011-01-01", tz=tz),
9921001
fill_val,
1002+
# Once deprecation is enforced, this becomes:
1003+
# fill_val.tz_convert(tz) if getattr(fill_val, "tz", None)
1004+
# is not None else fill_val,
9931005
pd.Timestamp("2011-01-03", tz=tz),
9941006
pd.Timestamp("2011-01-04", tz=tz),
9951007
]
9961008
)
997-
self._assert_fillna_conversion(obj, fill_val, exp, fill_dtype)
1009+
warn = None
1010+
if getattr(fill_val, "tz", None) is not None and fill_val.tz != obj[0].tz:
1011+
warn = FutureWarning
1012+
with tm.assert_produces_warning(warn, match="mismatched timezone"):
1013+
self._assert_fillna_conversion(obj, fill_val, exp, fill_dtype)
9981014

9991015
@pytest.mark.xfail(reason="Test not implemented")
10001016
def test_fillna_series_int64(self):

pandas/tests/indexing/test_loc.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1890,7 +1890,8 @@ def test_setitem_with_expansion(self):
18901890
# trying to set a single element on a part of a different timezone
18911891
# this converts to object
18921892
df2 = df.copy()
1893-
df2.loc[df2.new_col == "new", "time"] = v
1893+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
1894+
df2.loc[df2.new_col == "new", "time"] = v
18941895

18951896
expected = Series([v[0], df.loc[1, "time"]], name="time")
18961897
tm.assert_series_equal(df2.time, expected)

pandas/tests/series/indexing/test_setitem.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -898,6 +898,18 @@ def expected(self):
898898
)
899899
return expected
900900

901+
@pytest.fixture(autouse=True)
902+
def assert_warns(self, request):
903+
# check that we issue a FutureWarning about timezone-matching
904+
if request.function.__name__ == "test_slice_key":
905+
key = request.getfixturevalue("key")
906+
if not isinstance(key, slice):
907+
# The test is a no-op, so no warning will be issued
908+
yield
909+
return
910+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
911+
yield
912+
901913

902914
@pytest.mark.parametrize(
903915
"obj,expected",

pandas/tests/series/methods/test_fillna.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,8 @@ def test_datetime64_tz_fillna(self, tz):
523523
tm.assert_series_equal(expected, result)
524524
tm.assert_series_equal(isna(ser), null_loc)
525525

526-
result = ser.fillna(Timestamp("20130101", tz="US/Pacific"))
526+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
527+
result = ser.fillna(Timestamp("20130101", tz="US/Pacific"))
527528
expected = Series(
528529
[
529530
Timestamp("2011-01-01 10:00", tz=tz),
@@ -766,8 +767,15 @@ def test_fillna_datetime64_with_timezone_tzinfo(self):
766767
# but we dont (yet) consider distinct tzinfos for non-UTC tz equivalent
767768
ts = Timestamp("2000-01-01", tz="US/Pacific")
768769
ser2 = Series(ser._values.tz_convert("dateutil/US/Pacific"))
769-
result = ser2.fillna(ts)
770+
assert ser2.dtype.kind == "M"
771+
with tm.assert_produces_warning(FutureWarning, match="mismatched timezone"):
772+
result = ser2.fillna(ts)
770773
expected = Series([ser[0], ts, ser[2]], dtype=object)
774+
# once deprecation is enforced
775+
# expected = Series(
776+
# [ser2[0], ts.tz_convert(ser2.dtype.tz), ser2[2]],
777+
# dtype=ser2.dtype,
778+
# )
771779
tm.assert_series_equal(result, expected)
772780

773781
def test_fillna_pos_args_deprecation(self):

0 commit comments

Comments
 (0)