Merge branch 'master' into fixturize_frame_analytics

Author: h-vetinari

.travis.yml

@@ -64,7 +64,7 @@ matrix:
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6, NumPy dev" ENV_FILE="ci/travis-36-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
+        - JOB="3.7, NumPy dev" ENV_FILE="ci/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
@@ -79,7 +79,7 @@ matrix:
           - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
       - dist: trusty
         env:
-          - JOB="3.6, NumPy dev" ENV_FILE="ci/travis-36-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
+          - JOB="3.7, NumPy dev" ENV_FILE="ci/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
         addons:
           apt:
             packages:

ci/doctests.sh

@@ -21,21 +21,21 @@ if [ "$DOCTEST" ]; then
 
     # DataFrame / Series docstrings
     pytest --doctest-modules -v pandas/core/frame.py \
-        -k"-assign -axes -combine -isin -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_stata -transform"
+        -k"-assign -axes -combine -isin -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_stata"
 
     if [ $? -ne "0" ]; then
         RET=1
     fi
 
     pytest --doctest-modules -v pandas/core/series.py \
-        -k"-nlargest -nonzero -nsmallest -reindex -searchsorted -to_dict"
+        -k"-nonzero -reindex -searchsorted -to_dict"
 
     if [ $? -ne "0" ]; then
         RET=1
     fi
 
     pytest --doctest-modules -v pandas/core/generic.py \
-        -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -resample -sample -to_json -to_xarray -transform -transpose -values -xs"
+        -k"-_set_axis_name -_xs -describe -droplevel -groupby -interpolate -pct_change -pipe -reindex -reindex_axis -resample -sample -to_json -to_xarray -transpose -values -xs"
 
     if [ $? -ne "0" ]; then
         RET=1

ci/travis-36-numpydev.yaml renamed to ci/travis-37-numpydev.yaml

@@ -2,7 +2,7 @@ name: pandas
 channels:
   - defaults
 dependencies:
-  - python=3.6*
+  - python=3.7*
   - pytz
   - Cython>=0.28.2
   # universal

doc/source/api.rst

@@ -61,6 +61,12 @@ Excel
    read_excel
    ExcelFile.parse
 
+.. autosummary::
+   :toctree: generated/
+   :template: autosummary/class_without_autosummary.rst
+
+   ExcelWriter
+
 JSON
 ~~~~
 

doc/source/contributing.rst

@@ -632,6 +632,14 @@ Otherwise, you need to do it manually:
         warnings.warn('Use new_func instead.', FutureWarning, stacklevel=2)
         new_func()
 
+You'll also need to
+
+1. write a new test that asserts a warning is issued when calling with the deprecated argument
+2. Update all of pandas existing tests and code to use the new argument
+
+See :ref:`contributing.warnings` for more.
+
+
 .. _contributing.ci:
 
 Testing With Continuous Integration
@@ -859,6 +867,55 @@ preferred if the inputs or logic are simple, with Hypothesis tests reserved
 for cases with complex logic or where there are too many combinations of
 options or subtle interactions to test (or think of!) all of them.
 
+.. _contributing.warnings:
+
+Testing Warnings
+~~~~~~~~~~~~~~~~
+
+By default, one of pandas CI workers will fail if any unhandled warnings are emitted.
+
+If your change involves checking that a warning is actually emitted, use
+``tm.assert_produces_warning(ExpectedWarning)``.
+
+
+.. code-block:: python
+
+   with tm.assert_prodcues_warning(FutureWarning):
+       df.some_operation()
+
+We prefer this to the ``pytest.warns`` context manager because ours checks that the warning's
+stacklevel is set correctly. The stacklevel is what ensure the *user's* file name and line number
+is printed in the warning, rather than something internal to pandas. It represents the number of
+function calls from user code (e.g. ``df.some_operation()``) to the function that actually emits
+the warning. Our linter will fail the build if you use ``pytest.warns`` in a test.
+
+If you have a test that would emit a warning, but you aren't actually testing the
+warning itself (say because it's going to be removed in the future, or because we're
+matching a 3rd-party library's behavior), then use ``pytest.mark.filterwarnings`` to
+ignore the error.
+
+.. code-block:: python
+
+   @pytest.mark.filterwarnings("ignore:msg:category")
+   def test_thing(self):
+       ...
+
+If the test generates a warning of class ``category`` whose message starts
+with ``msg``, the warning will be ignored and the test will pass.
+
+If you need finer-grained control, you can use Python's usual
+`warnings module <https://docs.python.org/3/library/warnings.html>`__
+to control whether a warning is ignored / raised at different places within
+a single test.
+
+.. code-block:: python
+
+   with warch.catch_warnings():
+       warnings.simplefilter("ignore", FutureWarning)
+       # Or use warnings.filterwarnings(...)
+
+Alternatively, consider breaking up the unit test.
+
 
 Running the test suite
 ----------------------

doc/source/timeseries.rst

@@ -21,69 +21,114 @@
 Time Series / Date functionality
 ********************************
 
-pandas has proven very successful as a tool for working with time series data,
-especially in the financial data analysis space. Using the NumPy ``datetime64`` and ``timedelta64`` dtypes,
-we have consolidated a large number of features from other Python libraries like ``scikits.timeseries`` as well as created
+pandas contains extensive capabilities and features for working with time series data for all domains.
+Using the NumPy ``datetime64`` and ``timedelta64`` dtypes, pandas has consolidated a large number of
+features from other Python libraries like ``scikits.timeseries`` as well as created
 a tremendous amount of new functionality for manipulating time series data.
 
-In working with time series data, we will frequently seek to:
+For example, pandas supports:
 
-* generate sequences of fixed-frequency dates and time spans
-* conform or convert time series to a particular frequency
-* compute "relative" dates based on various non-standard time increments
-  (e.g. 5 business days before the last business day of the year), or "roll"
-  dates forward or backward
+Parsing time series information from various sources and formats
 
-pandas provides a relatively compact and self-contained set of tools for
-performing the above tasks.
+.. ipython:: python
+
+   dti = pd.to_datetime(['1/1/2018', np.datetime64('2018-01-01'), datetime(2018, 1, 1)])
+   dti
 
-Create a range of dates:
+Generate sequences of fixed-frequency dates and time spans
 
 .. ipython:: python
 
-   # 72 hours starting with midnight Jan 1st, 2011
-   rng = pd.date_range('1/1/2011', periods=72, freq='H')
-   rng[:5]
+   dti = pd.date_range('2018-01-01', periods=3, freq='H')
+   dti
 
-Index pandas objects with dates:
+Manipulating and converting date times with timezone information
 
 .. ipython:: python
 
-   ts = pd.Series(np.random.randn(len(rng)), index=rng)
-   ts.head()
+   dti = dti.tz_localize('UTC')
+   dti
+   dti.tz_convert('US/Pacific')
 
-Change frequency and fill gaps:
+Resampling or converting a time series to a particular frequency
 
 .. ipython:: python
 
-   # to 45 minute frequency and forward fill
-   converted = ts.asfreq('45Min', method='pad')
-   converted.head()
+   idx = pd.date_range('2018-01-01', periods=5, freq='H')
+   ts = pd.Series(range(len(idx)), index=idx)
+   ts
+   ts.resample('2H').mean()
 
-Resample the series to a daily frequency:
+Performing date and time arithmetic with absolute or relative time increments
 
 .. ipython:: python
 
-   # Daily means
-   ts.resample('D').mean()
+    friday = pd.Timestamp('2018-01-05')
+    friday.day_name()
+    # Add 1 day
+    saturday = friday + pd.Timedelta('1 day')
+    saturday.day_name()
+    # Add 1 business day (Friday --> Monday)
+    monday = friday + pd.tseries.offsets.BDay()
+    monday.day_name()
+
+pandas provides a relatively compact and self-contained set of tools for
+performing the above tasks and more.
 
 
 .. _timeseries.overview:
 
 Overview
 --------
 
-The following table shows the type of time-related classes pandas can handle and
-how to create them.
+pandas captures 4 general time related concepts:
+
+#. Date times: A specific date and time with timezone support. Similar to ``datetime.datetime`` from the standard library.
+#. Time deltas: An absolute time duration. Similar to ``datetime.timedelta`` from the standard library.
+#. Time spans: A span of time defined by a point in time and its associated frequency.
+#. Date offsets: A relative time duration that respects calendar arithmetic. Similar to ``dateutil.relativedelta.relativedelta`` from the ``dateutil`` package.
 
-=================  =============================== ===================================================================
-Class              Remarks                         How to create
-=================  =============================== ===================================================================
-``Timestamp``      Represents a single timestamp   ``to_datetime``, ``Timestamp``
-``DatetimeIndex``  Index of ``Timestamp``          ``to_datetime``, ``date_range``, ``bdate_range``, ``DatetimeIndex``
-``Period``         Represents a single time span   ``Period``
-``PeriodIndex``    Index of ``Period``             ``period_range``, ``PeriodIndex``
-=================  =============================== ===================================================================
+=====================   =================  ===================   ============================================  ========================================
+Concept                 Scalar Class       Array Class           pandas Data Type                              Primary Creation Method
+=====================   =================  ===================   ============================================  ========================================
+Date times              ``Timestamp``      ``DatetimeIndex``     ``datetime64[ns]`` or ``datetime64[ns, tz]``  ``to_datetime`` or ``date_range``
+Time deltas             ``Timedelta``      ``TimedeltaIndex``    ``timedelta64[ns]``                           ``to_timedelta`` or ``timedelta_range``
+Time spans              ``Period``         ``PeriodIndex``       ``period[freq]``                              ``Period`` or ``period_range``
+Date offsets            ``DateOffset``     ``None``              ``None``                                      ``DateOffset``
+=====================   =================  ===================   ============================================  ========================================
+
+For time series data, it's conventional to represent the time component in the index of a :class:`Series` or :class:`DataFrame`
+so manipulations can be performed with respect to the time element.
+
+.. ipython:: python
+
+   pd.Series(range(3), index=pd.date_range('2000', freq='D', periods=3))
+
+However, :class:`Series` and :class:`DataFrame` can directly also support the time component as data itself.
+
+.. ipython:: python
+
+   pd.Series(pd.date_range('2000', freq='D', periods=3))
+
+:class:`Series` and :class:`DataFrame` have extended data type support and functionality for ``datetime`` and ``timedelta``
+data when the time data is used as data itself. The ``Period`` and ``DateOffset`` data will be stored as ``object`` data.
+
+.. ipython:: python
+
+   pd.Series(pd.period_range('1/1/2011', freq='M', periods=3))
+   pd.Series(pd.date_range('1/1/2011', freq='M', periods=3))
+
+Lastly, pandas represents null date times, time deltas, and time spans as ``NaT`` which
+is useful for representing missing or null date like values and behaves similar
+as ``np.nan`` does for float data.
+
+.. ipython:: python
+
+   pd.Timestamp(pd.NaT)
+   pd.Timedelta(pd.NaT)
+   pd.Period(pd.NaT)
+   # Equality acts as np.nan would
+   pd.NaT == pd.NaT
 
 .. _timeseries.representation:
 
@@ -1443,7 +1488,7 @@ time. The method for this is :meth:`~Series.shift`, which is available on all of
 the pandas objects.
 
 .. ipython:: python
-
+   ts = pd.Series(range(len(rng)), index=rng)
    ts = ts[:5]
    ts.shift(1)
 

doc/source/whatsnew/v0.24.0.txt

@@ -170,9 +170,9 @@ Other Enhancements
 - :meth:`Series.droplevel` and :meth:`DataFrame.droplevel` are now implemented (:issue:`20342`)
 - Added support for reading from Google Cloud Storage via the ``gcsfs`` library (:issue:`19454`)
 - :func:`to_gbq` and :func:`read_gbq` signature and documentation updated to
-  reflect changes from the `Pandas-GBQ library version 0.5.0
-  <https://pandas-gbq.readthedocs.io/en/latest/changelog.html#changelog-0-5-0>`__.
-  (:issue:`21627`)
+  reflect changes from the `Pandas-GBQ library version 0.6.0
+  <https://pandas-gbq.readthedocs.io/en/latest/changelog.html#changelog-0-6-0>`__.
+  (:issue:`21627`, :issue:`22557`)
 - New method :meth:`HDFStore.walk` will recursively walk the group hierarchy of an HDF5 file (:issue:`10932`)
 - :func:`read_html` copies cell data across ``colspan`` and ``rowspan``, and it treats all-``th`` table rows as headers if ``header`` kwarg is not given and there is no ``thead`` (:issue:`17054`)
 - :meth:`Series.nlargest`, :meth:`Series.nsmallest`, :meth:`DataFrame.nlargest`, and :meth:`DataFrame.nsmallest` now accept the value ``"all"`` for the ``keep`` argument. This keeps all ties for the nth largest/smallest value (:issue:`16818`)
@@ -577,6 +577,7 @@ Removal of prior version deprecations/changes
 - Removed the ``pandas.formats.style`` shim for :class:`pandas.io.formats.style.Styler` (:issue:`16059`)
 - :meth:`Categorical.searchsorted` and :meth:`Series.searchsorted` have renamed the ``v`` argument to ``value`` (:issue:`14645`)
 - :meth:`TimedeltaIndex.searchsorted`, :meth:`DatetimeIndex.searchsorted`, and :meth:`PeriodIndex.searchsorted` have renamed the ``key`` argument to ``value`` (:issue:`14645`)
+- Removal of the previously deprecated module ``pandas.json`` (:issue:`19944`)
 
 .. _whatsnew_0240.performance:
 
@@ -635,7 +636,7 @@ Datetimelike
 - Bug in :meth:`DataFrame.eq` comparison against ``NaT`` incorrectly returning ``True`` or ``NaN`` (:issue:`15697`, :issue:`22163`)
 - Bug in :class:`DatetimeIndex` subtraction that incorrectly failed to raise ``OverflowError`` (:issue:`22492`, :issue:`22508`)
 - Bug in :class:`DatetimeIndex` incorrectly allowing indexing with ``Timedelta`` object (:issue:`20464`)
--
+- Bug in :class:`DatetimeIndex` where frequency was being set if original frequency was ``None`` (:issue:`22150`)
 
 Timedelta
 ^^^^^^^^^
@@ -645,6 +646,7 @@ Timedelta
 - Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`22390`)
 - Bug in :class:`Index` with numeric dtype when multiplying or dividing an array with dtype ``timedelta64`` (:issue:`22390`)
 - Bug in :class:`TimedeltaIndex` incorrectly allowing indexing with ``Timestamp`` object (:issue:`20464`)
+- Fixed bug where subtracting :class:`Timedelta` from an object-dtyped array would raise ``TypeError`` (:issue:`21980`)
 -
 -
 
@@ -720,13 +722,16 @@ Indexing
 - Bug where mixed indexes wouldn't allow integers for ``.at`` (:issue:`19860`)
 - ``Float64Index.get_loc`` now raises ``KeyError`` when boolean key passed. (:issue:`19087`)
 - Bug in :meth:`DataFrame.loc` when indexing with an :class:`IntervalIndex` (:issue:`19977`)
+- :class:`Index` no longer mangles ``None``, ``NaN`` and ``NaT``, i.e. they are treated as three different keys. However, for numeric Index all three are still coerced to a ``NaN`` (:issue:`22332`)
 
 Missing
 ^^^^^^^
 
 - Bug in :func:`DataFrame.fillna` where a ``ValueError`` would raise when one column contained a ``datetime64[ns, tz]`` dtype (:issue:`15522`)
 - Bug in :func:`Series.hasnans` that could be incorrectly cached and return incorrect answers if null elements are introduced after an initial call (:issue:`19700`)
-- :func:`Series.isin` now treats all nans as equal also for ``np.object``-dtype. This behavior is consistent with the behavior for float64 (:issue:`22119`)
+- :func:`Series.isin` now treats all NaN-floats as equal also for `np.object`-dtype. This behavior is consistent with the behavior for float64 (:issue:`22119`)
+- :func:`unique` no longer mangles NaN-floats and the ``NaT``-object for `np.object`-dtype, i.e. ``NaT`` is no longer coerced to a NaN-value and is treated as a different entity. (:issue:`22295`)
+
 
 MultiIndex
 ^^^^^^^^^^
@@ -742,6 +747,8 @@ I/O
 - :func:`read_excel()` will correctly show the deprecation warning for previously deprecated ``sheetname`` (:issue:`17994`)
 - :func:`read_csv()` will correctly parse timezone-aware datetimes (:issue:`22256`)
 - :func:`read_sas()` will parse numbers in sas7bdat-files that have width less than 8 bytes correctly. (:issue:`21616`)
+- :func:`read_sas()` will correctly parse sas7bdat files with many columns (:issue:`22628`)
+- :func:`read_sas()` will correctly parse sas7bdat files with data page types having also bit 7 set (so page type is 128 + 256 = 384) (:issue:`16615`)
 
 Plotting
 ^^^^^^^^
@@ -761,6 +768,7 @@ Groupby/Resample/Rolling
 - Bug in :meth:`Resampler.apply` when passing postiional arguments to applied func (:issue:`14615`).
 - Bug in :meth:`Series.resample` when passing ``numpy.timedelta64`` to ``loffset`` kwarg (:issue:`7687`).
 - Bug in :meth:`Resampler.asfreq` when frequency of ``TimedeltaIndex`` is a subperiod of a new frequency (:issue:`13022`).
+- Bug in :meth:`SeriesGroupBy.mean` when values were integral but could not fit inside of int64, overflowing instead. (:issue:`22487`)
 
 Sparse
 ^^^^^^
@@ -796,7 +804,8 @@ Other
 - :meth:`~pandas.io.formats.style.Styler.background_gradient` now takes a ``text_color_threshold`` parameter to automatically lighten the text color based on the luminance of the background color. This improves readability with dark background colors without the need to limit the background colormap range. (:issue:`21258`)
 - Require at least 0.28.2 version of ``cython`` to support read-only memoryviews (:issue:`21688`)
 - :meth:`~pandas.io.formats.style.Styler.background_gradient` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` (:issue:`15204`)
-- :meth:`~pandas.io.formats.style.Styler.bar` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` and setting clipping range with ``vmin`` and ``vmax``. ``NaN`` values are also handled properly. (:issue:`21548`, :issue:`21526`)
+- :meth:`~pandas.io.formats.style.Styler.bar` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` and setting clipping range with ``vmin`` and ``vmax`` (:issue:`21548` and :issue:`21526`). ``NaN`` values are also handled properly.
+- Logical operations ``&, |, ^`` between :class:`Series` and :class:`Index` will no longer raise ``ValueError`` (:issue:`22092`)
 -
 -
 -

pandas/__init__.py

@@ -61,9 +61,6 @@
 # extension module deprecations
 from pandas.util._depr_module import _DeprecatedModule
 
-json = _DeprecatedModule(deprmod='pandas.json',
-                         moved={'dumps': 'pandas.io.json.dumps',
-                                'loads': 'pandas.io.json.loads'})
 parser = _DeprecatedModule(deprmod='pandas.parser',
                            removals=['na_values'],
                            moved={'CParserError': 'pandas.errors.ParserError'})