Merge remote-tracking branch 'upstream/master' into am-concat

Author: jorisvandenbossche

.pre-commit-config.yaml

@@ -127,6 +127,12 @@ repos:
         types: [python]
         files: ^pandas/tests/
         exclude: ^pandas/tests/extension/
+    -   id: unwanted-patters-pytest-xfail
+        name: Check for use of pytest.xfail
+        entry: pytest\.xfail
+        language: pygrep
+        types: [python]
+        files: ^pandas/tests/
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace in tests'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py

asv_bench/benchmarks/categoricals.py

@@ -323,7 +323,7 @@ def time_get_loc(self):
         self.index.get_loc(self.category)
 
     def time_shallow_copy(self):
-        self.index._shallow_copy()
+        self.index._view()
 
     def time_align(self):
         pd.DataFrame({"a": self.series, "b": self.series[:500]})

asv_bench/benchmarks/period.py

@@ -86,7 +86,7 @@ def time_get_loc(self):
         self.index.get_loc(self.period)
 
     def time_shallow_copy(self):
-        self.index._shallow_copy()
+        self.index._view()
 
     def time_series_loc(self):
         self.series.loc[self.period]

asv_bench/benchmarks/rolling.py

@@ -255,6 +255,19 @@ def time_rolling_multiindex_creation(self):
 
 class GroupbyEWM:
 
+    params = ["var", "std", "cov", "corr"]
+    param_names = ["method"]
+
+    def setup(self, method):
+        df = pd.DataFrame({"A": range(50), "B": range(50)})
+        self.gb_ewm = df.groupby("A").ewm(com=1.0)
+
+    def time_groupby_method(self, method):
+        getattr(self.gb_ewm, method)()
+
+
+class GroupbyEWMEngine:
+
     params = ["cython", "numba"]
     param_names = ["engine"]
 

asv_bench/benchmarks/timedelta.py

@@ -74,7 +74,7 @@ def time_get_loc(self):
         self.index.get_loc(self.timedelta)
 
     def time_shallow_copy(self):
-        self.index._shallow_copy()
+        self.index._view()
 
     def time_series_loc(self):
         self.series.loc[self.timedelta]

doc/source/development/code_style.rst

@@ -161,6 +161,46 @@ For example:
     # wrong
     from common import test_base
 
+Testing
+=======
+
+Failing tests
+--------------
+
+See https://docs.pytest.org/en/latest/skipping.html for background.
+
+Do not use ``pytest.xfail``
+---------------------------
+
+Do not use this method. It has the same behavior as ``pytest.skip``, namely
+it immediately stops the test and does not check if the test will fail. If
+this is the behavior you desire, use ``pytest.skip`` instead.
+
+Using ``pytest.mark.xfail``
+---------------------------
+
+Use this method if a test is known to fail but the manner in which it fails
+is not meant to be captured. It is common to use this method for a test that
+exhibits buggy behavior or a non-implemented feature. If
+the failing test has flaky behavior, use the argument ``strict=False``. This
+will make it so pytest does not fail if the test happens to pass.
+
+Prefer the decorator ``@pytest.mark.xfail`` and the argument ``pytest.param``
+over usage within a test so that the test is appropriately marked during the
+collection phase of pytest. For xfailing a test that involves multiple
+parameters, a fixture, or a combination of these, it is only possible to
+xfail during the testing phase. To do so, use the ``request`` fixture:
+
+.. code-block:: python
+
+    import pytest
+
+    def test_xfail(request):
+        request.node.add_marker(pytest.mark.xfail(reason="Indicate why here"))
+
+xfail is not to be used for tests involving failure due to invalid user arguments.
+For these tests, we need to verify the correct exception type and error message
+is being raised, using ``pytest.raises`` instead.
 
 Miscellaneous
 =============

doc/source/user_guide/dsintro.rst

@@ -126,7 +126,7 @@ However, operations such as slicing will also slice the index.
 .. note::
 
    We will address array-based indexing like ``s[[4, 3, 1]]``
-   in :ref:`section <indexing>`.
+   in :ref:`section on indexing <indexing>`.
 
 Like a NumPy array, a pandas Series has a :attr:`~Series.dtype`.
 

doc/source/user_guide/style.ipynb

@@ -140,9 +140,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "s = df.style.set_table_attributes('class=\"table-cls\"')\n",
-    "cls = pd.DataFrame(data=[['cls1', None], ['cls3', 'cls2 cls3']], index=[0,2], columns=['A', 'C'])\n",
-    "s.set_td_classes(cls)"
+    "css_classes = pd.DataFrame(data=[['cls1', None], ['cls3', 'cls2 cls3']], index=[0,2], columns=['A', 'C'])\n",
+    "df.style.\\\n",
+    "    set_table_attributes('class=\"table-cls\"').\\\n",
+    "    set_td_classes(css_classes)"
    ]
   },
   {
@@ -314,13 +315,10 @@
    "outputs": [],
    "source": [
     "def color_negative_red(val):\n",
-    "    \"\"\"\n",
-    "    Takes a scalar and returns a string with\n",
-    "    the css property `'color: red'` for negative\n",
-    "    strings, black otherwise.\n",
-    "    \"\"\"\n",
-    "    color = 'red' if val < 0 else 'black'\n",
-    "    return 'color: %s' % color"
+    "    \"\"\"Color negative scalars red.\"\"\"\n",
+    "    css = 'color: red;'\n",
+    "    if val < 0: return css\n",
+    "    return None"
    ]
   },
   {
@@ -368,11 +366,9 @@
    "outputs": [],
    "source": [
     "def highlight_max(s):\n",
-    "    '''\n",
-    "    highlight the maximum in a Series yellow.\n",
-    "    '''\n",
-    "    is_max = s == s.max()\n",
-    "    return ['background-color: yellow' if v else '' for v in is_max]"
+    "    \"\"\"Highlight the maximum in a Series bold-orange.\"\"\"\n",
+    "    css = 'background-color: orange; font-weight: bold;'\n",
+    "    return np.where(s == np.nanmax(s.values), css, None)"
    ]
   },
   {
@@ -384,11 +380,20 @@
     "df.style.apply(highlight_max)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.style.apply(highlight_max, axis=1)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "In this case the input is a `Series`, one column at a time.\n",
+    "In this case the input is a `Series`, one column (or row) at a time.\n",
     "Notice that the output shape of `highlight_max` matches the input shape, an array with `len(s)` items."
    ]
   },
@@ -406,8 +411,8 @@
    "outputs": [],
    "source": [
     "def compare_col(s, comparator=None):\n",
-    "    attr = 'background-color: #00BFFF;'\n",
-    "    return np.where(s < comparator, attr, '')"
+    "    css = 'background-color: #00BFFF;'\n",
+    "    return np.where(s < comparator, css, None)"
    ]
   },
   {
@@ -442,41 +447,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Above we used `Styler.apply` to pass in each column one at a time.\n",
+    "Above we used `Styler.apply` to pass in each column (or row) one at a time.\n",
     "\n",
     "<span style=\"background-color: #DEDEBE\">*Debugging Tip*: If you're having trouble writing your style function, try just passing it into <code style=\"background-color: #DEDEBE\">DataFrame.apply</code>. Internally, <code style=\"background-color: #DEDEBE\">Styler.apply</code> uses <code style=\"background-color: #DEDEBE\">DataFrame.apply</code> so the result should be the same.</span>\n",
     "\n",
     "What if you wanted to highlight just the maximum value in the entire table?\n",
-    "Use `.apply(function, axis=None)` to indicate that your function wants the entire table, not one column or row at a time. Let's try that next.\n",
-    "\n",
-    "We'll rewrite our `highlight-max` to handle either Series (from `.apply(axis=0 or 1)`) or DataFrames (from `.apply(axis=None)`). We'll also allow the color to be adjustable, to demonstrate that `.apply`, and `.applymap` pass along keyword arguments."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def highlight_max(data, color='yellow'):\n",
-    "    '''\n",
-    "    highlight the maximum in a Series or DataFrame\n",
-    "    '''\n",
-    "    attr = 'background-color: {}'.format(color)\n",
-    "    if data.ndim == 1:  # Series from .apply(axis=0) or axis=1\n",
-    "        is_max = data == data.max()\n",
-    "        return [attr if v else '' for v in is_max]\n",
-    "    else:  # from .apply(axis=None)\n",
-    "        is_max = data == data.max().max()\n",
-    "        return pd.DataFrame(np.where(is_max, attr, ''),\n",
-    "                            index=data.index, columns=data.columns)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "When using ``Styler.apply(func, axis=None)``, the function must return a DataFrame with the same index and column labels."
+    "Use `.apply(function, axis=None)` to indicate that your function wants the entire table, not one column or row at a time. In this case the return must be a DataFrame or ndarray of the same shape as the input. Let's try that next. "
    ]
   },
   {
@@ -485,7 +461,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "s = df.style.apply(highlight_max, color='darkorange', axis=None)\n",
+    "s = df.style.apply(highlight_max, axis=None)\n",
     "s"
    ]
   },

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.2
 .. toctree::
    :maxdepth: 2
 
+   v1.2.3
    v1.2.2
    v1.2.1
    v1.2.0

doc/source/whatsnew/v1.2.2.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_122:
 
-What's new in 1.2.2 (February ??, 2021)
+What's new in 1.2.2 (February 09, 2021)
 ---------------------------------------
 
 These are the changes in pandas 1.2.2. See :ref:`release` for a full changelog
@@ -21,11 +21,12 @@ Fixed regressions
 - Fixed regression in :meth:`~DataFrame.to_pickle` failing to create bz2/xz compressed pickle files with ``protocol=5`` (:issue:`39002`)
 - Fixed regression in :func:`pandas.testing.assert_series_equal` and :func:`pandas.testing.assert_frame_equal` always raising ``AssertionError`` when comparing extension dtypes (:issue:`39410`)
 - Fixed regression in :meth:`~DataFrame.to_csv` opening ``codecs.StreamWriter`` in binary mode instead of in text mode and ignoring user-provided ``mode`` (:issue:`39247`)
+- Fixed regression in :meth:`Categorical.astype` casting to incorrect dtype when ``np.int32`` is passed to dtype argument (:issue:`39402`)
 - Fixed regression in :meth:`~DataFrame.to_excel` creating corrupt files when appending (``mode="a"``) to an existing file (:issue:`39576`)
 - Fixed regression in :meth:`DataFrame.transform` failing in case of an empty DataFrame or Series (:issue:`39636`)
-- Fixed regression in :meth:`core.window.rolling.Rolling.count` where the ``min_periods`` argument would be set to ``0`` after the operation (:issue:`39554`)
+- Fixed regression in :meth:`~DataFrame.groupby` or :meth:`~DataFrame.resample` when aggregating an all-NaN or numeric object dtype column (:issue:`39329`)
+- Fixed regression in :meth:`.Rolling.count` where the ``min_periods`` argument would be set to ``0`` after the operation (:issue:`39554`)
 - Fixed regression in :func:`read_excel` that incorrectly raised when the argument ``io`` was a non-path and non-buffer and the ``engine`` argument was specified (:issue:`39528`)
--
 
 .. ---------------------------------------------------------------------------
 
@@ -36,17 +37,7 @@ Bug fixes
 
 - :func:`pandas.read_excel` error message when a specified ``sheetname`` does not exist is now uniform across engines (:issue:`39250`)
 - Fixed bug in :func:`pandas.read_excel` producing incorrect results when the engine ``openpyxl`` is used and the excel file is missing or has incorrect dimension information; the fix requires ``openpyxl`` >= 3.0.0, prior versions may still fail (:issue:`38956`, :issue:`39001`)
--
-
-.. ---------------------------------------------------------------------------
-
-.. _whatsnew_122.other:
-
-Other
-~~~~~
-
--
--
+- Fixed bug in :func:`pandas.read_excel` sometimes producing a ``DataFrame`` with trailing rows of ``np.nan`` when the engine ``openpyxl`` is used (:issue:`39181`)
 
 .. ---------------------------------------------------------------------------
 
@@ -55,4 +46,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.2.1..v1.2.2|HEAD
+.. contributors:: v1.2.1..v1.2.2

doc/source/whatsnew/v1.2.3.rst

@@ -0,0 +1,48 @@
+.. _whatsnew_123:
+
+What's new in 1.2.3 (March ??, 2021)
+------------------------------------
+
+These are the changes in pandas 1.2.3. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.other:
+
+Other
+~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.2.2..v1.2.3|HEAD

doc/source/whatsnew/v1.3.0.rst

@@ -53,8 +53,9 @@ Other enhancements
 - :meth:`DataFrame.apply` can now accept non-callable DataFrame properties as strings, e.g. ``df.apply("size")``, which was already the case for :meth:`Series.apply` (:issue:`39116`)
 - :meth:`Series.apply` can now accept list-like or dictionary-like arguments that aren't lists or dictionaries, e.g. ``ser.apply(np.array(["sum", "mean"]))``, which was already the case for :meth:`DataFrame.apply` (:issue:`39140`)
 - :meth:`DataFrame.plot.scatter` can now accept a categorical column as the argument to ``c`` (:issue:`12380`, :issue:`31357`)
-- :meth:`.Styler.set_tooltips` allows on hover tooltips to be added to styled HTML dataframes (:issue:`35643`)
+- :meth:`.Styler.set_tooltips` allows on hover tooltips to be added to styled HTML dataframes (:issue:`35643`, :issue:`21266`, :issue:`39317`)
 - :meth:`.Styler.set_tooltips_class` and :meth:`.Styler.set_table_styles` amended to optionally allow certain css-string input arguments (:issue:`39564`)
+- :meth:`.Styler.apply` now more consistently accepts ndarray function returns, i.e. in all cases for ``axis`` is ``0, 1 or None``. (:issue:`39359`)
 - :meth:`Series.loc.__getitem__` and :meth:`Series.loc.__setitem__` with :class:`MultiIndex` now raising helpful error message when indexer has too many dimensions (:issue:`35349`)
 - :meth:`pandas.read_stata` and :class:`StataReader` support reading data from compressed files.
 
@@ -254,6 +255,7 @@ Performance improvements
 - Performance improvement in :meth:`core.window.rolling.Rolling.corr` and :meth:`core.window.rolling.Rolling.cov` (:issue:`39388`)
 - Performance improvement in :meth:`core.window.rolling.RollingGroupby.corr`, :meth:`core.window.expanding.ExpandingGroupby.corr`, :meth:`core.window.expanding.ExpandingGroupby.corr` and :meth:`core.window.expanding.ExpandingGroupby.cov` (:issue:`39591`)
 - Performance improvement in :func:`unique` for object data type (:issue:`37615`)
+- Performance improvement in :class:`core.window.rolling.ExpandingGroupby` aggregation methods (:issue:`39664`)
 
 .. ---------------------------------------------------------------------------
 

environment.yml

@@ -113,5 +113,5 @@ dependencies:
   - tabulate>=0.8.3  # DataFrame.to_markdown
   - natsort  # DataFrame.sort_values
   - pip:
-    - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master
+    - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@2488b7defbd3d753dd5fcfc890fc4a7e79d25103
     - git+https://github.com/numpy/numpydoc