Merge master branch into GH13936

Author: Christopher C. Aycock

asv_bench/benchmarks/reshape.py

@@ -1,5 +1,5 @@
 from .pandas_vb_common import *
-from pandas.core.reshape import melt
+from pandas.core.reshape import melt, wide_to_long
 
 
 class melt_dataframe(object):
@@ -74,3 +74,25 @@ def setup(self):
 
     def time_unstack_sparse_keyspace(self):
         self.idf.unstack()
+
+
+class wide_to_long_big(object):
+    goal_time = 0.2
+
+    def setup(self):
+        vars = 'ABCD'
+        nyrs = 20
+        nidvars = 20
+        N = 5000
+        yrvars = []
+        for var in vars:
+            for yr in range(1, nyrs + 1):
+                yrvars.append(var + str(yr))
+
+        self.df = pd.DataFrame(np.random.randn(N, nidvars + len(yrvars)),
+                               columns=list(range(nidvars)) + yrvars)
+        self.vars = vars
+
+    def time_wide_to_long_big(self):
+        self.df['id'] = self.df.index
+        wide_to_long(self.df, list(self.vars), i='id', j='year')

doc/source/api.rst

@@ -157,6 +157,7 @@ Data manipulations
    concat
    get_dummies
    factorize
+   wide_to_long
 
 Top-level missing data
 ~~~~~~~~~~~~~~~~~~~~~~

doc/source/basics.rst

@@ -486,7 +486,9 @@ standard deviation 1), very concisely:
    xs_stand.std(1)
 
 Note that methods like :meth:`~DataFrame.cumsum` and :meth:`~DataFrame.cumprod`
-preserve the location of NA values:
+preserve the location of ``NaN`` values. This is somewhat different from
+:meth:`~DataFrame.expanding` and :meth:`~DataFrame.rolling`.
+For more details please see :ref:`this note <stats.moments.expanding.note>`.
 
 .. ipython:: python
 

doc/source/computation.rst

@@ -691,6 +691,8 @@ Method Summary
     :meth:`~Expanding.cov`, Unbiased covariance (binary)
     :meth:`~Expanding.corr`, Correlation (binary)
 
+.. currentmodule:: pandas
+
 Aside from not having a ``window`` parameter, these functions have the same
 interfaces as their ``.rolling`` counterparts. Like above, the parameters they
 all accept are:
@@ -700,18 +702,37 @@ all accept are:
   ``min_periods`` non-null data points have been seen.
 - ``center``: boolean, whether to set the labels at the center (default is False)
 
+.. _stats.moments.expanding.note:
 .. note::
 
    The output of the ``.rolling`` and ``.expanding`` methods do not return a
    ``NaN`` if there are at least ``min_periods`` non-null values in the current
-   window. This differs from ``cumsum``, ``cumprod``, ``cummax``, and
-   ``cummin``, which return ``NaN`` in the output wherever a ``NaN`` is
-   encountered in the input.
+   window. For example,
+
+   .. ipython:: python
+
+        sn = pd.Series([1, 2, np.nan, 3, np.nan, 4])
+        sn
+        sn.rolling(2).max()
+        sn.rolling(2, min_periods=1).max()
+
+   In case of expanding functions, this differs from :meth:`~DataFrame.cumsum`,
+   :meth:`~DataFrame.cumprod`, :meth:`~DataFrame.cummax`,
+   and :meth:`~DataFrame.cummin`, which return ``NaN`` in the output wherever
+   a ``NaN`` is encountered in the input. In order to match the output of ``cumsum``
+   with ``expanding``, use :meth:`~DataFrame.fillna`:
+
+   .. ipython:: python
+
+        sn.expanding().sum()
+        sn.cumsum()
+        sn.cumsum().fillna(method='ffill')
+
 
 An expanding window statistic will be more stable (and less responsive) than
 its rolling window counterpart as the increasing window size decreases the
 relative impact of an individual data point. As an example, here is the
-:meth:`~Expanding.mean` output for the previous time series dataset:
+:meth:`~core.window.Expanding.mean` output for the previous time series dataset:
 
 .. ipython:: python
    :suppress:
@@ -731,13 +752,14 @@ relative impact of an individual data point. As an example, here is the
 Exponentially Weighted Windows
 ------------------------------
 
+.. currentmodule:: pandas.core.window
+
 A related set of functions are exponentially weighted versions of several of
 the above statistics. A similar interface to ``.rolling`` and ``.expanding`` is accessed
-thru the ``.ewm`` method to receive an :class:`~pandas.core.window.EWM` object.
+through the ``.ewm`` method to receive an :class:`~EWM` object.
 A number of expanding EW (exponentially weighted)
 methods are provided:
 
-.. currentmodule:: pandas.core.window
 
 .. csv-table::
     :header: "Function", "Description"

doc/source/io.rst

@@ -867,6 +867,12 @@ data columns:
                     index_col=0) #index is the nominal column
    df
 
+.. note::
+   If a column or index contains an unparseable date, the entire column or
+   index will be returned unaltered as an object data type. For non-standard
+   datetime parsing, use :func:`to_datetime` after ``pd.read_csv``.
+
+
 .. note::
    read_csv has a fast_path for parsing datetime strings in iso8601 format,
    e.g "2000-01-01T00:01:02+00:00" and similar variations. If you can arrange

doc/source/whatsnew.rst

@@ -18,6 +18,8 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.20.0.txt
+
 .. include:: whatsnew/v0.19.2.txt
 
 .. include:: whatsnew/v0.19.1.txt

doc/source/whatsnew/v0.19.1.txt

@@ -58,4 +58,4 @@ Bug Fixes
 - Bug in ``df.groupby`` causing an ``AttributeError`` when grouping a single index frame by a column and the index level (:issue`14327`)
 - Bug in ``df.groupby`` where ``TypeError`` raised when ``pd.Grouper(key=...)`` is passed in a list (:issue:`14334`)
 - Bug in ``pd.pivot_table`` may raise ``TypeError`` or ``ValueError`` when ``index`` or ``columns``
-  is not scalar and ``values`` is not specified (:issue:`14380`)
+  is not scalar and ``values`` is not specified (:issue:`14380`)

doc/source/whatsnew/v0.19.2.txt

@@ -23,6 +23,16 @@ Performance Improvements
 
 - Improved performance of ``.replace()`` (:issue:`12745`)
 
+.. _whatsnew_0192.enhancements.other:
+
+Other enhancements
+^^^^^^^^^^^^^^^^^^
+
+- ``pd.merge_asof()`` gained ``left_index``/``right_index`` and ``left_by``/``right_by`` arguments (:issue:`14253`)
+- ``pd.merge_asof()`` can take multiple columns in ``by`` parameter and has specialized dtypes for better performace (:issue:`13936`)
+
+
+
 .. _whatsnew_0192.bug_fixes:
 
 Bug Fixes
@@ -82,11 +92,3 @@ Bug Fixes
 - Bug in ``unstack()`` if called with a list of column(s) as an argument, regardless of the dtypes of all columns, they get coerced to ``object`` (:issue:`11847`)
 
 
-.. _whatsnew_0192.enhancements.other:
-
-Other enhancements
-^^^^^^^^^^^^^^^^^^
-
-- ``pd.merge_asof()`` can take multiple columns in ``by`` parameter and has specialized dtypes for better performace (:issue:`13936`)
-
-

doc/source/whatsnew/v0.20.0.txt

@@ -52,6 +52,9 @@ Other enhancements
 
 - ``pd.read_excel`` now preserves sheet order when using ``sheetname=None`` (:issue:`9930`)
 
+
+- Multiple offset aliases with decimal points are now supported (e.g. '0.5min' is parsed as '30s') (:issue:`8419`)
+
 - New ``UnsortedIndexError`` (subclass of ``KeyError``) raised when indexing/slicing into an
   unsorted MultiIndex (:issue:`11897`). This allows differentiation between errors due to lack
   of sorting or an incorrect key. See :ref:`here <advanced.unsorted>`
@@ -61,6 +64,8 @@ Other enhancements
 - The ``usecols`` argument in ``pd.read_csv`` now accepts a callable function as a value  (:issue:`14154`)
 - ``pd.DataFrame.plot`` now prints a title above each subplot if ``suplots=True`` and ``title`` is a list of strings (:issue:`14753`)
 - ``pd.Series.interpolate`` now supports timedelta as an index type with ``method='time'`` (:issue:`6424`)
+- ``pandas.io.json.json_normalize()`` gained the option ``errors='ignore'|'raise'``; the default is ``errors='raise'`` which is backward compatible. (:issue:`14583`)
+
 
 .. _whatsnew_0200.api_breaking:
 
@@ -111,6 +116,7 @@ Removal of prior version deprecations/changes
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
+- Improved performance of ``pd.wide_to_long()`` (:issue:`14779`)
 
 
 

pandas/core/generic.py

@@ -3354,12 +3354,16 @@ def fillna(self, value=None, method=None, axis=None, inplace=False,
             return self._constructor(new_data).__finalize__(self)
 
     def ffill(self, axis=None, inplace=False, limit=None, downcast=None):
-        """Synonym for NDFrame.fillna(method='ffill')"""
+        """
+        Synonym for :meth:`DataFrame.fillna(method='ffill') <DataFrame.fillna>`
+        """
         return self.fillna(method='ffill', axis=axis, inplace=inplace,
                            limit=limit, downcast=downcast)
 
     def bfill(self, axis=None, inplace=False, limit=None, downcast=None):
-        """Synonym for NDFrame.fillna(method='bfill')"""
+        """
+        Synonym for :meth:`DataFrame.fillna(method='bfill') <DataFrame.fillna>`
+        """
         return self.fillna(method='bfill', axis=axis, inplace=inplace,
                            limit=limit, downcast=downcast)
 
@@ -5477,16 +5481,18 @@ def compound(self, axis=None, skipna=None, level=None):
 
         cls.cummin = _make_cum_function(
             cls, 'cummin', name, name2, axis_descr, "cumulative minimum",
-            lambda y, axis: np.minimum.accumulate(y, axis), np.inf, np.nan)
+            lambda y, axis: np.minimum.accumulate(y, axis), "min",
+            np.inf, np.nan)
         cls.cumsum = _make_cum_function(
             cls, 'cumsum', name, name2, axis_descr, "cumulative sum",
-            lambda y, axis: y.cumsum(axis), 0., np.nan)
+            lambda y, axis: y.cumsum(axis), "sum", 0., np.nan)
         cls.cumprod = _make_cum_function(
             cls, 'cumprod', name, name2, axis_descr, "cumulative product",
-            lambda y, axis: y.cumprod(axis), 1., np.nan)
+            lambda y, axis: y.cumprod(axis), "prod", 1., np.nan)
         cls.cummax = _make_cum_function(
             cls, 'cummax', name, name2, axis_descr, "cumulative max",
-            lambda y, axis: np.maximum.accumulate(y, axis), -np.inf, np.nan)
+            lambda y, axis: np.maximum.accumulate(y, axis), "max",
+            -np.inf, np.nan)
 
         cls.sum = _make_stat_function(
             cls, 'sum', name, name2, axis_descr,
@@ -5674,7 +5680,15 @@ def _doc_parms(cls):
 
 Returns
 -------
-%(outname)s : %(name1)s\n"""
+%(outname)s : %(name1)s\n
+
+
+See also
+--------
+pandas.core.window.Expanding.%(accum_func_name)s : Similar functionality
+    but ignores ``NaN`` values.
+
+"""
 
 
 def _make_stat_function(cls, name, name1, name2, axis_descr, desc, f):
@@ -5717,10 +5731,10 @@ def stat_func(self, axis=None, skipna=None, level=None, ddof=1,
     return set_function_name(stat_func, name, cls)
 
 
-def _make_cum_function(cls, name, name1, name2, axis_descr, desc, accum_func,
-                       mask_a, mask_b):
+def _make_cum_function(cls, name, name1, name2, axis_descr, desc,
+                       accum_func, accum_func_name, mask_a, mask_b):
     @Substitution(outname=name, desc=desc, name1=name1, name2=name2,
-                  axis_descr=axis_descr)
+                  axis_descr=axis_descr, accum_func_name=accum_func_name)
     @Appender("Return {0} over requested axis.".format(desc) +
               _cnum_doc)
     def cum_func(self, axis=None, skipna=True, *args, **kwargs):