ENH: Provide dict object for to_dict() #16122

Author: dwkenefick

asv_bench/benchmarks/algorithms.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from pandas.util import testing as tm
 
-for imp in ['pandas.util.hashing', 'pandas.tools.hashing']:
+for imp in ['pandas.util', 'pandas.tools.hashing']:
     try:
         hashing = import_module(imp)
         break

asv_bench/benchmarks/attrs_caching.py

@@ -1,5 +1,9 @@
 from .pandas_vb_common import *
-from pandas.util.decorators import cache_readonly
+
+try:
+    from pandas.util import cache_readonly
+except ImportError:
+    from pandas.util.decorators import cache_readonly
 
 
 class DataFrameAttributes(object):

ci/requirements-3.6_WIN.run

@@ -1,6 +1,7 @@
 python-dateutil
 pytz
 numpy=1.12*
+bottleneck
 openpyxl
 xlsxwriter
 xlrd

doc/source/api.rst

@@ -618,7 +618,6 @@ strings and apply several methods to it. These can be accessed like
        Series.cat
        Series.dt
        Index.str
-       CategoricalIndex.str
        MultiIndex.str
        DatetimeIndex.str
        TimedeltaIndex.str
@@ -1404,6 +1403,7 @@ CategoricalIndex
 
 .. autosummary::
    :toctree: generated/
+   :template: autosummary/class_without_autosummary.rst
 
    CategoricalIndex
 
@@ -1432,6 +1432,7 @@ IntervalIndex
 
 .. autosummary::
    :toctree: generated/
+   :template: autosummary/class_without_autosummary.rst
 
    IntervalIndex
 

doc/source/contributing.rst

@@ -617,11 +617,11 @@ the expected correct result::
 Transitioning to ``pytest``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-*pandas* existing test structure is *mostly* classed based, meaning that you will typically find tests wrapped in a class, inheriting from ``tm.TestCase``.
+*pandas* existing test structure is *mostly* classed based, meaning that you will typically find tests wrapped in a class.
 
 .. code-block:: python
 
-    class TestReallyCoolFeature(tm.TestCase):
+    class TestReallyCoolFeature(object):
         ....
 
 Going forward, we are moving to a more *functional* style using the `pytest <http://doc.pytest.org/en/latest/>`__ framework, which offers a richer testing
@@ -632,14 +632,6 @@ framework that will facilitate testing and developing. Thus, instead of writing
     def test_really_cool_feature():
         ....
 
-Sometimes, it does make sense to bundle test functions together into a single class, either because the test file is testing multiple functions from a single module, and
-using test classes allows for better organization. However, instead of inheriting from ``tm.TestCase``, we should just inherit from ``object``:
-
-.. code-block:: python
-
-    class TestReallyCoolFeature(object):
-        ....
-
 Using ``pytest``
 ~~~~~~~~~~~~~~~~
 

doc/source/merging.rst

@@ -13,7 +13,7 @@
 
    import matplotlib.pyplot as plt
    plt.close('all')
-   import pandas.util.doctools as doctools
+   import pandas.util._doctools as doctools
    p = doctools.TablePlotter()
 
 

doc/source/whatsnew/v0.20.0.txt

@@ -18,6 +18,7 @@ Highlights include:
 
 Enhancements
 ~~~~~~~~~~~~
+- ``Series.to_dict()`` and ``DataFrame.to_dict()`` now support an ``into`` keyword which allows you to specify the ``collections.Mapping`` subclass that you would like returned.  The default is ``dict``, which is backwards compatible. (:issue:`16122`)
 
 
 

doc/source/whatsnew/v0.20.1.txt

@@ -0,0 +1,113 @@
+.. _whatsnew_0210:
+
+v0.21.0 (???)
+-------------
+
+This is a major release from 0.20.x and includes a number of API changes, deprecations, new features,
+enhancements, and performance improvements along with a large number of bug fixes. We recommend that all
+users upgrade to this version.
+
+Highlights include:
+
+Check the :ref:`API Changes <whatsnew_0210.api_breaking>` and :ref:`deprecations <whatsnew_0210.deprecations>` before updating.
+
+.. contents:: What's new in v0.21.0
+    :local:
+    :backlinks: none
+
+.. _whatsnew_0210.enhancements:
+
+New features
+~~~~~~~~~~~~
+
+
+
+.. _whatsnew_0210.enhancements.other:
+
+Other Enhancements
+^^^^^^^^^^^^^^^^^^
+
+
+
+.. _whatsnew_0210.api_breaking:
+
+Backwards incompatible API changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+
+.. _whatsnew_0210.api:
+
+Other API Changes
+^^^^^^^^^^^^^^^^^
+
+
+
+.. _whatsnew_0210.deprecations:
+
+Deprecations
+~~~~~~~~~~~~
+
+
+
+.. _whatsnew_0210.prior_deprecations:
+
+Removal of prior version deprecations/changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+
+.. _whatsnew_0210.performance:
+
+Performance Improvements
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+
+.. _whatsnew_0210.bug_fixes:
+
+Bug Fixes
+~~~~~~~~~
+
+Conversion
+^^^^^^^^^^
+
+
+
+Indexing
+^^^^^^^^
+
+
+
+I/O
+^^^
+
+
+
+Plotting
+^^^^^^^^
+
+
+
+Groupby/Resample/Rolling
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+
+Sparse
+^^^^^^
+
+
+
+Reshaping
+^^^^^^^^^
+
+
+
+Numeric
+^^^^^^^
+
+
+
+Other
+^^^^^

doc/source/whatsnew/v0.21.0.txt

@@ -43,7 +43,9 @@ def mangle_docstrings(app, what, name, obj, options, lines,
               )
 
     # PANDAS HACK (to remove the list of methods/attributes for Categorical)
-    if what == "class" and name.endswith(".Categorical"):
+    if what == "class" and (name.endswith(".Categorical") or
+                            name.endswith("CategoricalIndex") or
+                            name.endswith("IntervalIndex")):
         cfg['class_members_list'] = False
 
     if what == 'module':

doc/sphinxext/numpydoc/numpydoc.py

@@ -50,17 +50,17 @@
 import pandas.tools.plotting
 plot_params = pandas.plotting._style._Options(deprecated=True)
 # do not import deprecate to top namespace
-scatter_matrix = pandas.util.decorators.deprecate(
+scatter_matrix = pandas.util._decorators.deprecate(
     'pandas.scatter_matrix', pandas.plotting.scatter_matrix,
     'pandas.plotting.scatter_matrix')
 
-from pandas.util.print_versions import show_versions
+from pandas.util._print_versions import show_versions
 from pandas.io.api import *
 from pandas.util._tester import test
 import pandas.testing
 
 # extension module deprecations
-from pandas.util.depr_module import _DeprecatedModule
+from pandas.util._depr_module import _DeprecatedModule
 
 json = _DeprecatedModule(deprmod='pandas.json',
                          moved={'dumps': 'pandas.io.json.dumps',

pandas/__init__.py

@@ -90,14 +90,14 @@ static struct PyModuleDef moduledef = {
     NULL          /* m_free */
 };
 
-#define PYMODINITFUNC PyMODINIT_FUNC PyInit_libjson(void)
+#define PYMODINITFUNC PyMODINIT_FUNC PyInit_json(void)
 #define PYMODULE_CREATE() PyModule_Create(&moduledef)
 #define MODINITERROR return NULL
 
 #else
 
-#define PYMODINITFUNC PyMODINIT_FUNC initlibjson(void)
-#define PYMODULE_CREATE() Py_InitModule("libjson", ujsonMethods)
+#define PYMODINITFUNC PyMODINIT_FUNC initjson(void)
+#define PYMODULE_CREATE() Py_InitModule("json", ujsonMethods)
 #define MODINITERROR return
 
 #endif

pandas/util/hashing.pyx renamed to pandas/_libs/hashing.pyx

@@ -19,8 +19,8 @@
 """
 
 from numpy import ndarray
-from pandas.util.validators import (validate_args, validate_kwargs,
-                                    validate_args_and_kwargs)
+from pandas.util._validators import (validate_args, validate_kwargs,
+                                     validate_args_and_kwargs)
 from pandas.errors import UnsupportedFunctionCall
 from pandas.core.dtypes.common import is_integer, is_bool
 from pandas.compat import OrderedDict

pandas/io/parsers.pyx renamed to pandas/_libs/parsers.pyx

@@ -71,7 +71,7 @@ def load_reduce(self):
 
     # 12588, extensions moving
     ('pandas._sparse', 'BlockIndex'):
-        ('pandas.core.sparse.libsparse', 'BlockIndex'),
+        ('pandas._libs.sparse', 'BlockIndex'),
     ('pandas.tslib', 'Timestamp'):
         ('pandas._libs.tslib', 'Timestamp'),
     ('pandas.tslib', '__nat_unpickle'):

pandas/core/sparse/sparse.pyx renamed to pandas/_libs/sparse.pyx

@@ -25,6 +25,13 @@ def pytest_runtest_setup(item):
         pytest.skip("skipping due to --skip-network")
 
 
+# Configurations for all tests and all test modules
+
+@pytest.fixture(autouse=True)
+def configure_tests():
+    pandas.set_option('chained_assignment', 'raise')
+
+
 # For running doctests: make np and pd names available
 
 @pytest.fixture(autouse=True)

pandas/core/sparse/sparse_op_helper.pxi.in renamed to pandas/_libs/sparse_op_helper.pxi.in

@@ -35,7 +35,7 @@
 from pandas.core.resample import TimeGrouper
 
 # see gh-14094.
-from pandas.util.depr_module import _DeprecatedModule
+from pandas.util._depr_module import _DeprecatedModule
 
 _removals = ['day', 'bday', 'businessDay', 'cday', 'customBusinessDay',
              'customBusinessMonthEnd', 'customBusinessMonthBegin',

pandas/_libs/src/ujson/python/ujson.c

@@ -9,14 +9,14 @@
 from pandas.core.dtypes.missing import isnull
 from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCIndexClass
 from pandas.core.dtypes.common import is_object_dtype, is_list_like, is_scalar
-from pandas.util.validators import validate_bool_kwarg
+from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core import common as com
 import pandas.core.nanops as nanops
 import pandas._libs.lib as lib
 from pandas.compat.numpy import function as nv
-from pandas.util.decorators import (Appender, cache_readonly,
-                                    deprecate_kwarg, Substitution)
+from pandas.util._decorators import (Appender, cache_readonly,
+                                     deprecate_kwarg, Substitution)
 from pandas.core.common import AbstractMethodError
 
 _shared_docs = dict()

pandas/util/testing.pyx renamed to pandas/_libs/testing.pyx

@@ -34,11 +34,11 @@
 import pandas.core.common as com
 from pandas.core.missing import interpolate_2d
 from pandas.compat.numpy import function as nv
-from pandas.util.decorators import (Appender, cache_readonly,
-                                    deprecate_kwarg, Substitution)
+from pandas.util._decorators import (Appender, cache_readonly,
+                                     deprecate_kwarg, Substitution)
 
-from pandas.util.terminal import get_terminal_size
-from pandas.util.validators import validate_bool_kwarg
+from pandas.io.formats.terminal import get_terminal_size
+from pandas.util._validators import validate_bool_kwarg
 from pandas.core.config import get_option
 
 

pandas/core/window.pyx renamed to pandas/_libs/window.pyx

@@ -6,6 +6,8 @@
 import warnings
 from datetime import datetime, timedelta
 from functools import partial
+import inspect
+import collections
 
 import numpy as np
 from pandas._libs import lib, tslib
@@ -479,6 +481,43 @@ def _dict_compat(d):
                 for key, value in iteritems(d))
 
 
+def _standardize_mapping(into):
+    """
+    Helper function to standardize the supplied mapping so it can 
+    be passed to the ``Series.to_dict()`` and ``DataFrame.to_dict()``
+
+    Parameters
+    ----------
+    into : instance or subclass of collections.Mapping
+        The argument supplied to ``to_dict``.  Must be a class, an
+        initialized collections.defaultdict, or an empty instance
+        of a collections.Mapping subclass.  
+
+    Returns
+    -------
+    mapping : a collections.Mapping subclass or other constructor
+        a callable object that can accept an iterator to create
+        the desired Mapping.
+        
+    """
+    if not inspect.isclass(into):
+        if len(into) > 0:
+            raise ValueError(
+                "to_dict() only accepts empty mappings.")
+        elif type(into) == collections.defaultdict:
+            return partial(
+                collections.defaultdict, into.default_factory)
+        else:
+            return _standardize_mapping(type(into))
+    elif not issubclass(into, collections.Mapping):
+        raise TypeError('unsupported type: {}'.format(into))
+    elif into == collections.defaultdict:
+        raise TypeError(
+            'to_dict() only accepts initialized defaultdicts')
+    else:
+        return into
+
+
 def sentinel_factory():
     class Sentinel(object):
         pass