diff --git a/.coveragerc b/.coveragerc index 3f630aa6cf8f5..f5c8b701a79a8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,7 @@ [run] branch = False omit = */tests/* +plugins = Cython.Coverage [report] # Regexes for lines to exclude from consideration @@ -22,6 +23,7 @@ exclude_lines = if __name__ == .__main__.: ignore_errors = False +show_missing = True [html] directory = coverage_html_report diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index 76dadb4ec3e23..cd3ce5c1a8f09 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -1655,8 +1655,8 @@ cdef class _Period(object): return value def __setstate__(self, state): - self.freq=state[1] - self.ordinal=state[2] + self.freq = state[1] + self.ordinal = state[2] def __reduce__(self): object_state = None, self.freq, self.ordinal diff --git a/pandas/_libs/src/util.pxd b/pandas/_libs/tslibs/util.pxd similarity index 97% rename from pandas/_libs/src/util.pxd rename to pandas/_libs/tslibs/util.pxd index 7ce2181f32553..305c4f8f908e0 100644 --- a/pandas/_libs/src/util.pxd +++ b/pandas/_libs/tslibs/util.pxd @@ -1,4 +1,4 @@ -from numpy cimport ndarray, NPY_C_CONTIGUOUS, NPY_F_CONTIGUOUS +from numpy cimport ndarray cimport numpy as cnp cnp.import_array() @@ -64,7 +64,7 @@ cdef inline bint is_datetime64_object(object obj) nogil: # -------------------------------------------------------------------- -cdef extern from "numpy_helper.h": +cdef extern from "../src/numpy_helper.h": void set_array_not_contiguous(ndarray ao) int assign_value_1d(ndarray, Py_ssize_t, object) except -1 @@ -87,7 +87,7 @@ ctypedef fused numeric: cnp.float32_t cnp.float64_t -cdef extern from "headers/stdint.h": +cdef extern from "../src/headers/stdint.h": enum: UINT8_MAX enum: UINT16_MAX enum: UINT32_MAX diff --git a/pandas/_libs/util.pxd b/pandas/_libs/util.pxd new file mode 100644 index 0000000000000..0b7e66902cbb1 --- /dev/null +++ b/pandas/_libs/util.pxd @@ -0,0 +1 @@ +from tslibs.util cimport * diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index cea77e2c88b1b..b9dd46a10dfda 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -13,7 +13,7 @@ from numpy cimport ndarray, double_t, int64_t, float64_t cnp.import_array() -cdef extern from "../src/headers/cmath" namespace "std": +cdef extern from "src/headers/cmath" namespace "std": int signbit(double) nogil double sqrt(double x) nogil diff --git a/setup.py b/setup.py index 4910fcf292ca6..85c5970af018f 100755 --- a/setup.py +++ b/setup.py @@ -40,9 +40,11 @@ def is_platform_windows(): try: import Cython ver = Cython.__version__ + from Cython.Build import cythonize _CYTHON_INSTALLED = ver >= LooseVersion(min_cython_ver) except ImportError: _CYTHON_INSTALLED = False + cythonize = lambda x, *args, **kwargs: x # dummy func # The import of Extension must be after the import of Cython, otherwise # we do not get the appropriately patched class. @@ -419,11 +421,66 @@ def get_tag(self): cmdclass['build_src'] = DummyBuildSrc cmdclass['build_ext'] = CheckingBuildExt +# ---------------------------------------------------------------------- +# Preparation of compiler arguments + if sys.byteorder == 'big': endian_macro = [('__BIG_ENDIAN__', '1')] else: endian_macro = [('__LITTLE_ENDIAN__', '1')] + +if is_platform_windows(): + extra_compile_args = [] +else: + # args to ignore warnings + extra_compile_args = ['-Wno-unused-function'] + + +# enable coverage by building cython files by setting the environment variable +# "PANDAS_CYTHON_COVERAGE" (with a Truthy value) +linetrace = os.environ.get('PANDAS_CYTHON_COVERAGE', False) +CYTHON_TRACE = str(int(bool(linetrace))) + +# Note: if not using `cythonize`, coverage can be enabled by +# pinning `ext.cython_directives = directives` to each ext in extensions. +# github.com/cython/cython/wiki/enhancements-compilerdirectives#in-setuppy +directives = {'linetrace': False} +macros = [] +if linetrace: + # https://pypkg.com/pypi/pytest-cython/f/tests/example-project/setup.py + directives['linetrace'] = True + macros = [('CYTHON_TRACE', '1'), ('CYTHON_TRACE_NOGIL', '1')] + + +# ---------------------------------------------------------------------- +# Specification of Dependencies + +# TODO: Need to check to see if e.g. `linetrace` has changed and possibly +# re-compile. +def maybe_cythonize(extensions, *args, **kwargs): + """ + Render tempita templates before calling cythonize + """ + if len(sys.argv) > 1 and 'clean' in sys.argv: + # Avoid running cythonize on `python setup.py clean` + # See https://github.com/cython/cython/issues/1495 + return extensions + + numpy_incl = pkg_resources.resource_filename('numpy', 'core/include') + # TODO: Is this really necessary here? + for ext in extensions: + if (hasattr(ext, 'include_dirs') and + numpy_incl not in ext.include_dirs): + ext.include_dirs.append(numpy_incl) + + if cython: + build_ext.render_templates(_pxifiles) + return cythonize(extensions, *args, **kwargs) + else: + return extensions + + lib_depends = ['inference'] @@ -434,23 +491,13 @@ def srcpath(name=None, suffix='.pyx', subdir='src'): if suffix == '.pyx': lib_depends = [srcpath(f, suffix='.pyx', subdir='_libs/src') for f in lib_depends] - lib_depends.append('pandas/_libs/src/util.pxd') + lib_depends.append('pandas/_libs/util.pxd') else: lib_depends = [] common_include = ['pandas/_libs/src/klib', 'pandas/_libs/src'] -def pxd(name): - return pjoin('pandas', name + '.pxd') - - -if is_platform_windows(): - extra_compile_args = [] -else: - # args to ignore warnings - extra_compile_args = ['-Wno-unused-function'] - lib_depends = lib_depends + ['pandas/_libs/src/numpy_helper.h', 'pandas/_libs/src/parse_helper.h', 'pandas/_libs/src/compat_helper.h'] @@ -466,22 +513,18 @@ def pxd(name): ext_data = { '_libs.algos': { 'pyxfile': '_libs/algos', - 'pxdfiles': ['_libs/src/util', '_libs/algos', '_libs/hashtable'], 'depends': _pxi_dep['algos']}, '_libs.groupby': { 'pyxfile': '_libs/groupby', - 'pxdfiles': ['_libs/src/util', '_libs/algos'], 'depends': _pxi_dep['groupby']}, '_libs.hashing': { 'pyxfile': '_libs/hashing'}, '_libs.hashtable': { 'pyxfile': '_libs/hashtable', - 'pxdfiles': ['_libs/hashtable', '_libs/missing', '_libs/khash'], 'depends': (['pandas/_libs/src/klib/khash_python.h'] + _pxi_dep['hashtable'])}, '_libs.index': { 'pyxfile': '_libs/index', - 'pxdfiles': ['_libs/src/util', '_libs/hashtable'], 'depends': _pxi_dep['index'], 'sources': np_datetime_sources}, '_libs.indexing': { @@ -490,21 +533,15 @@ def pxd(name): 'pyxfile': '_libs/internals'}, '_libs.interval': { 'pyxfile': '_libs/interval', - 'pxdfiles': ['_libs/hashtable'], 'depends': _pxi_dep['interval']}, '_libs.join': { 'pyxfile': '_libs/join', - 'pxdfiles': ['_libs/src/util', '_libs/hashtable'], 'depends': _pxi_dep['join']}, '_libs.lib': { 'pyxfile': '_libs/lib', - 'pxdfiles': ['_libs/src/util', - '_libs/missing', - '_libs/tslibs/conversion'], 'depends': lib_depends + tseries_depends}, '_libs.missing': { 'pyxfile': '_libs/missing', - 'pxdfiles': ['_libs/src/util'], 'depends': tseries_depends}, '_libs.parsers': { 'pyxfile': '_libs/parsers', @@ -514,12 +551,9 @@ def pxd(name): 'sources': ['pandas/_libs/src/parser/tokenizer.c', 'pandas/_libs/src/parser/io.c']}, '_libs.reduction': { - 'pyxfile': '_libs/reduction', - 'pxdfiles': ['_libs/src/util']}, + 'pyxfile': '_libs/reduction'}, '_libs.ops': { - 'pyxfile': '_libs/ops', - 'pxdfiles': ['_libs/src/util', - '_libs/missing']}, + 'pyxfile': '_libs/ops'}, '_libs.properties': { 'pyxfile': '_libs/properties', 'include': []}, @@ -534,113 +568,66 @@ def pxd(name): 'depends': _pxi_dep['sparse']}, '_libs.tslib': { 'pyxfile': '_libs/tslib', - 'pxdfiles': ['_libs/src/util', - '_libs/tslibs/conversion', - '_libs/tslibs/timedeltas', - '_libs/tslibs/timestamps', - '_libs/tslibs/timezones', - '_libs/tslibs/nattype', - '_libs/tslibs/offsets'], 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.ccalendar': { 'pyxfile': '_libs/tslibs/ccalendar'}, '_libs.tslibs.conversion': { 'pyxfile': '_libs/tslibs/conversion', - 'pxdfiles': ['_libs/src/util', - '_libs/tslibs/nattype', - '_libs/tslibs/timezones', - '_libs/tslibs/timedeltas'], 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.fields': { 'pyxfile': '_libs/tslibs/fields', - 'pxdfiles': ['_libs/tslibs/ccalendar', - '_libs/tslibs/nattype'], 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.frequencies': { - 'pyxfile': '_libs/tslibs/frequencies', - 'pxdfiles': ['_libs/src/util']}, + 'pyxfile': '_libs/tslibs/frequencies'}, '_libs.tslibs.nattype': { - 'pyxfile': '_libs/tslibs/nattype', - 'pxdfiles': ['_libs/src/util']}, + 'pyxfile': '_libs/tslibs/nattype'}, '_libs.tslibs.np_datetime': { 'pyxfile': '_libs/tslibs/np_datetime', 'depends': np_datetime_headers, 'sources': np_datetime_sources}, '_libs.tslibs.offsets': { 'pyxfile': '_libs/tslibs/offsets', - 'pxdfiles': ['_libs/src/util', - '_libs/tslibs/ccalendar', - '_libs/tslibs/conversion', - '_libs/tslibs/frequencies', - '_libs/tslibs/nattype'], 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.parsing': { - 'pyxfile': '_libs/tslibs/parsing', - 'pxdfiles': ['_libs/src/util']}, + 'pyxfile': '_libs/tslibs/parsing'}, '_libs.tslibs.period': { 'pyxfile': '_libs/tslibs/period', - 'pxdfiles': ['_libs/src/util', - '_libs/tslibs/ccalendar', - '_libs/tslibs/timedeltas', - '_libs/tslibs/timezones', - '_libs/tslibs/nattype', - '_libs/tslibs/offsets'], 'depends': tseries_depends + ['pandas/_libs/src/period_helper.h'], 'sources': np_datetime_sources + ['pandas/_libs/src/period_helper.c']}, '_libs.tslibs.resolution': { 'pyxfile': '_libs/tslibs/resolution', - 'pxdfiles': ['_libs/src/util', - '_libs/khash', - '_libs/tslibs/ccalendar', - '_libs/tslibs/frequencies', - '_libs/tslibs/timezones'], 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.strptime': { 'pyxfile': '_libs/tslibs/strptime', - 'pxdfiles': ['_libs/src/util', - '_libs/tslibs/nattype'], 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.timedeltas': { 'pyxfile': '_libs/tslibs/timedeltas', - 'pxdfiles': ['_libs/src/util', - '_libs/tslibs/nattype', - '_libs/tslibs/offsets'], 'depends': np_datetime_headers, 'sources': np_datetime_sources}, '_libs.tslibs.timestamps': { 'pyxfile': '_libs/tslibs/timestamps', - 'pxdfiles': ['_libs/src/util', - '_libs/tslibs/ccalendar', - '_libs/tslibs/conversion', - '_libs/tslibs/nattype', - '_libs/tslibs/offsets', - '_libs/tslibs/timedeltas', - '_libs/tslibs/timezones'], 'depends': tseries_depends, 'sources': np_datetime_sources}, '_libs.tslibs.timezones': { - 'pyxfile': '_libs/tslibs/timezones', - 'pxdfiles': ['_libs/src/util']}, + 'pyxfile': '_libs/tslibs/timezones'}, '_libs.testing': { 'pyxfile': '_libs/testing'}, '_libs.window': { 'pyxfile': '_libs/window', - 'pxdfiles': ['_libs/skiplist', '_libs/src/util'], 'language': 'c++', 'suffix': '.cpp'}, '_libs.writers': { - 'pyxfile': '_libs/writers', - 'pxdfiles': ['_libs/src/util']}, + 'pyxfile': '_libs/writers'}, 'io.sas._sas': { 'pyxfile': 'io/sas/sas'}, 'io.msgpack._packer': { - 'macros': endian_macro, + 'macros': endian_macro + macros, 'depends': ['pandas/_libs/src/msgpack/pack.h', 'pandas/_libs/src/msgpack/pack_template.h'], 'include': ['pandas/_libs/src/msgpack'] + common_include, @@ -652,7 +639,7 @@ def pxd(name): 'depends': ['pandas/_libs/src/msgpack/unpack.h', 'pandas/_libs/src/msgpack/unpack_define.h', 'pandas/_libs/src/msgpack/unpack_template.h'], - 'macros': endian_macro, + 'macros': endian_macro + macros, 'include': ['pandas/_libs/src/msgpack'] + common_include, 'language': 'c++', 'suffix': '.cpp', @@ -668,10 +655,6 @@ def pxd(name): sources = [srcpath(data['pyxfile'], suffix=source_suffix, subdir='')] - pxds = [pxd(x) for x in data.get('pxdfiles', [])] - if suffix == '.pyx' and pxds: - sources.extend(pxds) - sources.extend(data.get('sources', [])) include = data.get('include', common_include) @@ -681,7 +664,7 @@ def pxd(name): depends=data.get('depends', []), include_dirs=include, language=data.get('language', 'c'), - define_macros=data.get('macros', []), + define_macros=data.get('macros', macros), extra_compile_args=extra_compile_args) extensions.append(obj) @@ -708,7 +691,8 @@ def pxd(name): 'pandas/_libs/src/ujson/lib', 'pandas/_libs/src/datetime'], extra_compile_args=(['-D_GNU_SOURCE'] + - extra_compile_args)) + extra_compile_args), + define_macros=macros) extensions.append(ujson_ext) @@ -718,7 +702,8 @@ def pxd(name): # extension for pseudo-safely moving bytes into mutable buffers _move_ext = Extension('pandas.util._move', depends=[], - sources=['pandas/util/move.c']) + sources=['pandas/util/move.c'], + define_macros=macros) extensions.append(_move_ext) # The build cache system does string matching below this point. @@ -729,7 +714,7 @@ def pxd(name): version=versioneer.get_version(), packages=find_packages(include=['pandas', 'pandas.*']), package_data={'': ['templates/*', '_libs/*.dll']}, - ext_modules=extensions, + ext_modules=maybe_cythonize(extensions, compiler_directives=directives), maintainer_email=EMAIL, description=DESCRIPTION, license=LICENSE,