Modernize and improve setup.py

The current setup/CI configuration has rotted a bit, especially with
respect to the control of requirements.  This is a modernization pass to
fix that and make some improvements:

1) Various requirements are now specified directly in setup.py through
   "extras_require", so to install the dev dependencies it's sufficient
   to run `$ pip install -e .[dev]`

2) When building or installing unreleased versions of asyncpg, the dev
   version is generated automatically with setuptools_scm.

3) When cythonization is necessary, the correct version of Cython is now
   used automatically through the use of setup_requires.

Author: elprans

.ci/appveyor.yml

@@ -26,10 +26,9 @@ branches:
 
 install:
     - "%PYTHON% -m pip install --upgrade pip wheel setuptools"
-    - "%PYTHON% -m pip install --upgrade -r .ci/requirements-win.txt"
 
 build_script:
-    - "%PYTHON% setup.py build_ext --inplace"
+    - "%PYTHON% setup.py build_ext --inplace --cython-always"
 
 test_script:
     - "%PYTHON% setup.py test"
@@ -41,6 +40,7 @@ artifacts:
     - path: dist\*
 
 deploy_script:
+    - "%PYTHON% -m pip install --upgrade -r .ci/requirements-publish.txt"
     - ps: |
         if ($env:appveyor_repo_branch -eq 'releases') {
             $PACKAGE_VERSION = & "$env:PYTHON" ".ci/package-version.py"

.ci/requirements-publish.txt

@@ -0,0 +1,2 @@
+tinys3
+twine

.ci/requirements-win.txt

@@ -7,5 +7,5 @@ if [[ "${BUILD}" != *docs* ]]; then
     exit 0
 fi
 
-pip install -r docs/requirements.txt
+pip install -U .[dev]
 make htmldocs SPHINXOPTS="-q -W -j4"

.ci/requirements.txt

@@ -8,6 +8,5 @@ if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
     eval "$(pyenv init -)"
 fi
 
-pip install --upgrade pip wheel
-pip install --upgrade setuptools
-pip install --upgrade -r .ci/requirements.txt
+pip install --upgrade setuptools pip wheel
+pip install --upgrade -e .[dev]

.ci/travis-build-docs.sh

@@ -13,7 +13,7 @@ if [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then
     exit 0
 fi
 
-pip install -r docs/requirements.txt
+pip install -U .[dev]
 make htmldocs
 
 git config --global user.email "[email protected]"

.ci/travis-install.sh

@@ -7,6 +7,7 @@ if [ -z "${TRAVIS_TAG}" ]; then
     exit 0
 fi
 
+pip install -U ".ci/requirements-publish.txt"
 
 PACKAGE_VERSION=$(python ".ci/package-version.py")
 PYPI_VERSION=$(python ".ci/pypi-check.py" "${PYMODULE}")

.ci/travis-publish-docs.sh

@@ -31,3 +31,4 @@ docs/_build
 *,cover
 .coverage
 /.pytest_cache/
+/.eggs

.ci/travis-release.sh

@@ -20,11 +20,7 @@ compile:
 
 
 debug:
-	$(PYTHON) setup.py build_ext --inplace --debug \
-		--cython-always \
-		--cython-annotate \
-		--cython-directives="linetrace=True" \
-		--define ASYNCPG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL
+	ASYNCPG_DEBUG=1 $(PYTHON) setup.py build_ext --inplace
 
 
 test:

.gitignore

@@ -15,4 +15,20 @@
 __all__ = ('connect', 'create_pool', 'Record', 'Connection') + \
           exceptions.__all__  # NOQA
 
-__version__ = '0.16.0.dev1'
+# The rules of changing __version__:
+#
+#    In a release revision, __version__ must be set to 'x.y.z',
+#    and the release revision tagged with the 'vx.y.z' tag.
+#    For example, asyncpg release 0.15.0 should have
+#    __version__ set to '0.15.0', and tagged with 'v0.15.0'.
+#
+#    In between releases, __version__ must be set to
+#    'x.y+1.0.dev0', so asyncpg revisions between 0.15.0 and
+#    0.16.0 should have __version__ set to '0.16.0.dev0' in
+#    the source.
+#
+#    Source and wheel distributions built from development
+#    snapshots will automatically include the git revision
+#    in __version__, for example: '0.16.0.dev22+ge06ad03'
+
+__version__ = '0.16.0.dev0'

Makefile

@@ -29,29 +29,27 @@ If you want to build **asyncpg** from a Git checkout you will need:
   * CPython header files.  These can usually be obtained by installing
     the relevant Python development package: **python3-dev** on Debian/Ubuntu,
     **python3-devel** on RHEL/Fedora.
-  * Cython version 0.24 or later.  The easiest way to install it to use
-    virtualenv and pip, however a system package should also suffice.
-  * GNU make
 
-Once the above requirements are satisfied, run:
+Once the above requirements are satisfied, run the following command
+in the root of the source checkout:
 
 .. code-block:: bash
 
-    $ make
+    $ pip install -e .
 
-At this point you can run the usual ``setup.py`` commands or
-``pip install -e .`` to install the newly built version.
+A debug build containing more runtime checks can be created by setting
+the ``ASYNCPG_DEBUG`` environment variable when building:
 
-.. note::
+.. code-block:: bash
 
-   A debug build can be created by running ``make debug``.
+    $ env ASYNCPG_DEBUG=1 pip install -e .
 
 
 Running tests
 -------------
 
-To execute the testsuite simply run:
+To execute the testsuite run:
 
 .. code-block:: bash
 
-    $ make test
+    $ python setup.py test

asyncpg/__init__.py

@@ -5,17 +5,43 @@
 # the Apache 2.0 License: http://www.apache.org/licenses/LICENSE-2.0
 
 
+import sys
+
+if sys.version_info < (3, 5):
+    raise RuntimeError('asyncpg requires Python 3.5 or greater')
+
 import os
 import os.path
+import pathlib
 import platform
-import sys
+import re
+
+# We use vanilla build_ext, to avoid importing Cython via
+# the setuptools version.
+from distutils import extension as distutils_extension
+from distutils.command import build_ext as distutils_build_ext
 
 import setuptools
-from setuptools.command import build_ext as _build_ext
+from setuptools.command import build_py as setuptools_build_py
+from setuptools.command import sdist as setuptools_sdist
 
 
-if sys.version_info < (3, 5):
-    raise RuntimeError('asyncpg requires Python 3.5 or greater')
+CYTHON_DEPENDENCY = 'Cython==0.28.3'
+
+EXTRA_DEPENDENCIES = {
+    # Dependencies required to develop asyncpg.
+    'dev': [
+        CYTHON_DEPENDENCY,
+        'flake8~=3.5.0',
+        'pytest~=3.0.7',
+        'uvloop>=0.8.0;platform_system!="Windows"',
+        # Docs
+        'Sphinx~=1.7.3',
+        'sphinxcontrib-asyncio~=0.2.0',
+        'sphinx_rtd_theme~=0.2.4',
+    ]
+}
+
 
 CFLAGS = ['-O2']
 LDFLAGS = []
@@ -24,8 +50,44 @@
     CFLAGS.extend(['-fsigned-char', '-Wall', '-Wsign-compare', '-Wconversion'])
 
 
-class build_ext(_build_ext.build_ext):
-    user_options = _build_ext.build_ext.user_options + [
+class VersionMixin:
+
+    def _fix_version(self, filename):
+        # Replace asyncpg.__version__ with the actual version
+        # of the distribution (possibly inferred from git).
+
+        with open(str(filename)) as f:
+            content = f.read()
+
+        version_re = r"(.*__version__\s*=\s*)'[^']+'(.*)"
+        repl = r"\1'{}'\2".format(self.distribution.metadata.version)
+        content = re.sub(version_re, repl, content)
+
+        with open(str(filename), 'w') as f:
+            f.write(content)
+
+
+class sdist(setuptools_sdist.sdist, VersionMixin):
+
+    def make_release_tree(self, base_dir, files):
+        super().make_release_tree(base_dir, files)
+        self._fix_version(pathlib.Path(base_dir) / 'asyncpg' / '__init__.py')
+
+
+class build_py(setuptools_build_py.build_py, VersionMixin):
+
+    def build_module(self, module, module_file, package):
+        outfile, copied = super().build_module(module, module_file, package)
+
+        if module == '__init__' and package == 'asyncpg':
+            self._fix_version(outfile)
+
+        return outfile, copied
+
+
+class build_ext(distutils_build_ext.build_ext):
+
+    user_options = distutils_build_ext.build_ext.user_options + [
         ('cython-always', None,
             'run cythonize() even if .c files are present'),
         ('cython-annotate', None,
@@ -42,9 +104,17 @@ def initialize_options(self):
             return
 
         super(build_ext, self).initialize_options()
-        self.cython_always = False
-        self.cython_annotate = None
-        self.cython_directives = None
+
+        if os.environ.get('ASYNCPG_DEBUG'):
+            self.cython_always = True
+            self.cython_annotate = True
+            self.cython_directives = "linetrace=True"
+            self.define = 'ASYNCPG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL'
+            self.debug = True
+        else:
+            self.cython_always = False
+            self.cython_annotate = None
+            self.cython_directives = None
 
     def finalize_options(self):
         # finalize_options() may be called multiple times on the
@@ -72,15 +142,25 @@ def finalize_options(self):
                         need_cythonize = True
 
         if need_cythonize:
+            import pkg_resources
+
+            # Double check Cython presence in case setup_requires
+            # didn't go into effect (most likely because someone
+            # imported Cython before setup_requires injected the
+            # correct egg into sys.path.
             try:
                 import Cython
             except ImportError:
                 raise RuntimeError(
-                    'please install Cython to compile asyncpg from source')
+                    'please install {} to compile asyncpg from source'.format(
+                        CYTHON_DEPENDENCY))
 
-            if Cython.__version__ < '0.24':
+            cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY)
+            if Cython.__version__ not in cython_dep:
                 raise RuntimeError(
-                    'asyncpg requires Cython version 0.24 or greater')
+                    'asyncpg requires {}, got Cython=={}'.format(
+                        CYTHON_DEPENDENCY, Cython.__version__
+                    ))
 
             from Cython.Build import cythonize
 
@@ -103,12 +183,14 @@ def finalize_options(self):
         super(build_ext, self).finalize_options()
 
 
-with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f:
+_ROOT = pathlib.Path(__file__).parent
+
+
+with open(str(_ROOT / 'README.rst')) as f:
     readme = f.read()
 
 
-with open(os.path.join(
-        os.path.dirname(__file__), 'asyncpg', '__init__.py')) as f:
+with open(str(_ROOT / 'asyncpg' / '__init__.py')) as f:
     for line in f:
         if line.startswith('__version__ ='):
             _, _, version = line.partition('=')
@@ -119,6 +201,41 @@ def finalize_options(self):
             'unable to read the version from asyncpg/__init__.py')
 
 
+extra_setup_kwargs = {}
+setup_requires = []
+
+if (_ROOT / '.git').is_dir():
+    # This is a git checkout, use setuptools_scm to
+    # generage a precise version.
+    def version_scheme(v):
+        from setuptools_scm import version
+
+        if v.exact:
+            fv = v.format_with("{tag}")
+            if fv != VERSION:
+                raise RuntimeError(
+                    'asyncpg.__version__ does not match the git tag')
+        else:
+            fv = v.format_next_version(
+                version.guess_next_simple_semver,
+                retain=version.SEMVER_MINOR)
+
+            if not fv.startswith(VERSION[:-1]):
+                raise RuntimeError(
+                    'asyncpg.__version__ does not match the git tag')
+
+        return fv
+
+    setup_requires.append('setuptools_scm')
+    extra_setup_kwargs['use_scm_version'] = {
+        'version_scheme': version_scheme
+    }
+
+if not (_ROOT / 'asyncpg' / 'protocol' / 'protocol.c').exists():
+    # No Cython output, require Cython to build.
+    setup_requires.append(CYTHON_DEPENDENCY)
+
+
 setuptools.setup(
     name='asyncpg',
     version=VERSION,
@@ -145,13 +262,16 @@ def finalize_options(self):
     provides=['asyncpg'],
     include_package_data=True,
     ext_modules=[
-        setuptools.Extension(
+        distutils_extension.Extension(
             "asyncpg.protocol.protocol",
             ["asyncpg/protocol/record/recordobj.c",
              "asyncpg/protocol/protocol.pyx"],
             extra_compile_args=CFLAGS,
             extra_link_args=LDFLAGS)
     ],
-    cmdclass={'build_ext': build_ext},
+    cmdclass={'build_ext': build_ext, 'build_py': build_py, 'sdist': sdist},
     test_suite='tests.suite',
+    extras_require=EXTRA_DEPENDENCIES,
+    setup_requires=setup_requires,
+    **extra_setup_kwargs
 )