Support building C extensions in benchmarks

Add ctypes and ctypes_argtypes benchmarks

Author: mdboom

.gitignore

@@ -10,7 +10,7 @@
 # Created by setup.py sdist
 build/
 dist/
-pyperformance.egg-info/
+*.egg-info/
 
 # Created by the pyperformance script
 venv/

MANIFEST.in

@@ -15,5 +15,6 @@ include pyperformance/data-files/benchmarks/MANIFEST
 include pyperformance/data-files/benchmarks/bm_*/*.toml
 include pyperformance/data-files/benchmarks/bm_*/*.py
 include pyperformance/data-files/benchmarks/bm_*/requirements.txt
+include pyperformance/data-files/benchmarks/bm_*/*.c
 recursive-include pyperformance/data-files/benchmarks/bm_*/data *
 recursive-exclude pyperformance/tests *

doc/benchmarks.rst

@@ -130,6 +130,16 @@ deepcopy
 Benchmark the Python `copy.deepcopy` method. The `deepcopy` method is
 performed on a nested dictionary and a dataclass.
 
+ctypes
+------
+
+Benchmark to measure the function call overhead of calling C functions using ctypes.
+
+The ``ctypes`` benchmark lets ``ctypes`` infer the argument types from the passed in
+values. The ``ctypes_argtypes`` benchmark `explicitly specifies the argument types
+<https://docs.python.org/3.10/library/ctypes.html?highlight=ctypes#specifying-the-required-argument-types-function-prototypes>`_,
+which is slower than inferred argument types.
+
 deltablue
 ---------
 

doc/custom_benchmarks.rst

@@ -324,16 +324,17 @@ All other PEP 621 fields are optional (e.g. ``requires-python = ">=3.8"``,
 The ``[tool.pyperformance]`` Section
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-=============== ===== === === ===
-field           type  R   B   F 
-=============== ===== === === ===
-tool.name       str   X       X
-tool.tags       [str]     X    
-tool.extra_opts [str]     X    
-tool.inherits   file           
-tool.runscript  file      X    
-tool.datadir    file      X    
-=============== ===== === === ===
+================== ===== === === ===
+field              type  R   B   F
+================== ===== === === ===
+tool.name          str   X       X
+tool.tags          [str]     X
+tool.extra_opts    [str]     X
+tool.inherits      file
+tool.runscript     file      X
+tool.datadir       file      X
+tool.install_setup bool
+================== ===== === === ===
 
 "R": required
 "B": inferred from the inherited metadata
@@ -342,3 +343,6 @@ tool.datadir    file      X
 * tags: optional list of names to group benchmarks
 * extra_opts: optional list of args to pass to ``tool.runscript``
 * runscript: the benchmark script to use instead of run_benchmark.py.
+* install_setup: when ``true``, run ``pip install -e .`` in the
+  benchmark directory to install it in the virtual environment. This has the
+  effect of running a ``setup.py`` file, if present.

pyperformance/_benchmark.py

@@ -164,6 +164,13 @@ def runscript(self):
     def extra_opts(self):
         return self._get_metadata_value('extra_opts', ())
 
+    @property
+    def setup_py(self):
+        if not self._get_metadata_value('install_setup', False):
+            return None
+        filename = os.path.join(os.path.dirname(self.metafile), 'setup.py')
+        return filename if os.path.exists(filename) else None
+
     # Other metadata keys:
     # * base
     # * python

pyperformance/_benchmark_metadata.py

@@ -32,6 +32,7 @@
     'datadir': None,
     'runscript': None,
     'extra_opts': None,
+    'install_setup': None,
 }
 
 
@@ -228,6 +229,9 @@ def _resolve_value(field, value, rootdir):
         for opt in value:
             if not opt or not isinstance(opt, str):
                 raise TypeError(f'extra_opts should be a list of strings, got {value!r}')
+    elif field == 'install_setup':
+        if not isinstance(value, bool):
+            raise TypeError(f'install_setup should be a bool, got {value!r}')
     else:
         raise NotImplementedError(field)
     return value

pyperformance/_pip.py

@@ -150,7 +150,7 @@ def install_requirements(reqs, *extra,
     if upgrade:
         args.append('-U')  # --upgrade
     for reqs in [reqs, *extra]:
-        if os.path.exists(reqs):
+        if os.path.isfile(reqs):
             args.append('-r')  # --requirement
         args.append(reqs)
     return run_pip('install', *args, **kwargs)

pyperformance/data-files/benchmarks/MANIFEST

@@ -11,6 +11,8 @@ generators	<local>
 chameleon	<local>
 chaos	<local>
 crypto_pyaes	<local>
+ctypes	<local>
+ctypes_argtypes	<local:ctypes>
 deepcopy	<local>
 deltablue	<local>
 django_template	<local>
@@ -69,6 +71,7 @@ xml_etree	<local>
 #apps
 #math
 #template
+#extension
 
 
 [group default]

pyperformance/data-files/benchmarks/bm_ctypes/bm_ctypes_argtypes.toml

@@ -0,0 +1,11 @@
+[project]
+name = "pyperformance_bm_ctypes_argtypes"
+requires-python = ">=3.7"
+dependencies = ["pyperf"]
+urls = {repository = "https://github.com/python/pyperformance"}
+dynamic = ["version"]
+
+[tool.pyperformance]
+name = "ctypes_argtypes"
+tags = "extension"
+extra_opts = ["--argtypes"]

pyperformance/data-files/benchmarks/bm_ctypes/cmodule.c

@@ -0,0 +1,54 @@
+#include <Python.h>
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#define EXPORTED_SYMBOL __declspec(dllexport)
+#else
+#define EXPORTED_SYMBOL
+#endif
+
+
+EXPORTED_SYMBOL
+void void_foo_void(void) {
+
+}
+
+EXPORTED_SYMBOL
+int int_foo_int(int a) {
+    return a + 1;
+}
+
+EXPORTED_SYMBOL
+void void_foo_int(int a) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_int_int(int a, int b) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_int_int_int(int a, int b, int c) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_int_int_int_int(int a, int b, int c, int d) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_constchar(const char* str) {
+
+}
+
+PyMODINIT_FUNC
+PyInit_cmodule(void) {
+  // DELIBERATELY EMPTY
+
+  // This isn't actually a Python extension module (it's used via ctypes), so
+  // this entry point function will never be called. However, we are utilizing
+  // setuptools to build it, and on Windows, setuptools explicitly passes the
+  // flag /EXPORT:PyInit_cmodule, so it must be defined.
+  return NULL;
+}

pyperformance/data-files/benchmarks/bm_ctypes/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "pyperformance_bm_ctypes"
+requires-python = ">=3.7"
+dependencies = ["pyperf", "setuptools"]
+urls = {repository = "https://github.com/python/pyperformance"}
+dynamic = ["version"]
+
+[tool.pyperformance]
+name = "ctypes"
+tags = "extension"
+install_setup = true

pyperformance/data-files/benchmarks/bm_ctypes/requirements.txt

@@ -0,0 +1 @@
+setuptools==62.4.0

pyperformance/data-files/benchmarks/bm_ctypes/run_benchmark.py

@@ -0,0 +1,95 @@
+"""
+Test the function call overhead of ctypes.
+"""
+import pyperf
+
+
+import ctypes
+import importlib.util
+
+
+spec = importlib.util.find_spec("bm_ctypes.cmodule")
+if spec is None:
+    raise ImportError("Can't find bm_ctypes.cmodule shared object file")
+ext = ctypes.cdll.LoadLibrary(spec.origin)
+
+
+def benchmark_argtypes(loops):
+    void_foo_void = ext.void_foo_void
+    void_foo_void.argtypes = []
+    void_foo_void.restype = None
+
+    int_foo_int = ext.void_foo_int
+    int_foo_int.argtypes = [ctypes.c_int]
+    int_foo_int.restype = ctypes.c_int
+
+    void_foo_int = ext.void_foo_int
+    void_foo_int.argtypes = [ctypes.c_int]
+    void_foo_int.restype = None
+
+    void_foo_int_int = ext.void_foo_int_int
+    void_foo_int_int.argtypes = [ctypes.c_int, ctypes.c_int]
+    void_foo_int_int.restype = None
+
+    void_foo_int_int_int = ext.void_foo_int_int_int
+    void_foo_int_int_int.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
+    void_foo_int_int_int.restype = None
+
+    void_foo_int_int_int_int = ext.void_foo_int_int_int_int
+    void_foo_int_int_int_int.argtypes = [
+        ctypes.c_int,
+        ctypes.c_int,
+        ctypes.c_int,
+        ctypes.c_int,
+    ]
+    void_foo_int_int_int_int.restype = None
+
+    void_foo_constchar = ext.void_foo_constchar
+    void_foo_constchar.argtypes = [ctypes.c_char_p]
+    void_foo_constchar.restype = None
+
+    return benchmark(loops)
+
+
+def benchmark(loops):
+    void_foo_void = ext.void_foo_void
+    int_foo_int = ext.int_foo_int
+    void_foo_int = ext.void_foo_int
+    void_foo_int_int = ext.void_foo_int_int
+    void_foo_int_int_int = ext.void_foo_int_int_int
+    void_foo_int_int_int_int = ext.void_foo_int_int_int_int
+    void_foo_constchar = ext.void_foo_constchar
+
+    range_it = range(loops)
+
+    # Test calling the functions using the implied arguments mechanism
+    t0 = pyperf.perf_counter()
+
+    for _ in range_it:
+        void_foo_void()
+        int_foo_int(1)
+        void_foo_int(1)
+        void_foo_int_int(1, 2)
+        void_foo_int_int_int(1, 2, 3)
+        void_foo_int_int_int_int(1, 2, 3, 4)
+        void_foo_constchar(b"bytes")
+
+    return pyperf.perf_counter() - t0
+
+
+def add_cmdline_args(cmd, args):
+    if args.argtypes:
+        cmd.append("--argtypes")
+
+
+if __name__ == "__main__":
+    runner = pyperf.Runner(add_cmdline_args=add_cmdline_args)
+    runner.metadata["description"] = "ctypes function call overhead benchmark"
+
+    runner.argparser.add_argument("--argtypes", action="store_true")
+    options = runner.parse_args()
+
+    if options.argtypes:
+        runner.bench_time_func("ctypes_argtypes", benchmark_argtypes)
+    else:
+        runner.bench_time_func("ctypes", benchmark)

pyperformance/data-files/benchmarks/bm_ctypes/setup.py

@@ -0,0 +1,11 @@
+from setuptools import setup, Extension
+
+# Compile the C shared object containing functions to call through ctypes. It
+# isn't technically a Python C extension, but this is the easiest way to build
+# it in a cross-platform way.
+
+setup(
+    name="pyperformance_bm_ctypes",
+    ext_modules=[Extension("bm_ctypes.cmodule", sources=["cmodule.c"])],
+    package_dir={"bm_ctypes": "src"},
+)

pyperformance/data-files/benchmarks/bm_ctypes/src/__init__.py

@@ -24,6 +24,10 @@ def from_benchmarks(cls, benchmarks):
         for bench in benchmarks or ():
             filename = bench.requirements_lockfile
             self._add_from_file(filename)
+            if bench.setup_py:
+                # pip doesn't support installing a setup.py,
+                # but it does support installing from the directory it is in.
+                self._add(os.path.dirname(bench.setup_py))
         return self
 
     def __init__(self):