From 2260bcdcddd8ef81191f8757a03affa84c212667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Fri, 3 Jan 2020 14:40:32 +0000 Subject: [PATCH 01/18] implement switching using the Stackman submodule --- .gitmodules | 3 ++ Include/internal/slp_platformselect.h | 11 +++++ Include/internal/slp_switch_stackman.h | 66 ++++++++++++++++++++++++++ Include/internal/stackman | 1 + 4 files changed, 81 insertions(+) create mode 100644 .gitmodules create mode 100644 Include/internal/slp_switch_stackman.h create mode 160000 Include/internal/stackman diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000000..0ce920fc703a61 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "Include/internal/stackman"] + path = Include/internal/stackman + url = ssh://git@github.com/kristjanvalur/stackman diff --git a/Include/internal/slp_platformselect.h b/Include/internal/slp_platformselect.h index 9bc3ae4c13a2f0..f917f9bbd0a98c 100644 --- a/Include/internal/slp_platformselect.h +++ b/Include/internal/slp_platformselect.h @@ -5,6 +5,16 @@ * Platform Selection for Stackless */ +/* First, see if stackman an implementation without external + * assembler, use that if possible + */ +#define STACKMAN_OPTIONAL +#include "stackman/src/stackman.h" +#if defined(STACKMAN_PLATFORM) && !defined(STACKMAN_EXTERNAL_ASM) +#include "internal/slp_switch_stackman.h" + +#else /* use traditional stackless switching */ + #if defined(MS_WIN32) && !defined(MS_WIN64) && defined(_M_IX86) #include "internal/slp_switch_x86_msvc.h" /* MS Visual Studio on X86 */ #elif defined(MS_WIN64) && defined(_M_X64) @@ -32,6 +42,7 @@ #elif defined(SN_TARGET_PS3) #include "internal/slp_switch_ps3_SNTools.h" /* Sony PS3 */ #endif +#endif /* default definitions if not defined in above files */ diff --git a/Include/internal/slp_switch_stackman.h b/Include/internal/slp_switch_stackman.h new file mode 100644 index 00000000000000..09adb145c07b7f --- /dev/null +++ b/Include/internal/slp_switch_stackman.h @@ -0,0 +1,66 @@ +/* + * this is the internal transfer function, using + * the stackman platform library. + * We create a wrapper callback that employs the existing + * stack macros. + * At some later point in time, the callback could be + * written directly. + * + */ + +#define SLP_STACK_REFPLUS 1 + +#ifdef SLP_EVAL +#define SLP_STACK_MAGIC 0 + + +/* need a special function arount SLP_SAVE_STATE() because + * the macro has a built-in return of 0 or -1. Must catch + * that. + */ +static int slp_stackman_cb_save(void *sp, intptr_t *pdiff) +{ + intptr_t diff; + SLP_SAVE_STATE(sp, diff); + *pdiff = diff; + return 1; +} + +static void *slp_stackman_cb(void *_ctxt, int opcode, void *sp) +{ + int *error = (int*)_ctxt; + intptr_t stsizediff; + if (opcode == STACKMAN_OP_SAVE) + { + int ret = slp_stackman_cb_save(sp, &stsizediff); + if (ret == 1) { + /* regular switch */ + return (void*)((char*)sp + stsizediff); + } + if (ret == -1) + { + *error = -1; + } + /* error or save only, no change in sp */ + return sp; + } + else + { + if (*error != -1) + SLP_RESTORE_STATE(); + return sp; + } +} + +static int +slp_switch(void) +{ + /* this can be on the stack, because if there isn't a switch + * then it is intact. (error or save only) */ + int error = 0; + stackman_switch(&slp_stackman_cb, &error); + return error; +} + +#include "stackman/src/stackman_impl.h" +#endif diff --git a/Include/internal/stackman b/Include/internal/stackman new file mode 160000 index 00000000000000..eba3344e6df7bf --- /dev/null +++ b/Include/internal/stackman @@ -0,0 +1 @@ +Subproject commit eba3344e6df7bfb1028ae9c71f36e832892798ee From 55378475d013c84c82a2a91858a698891377d07f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Mon, 6 Jan 2020 12:18:18 +0000 Subject: [PATCH 02/18] update stackman for correct ARM code --- Include/internal/stackman | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/stackman b/Include/internal/stackman index eba3344e6df7bf..261a57b11e02b1 160000 --- a/Include/internal/stackman +++ b/Include/internal/stackman @@ -1 +1 @@ -Subproject commit eba3344e6df7bfb1028ae9c71f36e832892798ee +Subproject commit 261a57b11e02b1e32c4b11b412c3f9da620284b3 From e4e9cc8ff1028bfd1fe53569d05c173de421c816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Sun, 12 Jan 2020 15:08:29 +0000 Subject: [PATCH 03/18] change submodule to https --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 0ce920fc703a61..0b194e21364f9f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "Include/internal/stackman"] path = Include/internal/stackman - url = ssh://git@github.com/kristjanvalur/stackman + url = https://github.com/kristjanvalur/stackman From 8f36251161bf44a26cc5a67be1e9bfe72d4041f2 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Wed, 23 Jun 2021 08:34:52 +0200 Subject: [PATCH 04/18] Use stackman from https://github.com/stackless-dev/stackman.git Use the official repository. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 0b194e21364f9f..b045f96c6faf3f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "Include/internal/stackman"] path = Include/internal/stackman - url = https://github.com/kristjanvalur/stackman + url = https://github.com/stackless-dev/stackman.git From 8bac0cd926608ca2de160373f829622a515530c3 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Wed, 23 Jun 2021 09:08:05 +0200 Subject: [PATCH 05/18] Update submodule stackman to the latest version --- Include/internal/stackman | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/stackman b/Include/internal/stackman index 261a57b11e02b1..65d6d81b8c83ab 160000 --- a/Include/internal/stackman +++ b/Include/internal/stackman @@ -1 +1 @@ -Subproject commit 261a57b11e02b1e32c4b11b412c3f9da620284b3 +Subproject commit 65d6d81b8c83ab65a1412d7d8a2f8c87fd537f7c From d7704ee8bb0a53c25ff3f2897ab4f828db4afc80 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Wed, 23 Jun 2021 09:30:53 +0200 Subject: [PATCH 06/18] Fix #include statements The outcome does compile, but fails to link. --- Include/internal/pycore_slp_platformselect.h | 2 +- Include/internal/slp_switch_stackman.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_slp_platformselect.h b/Include/internal/pycore_slp_platformselect.h index ed24552b831115..7e170475b20c5b 100644 --- a/Include/internal/pycore_slp_platformselect.h +++ b/Include/internal/pycore_slp_platformselect.h @@ -9,7 +9,7 @@ * assembler, use that if possible */ #define STACKMAN_OPTIONAL -#include "stackman/src/stackman.h" +#include "stackman/stackman/stackman.h" #if defined(STACKMAN_PLATFORM) && !defined(STACKMAN_EXTERNAL_ASM) #include "internal/slp_switch_stackman.h" diff --git a/Include/internal/slp_switch_stackman.h b/Include/internal/slp_switch_stackman.h index 09adb145c07b7f..ce51239fcbfe9e 100644 --- a/Include/internal/slp_switch_stackman.h +++ b/Include/internal/slp_switch_stackman.h @@ -62,5 +62,5 @@ slp_switch(void) return error; } -#include "stackman/src/stackman_impl.h" +#include "stackman/stackman/stackman_impl.h" #endif From af23038e87c8d972c68d13817d831b3a84f692b0 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Fri, 9 Jul 2021 16:48:56 +0200 Subject: [PATCH 07/18] prepare for a merge with main-slp --- .gitmodules | 2 +- Include/internal/stackman | 1 - {Include/internal => Stackless/platf}/slp_switch_stackman.h | 0 3 files changed, 1 insertion(+), 2 deletions(-) delete mode 160000 Include/internal/stackman rename {Include/internal => Stackless/platf}/slp_switch_stackman.h (100%) diff --git a/.gitmodules b/.gitmodules index b045f96c6faf3f..21b0635c56906d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "Include/internal/stackman"] - path = Include/internal/stackman + path = Stackless/platf/stackman url = https://github.com/stackless-dev/stackman.git diff --git a/Include/internal/stackman b/Include/internal/stackman deleted file mode 160000 index 65d6d81b8c83ab..00000000000000 --- a/Include/internal/stackman +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 65d6d81b8c83ab65a1412d7d8a2f8c87fd537f7c diff --git a/Include/internal/slp_switch_stackman.h b/Stackless/platf/slp_switch_stackman.h similarity index 100% rename from Include/internal/slp_switch_stackman.h rename to Stackless/platf/slp_switch_stackman.h From 7d85c372fe7b0e461314168b2426a557b7df4b81 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Fri, 9 Jul 2021 17:17:42 +0200 Subject: [PATCH 08/18] Readd the stackman submodule --- Stackless/platf/stackman | 1 + 1 file changed, 1 insertion(+) create mode 160000 Stackless/platf/stackman diff --git a/Stackless/platf/stackman b/Stackless/platf/stackman new file mode 160000 index 00000000000000..63bb64b37d7720 --- /dev/null +++ b/Stackless/platf/stackman @@ -0,0 +1 @@ +Subproject commit 63bb64b37d7720230e6579e3a266e62a1f66fe31 From e98117930caacfed1a337af50787064aa08a0eb4 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Fri, 9 Jul 2021 17:40:42 +0200 Subject: [PATCH 09/18] Adapt the patch to upstream changes. --- Stackless/platf/slp_transfer.c | 9 +++++++++ .../platf/{slp_switch_stackman.h => switch_stackman.h} | 0 2 files changed, 9 insertions(+) rename Stackless/platf/{slp_switch_stackman.h => switch_stackman.h} (100%) diff --git a/Stackless/platf/slp_transfer.c b/Stackless/platf/slp_transfer.c index 522d602d4f4d34..ba5e2acb2578d4 100644 --- a/Stackless/platf/slp_transfer.c +++ b/Stackless/platf/slp_transfer.c @@ -39,6 +39,14 @@ */ #define SLP_EVAL /* enable code generation in the included header */ +/* First, see if stackman an implementation without external + * assembler, use that if possible + */ +#define STACKMAN_OPTIONAL +#include "stackman/stackman/stackman.h" +#if defined(STACKMAN_PLATFORM) && !defined(STACKMAN_EXTERNAL_ASM) +#include "switch_stackman.h" +#else /* use traditional stackless switching */ #if defined(MS_WIN32) && !defined(MS_WIN64) && defined(_M_IX86) #include "switch_x86_msvc.h" /* MS Visual Studio on X86 */ #elif defined(MS_WIN64) && defined(_M_X64) @@ -74,6 +82,7 @@ Please provide an implementation of the switch_XXX.h or disable the STACKLESS flag. ********** #endif +#endif /* use traditional stackless switching */ /* default definitions if not defined in above files */ diff --git a/Stackless/platf/slp_switch_stackman.h b/Stackless/platf/switch_stackman.h similarity index 100% rename from Stackless/platf/slp_switch_stackman.h rename to Stackless/platf/switch_stackman.h From 7e1bb4aaf8d911f2092ae9f480862df3066cf8c4 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Fri, 9 Jul 2021 18:25:05 +0200 Subject: [PATCH 10/18] Change "make clean" to not remove static library files from stackman. --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 851a5707625c23..4d0efd0461e010 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1790,7 +1790,7 @@ docclean: -rm -rf Doc/tools/sphinx Doc/tools/pygments Doc/tools/docutils clean: pycremoval - find . -name '*.[oa]' -exec rm -f {} ';' + find . -path '*Stackless/platf/stackman/*.a' -prune -o -name '*.[oa]' -exec rm -f {} ';' find . -name '*.s[ol]' -exec rm -f {} ';' find . -name '*.so.[0-9]*.[0-9]*' -exec rm -f {} ';' find build -name 'fficonfig.h' -exec rm -f {} ';' || true From 55e8946faf0be620899628cf463a25fc89b07e0b Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sun, 11 Jul 2021 20:44:17 +0200 Subject: [PATCH 11/18] Add configure options --with-stackman=dir and --without-stackman We need a way to force a build without stackman. Using the --with-MODULE mechanism of autoconf we can even configure the location of the Stackman directory. Move the Stackman directory to Stackless/stackman. --- .gitmodules | 2 +- Makefile.pre.in | 2 +- Stackless/platf/slp_transfer.c | 4 +- Stackless/platf/stackman | 1 - Stackless/platf/switch_stackman.h | 2 +- Stackless/stackman | 1 + configure | 63 +++++++++++++++++++++++++------ configure.ac | 52 +++++++++++++++++++------ 8 files changed, 99 insertions(+), 28 deletions(-) delete mode 160000 Stackless/platf/stackman create mode 160000 Stackless/stackman diff --git a/.gitmodules b/.gitmodules index 581162f60979a9..ff4bbbc21e7757 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "Stackless/platf/stackman"] - path = Stackless/platf/stackman + path = Stackless/stackman url = https://github.com/stackless-dev/stackman.git diff --git a/Makefile.pre.in b/Makefile.pre.in index 4d0efd0461e010..b5074f5cbc2360 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1790,7 +1790,7 @@ docclean: -rm -rf Doc/tools/sphinx Doc/tools/pygments Doc/tools/docutils clean: pycremoval - find . -path '*Stackless/platf/stackman/*.a' -prune -o -name '*.[oa]' -exec rm -f {} ';' + find . -path '*Stackless/stackman/*.a' -prune -o -name '*.[oa]' -exec rm -f {} ';' find . -name '*.s[ol]' -exec rm -f {} ';' find . -name '*.so.[0-9]*.[0-9]*' -exec rm -f {} ';' find build -name 'fficonfig.h' -exec rm -f {} ';' || true diff --git a/Stackless/platf/slp_transfer.c b/Stackless/platf/slp_transfer.c index ba5e2acb2578d4..771def9be2444a 100644 --- a/Stackless/platf/slp_transfer.c +++ b/Stackless/platf/slp_transfer.c @@ -39,11 +39,13 @@ */ #define SLP_EVAL /* enable code generation in the included header */ +#ifndef SLP_NO_STACKMAN /* defined by configure --without-stackman */ /* First, see if stackman an implementation without external * assembler, use that if possible */ #define STACKMAN_OPTIONAL -#include "stackman/stackman/stackman.h" +#include "stackman/stackman.h" +#endif /* #ifndef SLP_NO_STACKMAN */ #if defined(STACKMAN_PLATFORM) && !defined(STACKMAN_EXTERNAL_ASM) #include "switch_stackman.h" #else /* use traditional stackless switching */ diff --git a/Stackless/platf/stackman b/Stackless/platf/stackman deleted file mode 160000 index 63bb64b37d7720..00000000000000 --- a/Stackless/platf/stackman +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 63bb64b37d7720230e6579e3a266e62a1f66fe31 diff --git a/Stackless/platf/switch_stackman.h b/Stackless/platf/switch_stackman.h index ce51239fcbfe9e..e6c1f3ca39523c 100644 --- a/Stackless/platf/switch_stackman.h +++ b/Stackless/platf/switch_stackman.h @@ -62,5 +62,5 @@ slp_switch(void) return error; } -#include "stackman/stackman/stackman_impl.h" +#include "stackman/stackman_impl.h" #endif diff --git a/Stackless/stackman b/Stackless/stackman new file mode 160000 index 00000000000000..c572d4d07cc850 --- /dev/null +++ b/Stackless/stackman @@ -0,0 +1 @@ +Subproject commit c572d4d07cc8507bc5a519dbab6e461f718547d7 diff --git a/configure b/configure index 0f7cce3667e9d7..d80ba258803444 100755 --- a/configure +++ b/configure @@ -652,6 +652,7 @@ DTRACE TCLTK_LIBS TCLTK_INCLUDES LIBFFI_INCLUDEDIR +SLPFLAGS PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -729,7 +730,6 @@ CFLAGS CC EXPORT_MACOSX_DEPLOYMENT_TARGET CONFIGURE_MACOSX_DEPLOYMENT_TARGET -SLPFLAGS _PYTHON_HOST_PLATFORM MACHDEP FRAMEWORKINSTALLAPPSPREFIX @@ -826,6 +826,7 @@ with_address_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer with_libs +with_stackman with_system_expat with_system_ffi with_system_libmpdec @@ -1528,6 +1529,8 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer (ubsan) --with-libs='lib1 ...' link against additional libs + --with-stackman build python with an external stackman library or + without using stackman at all --with-system-expat build pyexpat module using an installed expat library --with-system-ffi build _ctypes module using an installed ffi library @@ -3335,17 +3338,6 @@ if test "$cross_compiling" = yes; then _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" fi -# Stackless flags for compiling the hard switching code -case $MACHDEP in - darwin) - SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" - ;; - *) - SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" - ;; -esac - - # Some systems cannot stand _XOPEN_SOURCE being defined at all; they # disable features if it is defined, without any means to access these # features as extensions. For these systems, we skip the definition of @@ -10336,6 +10328,53 @@ $as_echo "no" >&6; } fi fi +# Stackless flags for compiling the hard switching code +case $MACHDEP in + darwin) + SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" + ;; + *) + SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" + ;; +esac + +# Check for use of an external or no stackman library +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-stackman" >&5 +$as_echo_n "checking for --with-stackman... " >&6; } + +# Check whether --with-stackman was given. +if test "${with_stackman+set}" = set; then : + withval=$with_stackman; +fi + + +case "$with_stackman" in + "") + with_stackman="yes" + ;; + "yes"|"no") + ;; + "*") + if test -f "$with_stackman"/Makefile ; then + : + else + as_fn_error $? "--with-stackman argument must be a directory containing the stackman project" "$LINENO" 5 + fi + ;; +esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_stackman" >&5 +$as_echo "$with_stackman" >&6; } + +if test "$with_stackman" = "yes" ; then + with_stackman="Stackless/stackman" +fi +if test "$with_stackman" = "no" ; then + SLPFLAGS="-DSLP_NO_STACKMAN $SLPFLAGS" +else + SLPFLAGS="-I$with_stackman $SLPFLAGS" +fi + + # Check for use of the system expat library { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-expat" >&5 $as_echo_n "checking for --with-system-expat... " >&6; } diff --git a/configure.ac b/configure.ac index 00f60e9163255e..9049a240d13d6d 100644 --- a/configure.ac +++ b/configure.ac @@ -437,17 +437,6 @@ if test "$cross_compiling" = yes; then _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" fi -# Stackless flags for compiling the hard switching code -case $MACHDEP in - darwin) - SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" - ;; - *) - SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" - ;; -esac -AC_SUBST(SLPFLAGS) - # Some systems cannot stand _XOPEN_SOURCE being defined at all; they # disable features if it is defined, without any means to access these # features as extensions. For these systems, we skip the definition of @@ -2922,6 +2911,47 @@ LIBS="$withval $LIBS" PKG_PROG_PKG_CONFIG +# Stackless flags for compiling the hard switching code +case $MACHDEP in + darwin) + SLPFLAGS="-fomit-frame-pointer -fno-inline-functions" + ;; + *) + SLPFLAGS="-fno-omit-frame-pointer -fno-inline-functions" + ;; +esac + +# Check for use of an external or no stackman library +AC_MSG_CHECKING(for --with-stackman) +AC_ARG_WITH(stackman, + AS_HELP_STRING([--with-stackman], [build python with an external stackman library or without using stackman at all]),,,) + +case "$with_stackman" in + "") + with_stackman="yes" + ;; + "yes"|"no") + ;; + "*") + if test -f "$with_stackman"/Makefile ; then + : + else + AC_MSG_ERROR([--with-stackman argument must be a directory containing the stackman project]) + fi + ;; +esac +AC_MSG_RESULT($with_stackman) + +if test "$with_stackman" = "yes" ; then + with_stackman="Stackless/stackman" +fi +if test "$with_stackman" = "no" ; then + SLPFLAGS="-DSLP_NO_STACKMAN $SLPFLAGS" +else + SLPFLAGS="-I$with_stackman $SLPFLAGS" +fi +AC_SUBST(SLPFLAGS) + # Check for use of the system expat library AC_MSG_CHECKING(for --with-system-expat) AC_ARG_WITH(system_expat, From 7772ab70bba5a82728d6a09d2b610e019fac6d55 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sun, 11 Jul 2021 22:51:58 +0200 Subject: [PATCH 12/18] Update create_source_archive.sh to include Stackman. --- Stackless/Tools/create_source_archive.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Stackless/Tools/create_source_archive.sh b/Stackless/Tools/create_source_archive.sh index e19b475ee4cc6f..25ba184d4c2bf6 100644 --- a/Stackless/Tools/create_source_archive.sh +++ b/Stackless/Tools/create_source_archive.sh @@ -14,7 +14,7 @@ if [ $# != 3 ] ; then exit 1 fi -: ${excludes:=.gitignore .git* .hg* .bzrignore .mention-bot .travis.yml .readthedocs.yml .azure-pipelines} +: ${excludes:=.gitignore .git* .mention-bot .travis.yml .readthedocs.yml .azure-pipelines Stackless/stackman/.git* } git_python_dir="$1" ; shift if [ "x$1" = "x-v" ] ; then @@ -42,6 +42,7 @@ mkdir "$tmpdir" cd "$git_python_dir" git archive --format=tar --prefix="${srcdir}/" "$tag" | \ ( cd "${tmpdir}" && tar xf - ) +git submodule foreach --recursive "git archive --format=tar --prefix=${srcdir}/\$path/ HEAD | ( cd \"${tmpdir}\" && tar xf - )" cd "${tmpdir}" ( cd "${srcdir}" && set +f && rm -rf $excludes ) From 8f6a3df4e2f66734bf0f2350cd69d89cf4353711 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sun, 11 Jul 2021 23:13:58 +0200 Subject: [PATCH 13/18] Detect Stackman abiname and link the appropriate libstackman.a Actually we extract the object files and link the object files, because this is much simpler to integrate into the Python Makefile. --- Makefile.pre.in | 22 +++++++++++++++------- configure | 15 +++++++++++++++ configure.ac | 13 +++++++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index b5074f5cbc2360..2861575a18b64d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -119,6 +119,8 @@ CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Special C flags for slp_transfer.c SLPFLAGS= @SLPFLAGS@ +SLP_STACKMAN_LIB= @SLP_STACKMAN_LIB@ +SLP_STACKMAN_OBJS= @SLP_STACKMAN_OBJS@ # Machine-dependent subdirectories MACHDEP= @MACHDEP@ @@ -348,6 +350,7 @@ PYTHON_OBJS= \ Stackless/module/taskletobject.o \ Stackless/pickling/prickelpit.o \ Stackless/pickling/safe_pickle.o \ + $(SLP_STACKMAN_OBJS) \ Python/codecs.o \ Python/compile.o \ Python/coreconfig.o \ @@ -724,6 +727,17 @@ $(srcdir)/Include/cpython/slp_exttype.h: $(srcdir)/Objects/typeobject.c $(PYTHON_FOR_REGEN) $(srcdir)/Stackless/Tools/extract_slp_info.py $(UPDATE_FILE) $(srcdir)/Include/cpython/slp_exttype.h $(srcdir)/Include/cpython/slp_exttype.h.new +$(SLP_STACKMAN_OBJS): $(SLP_STACKMAN_LIB) + cp $(SLP_STACKMAN_LIB) Stackless/platf/libstackman.a + cd Stackless/platf && $(AR) x libstackman.a && rm libstackman.a + +Stackless/platf/slp_transfer.o: $(srcdir)/Stackless/platf/slp_transfer.c + $(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c + +.PHONY: teststackless +teststackless: @DEF_MAKE_RULE@ platform + $(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py + ############################################################################ # Importlib @@ -989,9 +1003,6 @@ regen-typeslots: $(srcdir)/Objects/typeslots.inc.new $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new -Stackless/platf/slp_transfer.o: $(srcdir)/Stackless/platf/slp_transfer.c - $(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c - ############################################################################ # Header files @@ -1149,7 +1160,7 @@ TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py TESTTIMEOUT= 1200 -.PHONY: test testall testuniversal buildbottest pythoninfo teststackless +.PHONY: test testall testuniversal buildbottest pythoninfo # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. @@ -1170,9 +1181,6 @@ testall: @DEF_MAKE_RULE@ platform -$(TESTRUNNER) -u all $(TESTOPTS) $(TESTRUNNER) -u all $(TESTOPTS) -teststackless: @DEF_MAKE_RULE@ platform - $(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py - # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. testuniversal: @DEF_MAKE_RULE@ platform diff --git a/configure b/configure index d80ba258803444..0423574b312749 100755 --- a/configure +++ b/configure @@ -652,6 +652,8 @@ DTRACE TCLTK_LIBS TCLTK_INCLUDES LIBFFI_INCLUDEDIR +SLP_STACKMAN_OBJS +SLP_STACKMAN_LIB SLPFLAGS PKG_CONFIG_LIBDIR PKG_CONFIG_PATH @@ -10368,13 +10370,26 @@ $as_echo "$with_stackman" >&6; } if test "$with_stackman" = "yes" ; then with_stackman="Stackless/stackman" fi +SLP_STACKMAN_LIB= +SLP_STACKMAN_OBJS= if test "$with_stackman" = "no" ; then SLPFLAGS="-DSLP_NO_STACKMAN $SLPFLAGS" else SLPFLAGS="-I$with_stackman $SLPFLAGS" + slp_stackman_abiname=`cd "$with_stackman" && make -s CC="$CC" CXX="$CXX" AR="$AR" abiname 2>/dev/null` + if test "X$slp_stackman_abiname" != "X" ; then + if test -f "$with_stackman/lib/$slp_stackman_abiname/libstackman.a" ; then + SLP_STACKMAN_LIB="$with_stackman/lib/$slp_stackman_abiname/libstackman.a" + SLP_STACKMAN_OBJS=`$AR t "$SLP_STACKMAN_LIB" | sed -e 's|^|Stackless/platf/|' | tr '\n' ' '` + else + as_fn_error $? "Stackman library not found: $with_stackman/lib/$slp_stackman_abiname/libstackman.a" "$LINENO" 5 + fi + fi fi + + # Check for use of the system expat library { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-system-expat" >&5 $as_echo_n "checking for --with-system-expat... " >&6; } diff --git a/configure.ac b/configure.ac index 9049a240d13d6d..a0d6075be8042d 100644 --- a/configure.ac +++ b/configure.ac @@ -2945,12 +2945,25 @@ AC_MSG_RESULT($with_stackman) if test "$with_stackman" = "yes" ; then with_stackman="Stackless/stackman" fi +SLP_STACKMAN_LIB= +SLP_STACKMAN_OBJS= if test "$with_stackman" = "no" ; then SLPFLAGS="-DSLP_NO_STACKMAN $SLPFLAGS" else SLPFLAGS="-I$with_stackman $SLPFLAGS" + slp_stackman_abiname=`cd "$with_stackman" && make -s CC="$CC" CXX="$CXX" AR="$AR" abiname 2>/dev/null` + if test "X$slp_stackman_abiname" != "X" ; then + if test -f "$with_stackman/lib/$slp_stackman_abiname/libstackman.a" ; then + SLP_STACKMAN_LIB="$with_stackman/lib/$slp_stackman_abiname/libstackman.a" + SLP_STACKMAN_OBJS=`$AR t "$SLP_STACKMAN_LIB" | sed -e 's|^|Stackless/platf/|' | tr '\n' ' '` + else + AC_MSG_ERROR([Stackman library not found: $with_stackman/lib/$slp_stackman_abiname/libstackman.a]) + fi + fi fi AC_SUBST(SLPFLAGS) +AC_SUBST(SLP_STACKMAN_LIB) +AC_SUBST(SLP_STACKMAN_OBJS) # Check for use of the system expat library AC_MSG_CHECKING(for --with-system-expat) From a20298bbd2aec243179b69ba92bbfa094528dc12 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sun, 11 Jul 2021 23:26:51 +0200 Subject: [PATCH 14/18] remove superfluous tabs --- Makefile.pre.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2861575a18b64d..b2988d39390d49 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -732,11 +732,11 @@ $(SLP_STACKMAN_OBJS): $(SLP_STACKMAN_LIB) cd Stackless/platf && $(AR) x libstackman.a && rm libstackman.a Stackless/platf/slp_transfer.o: $(srcdir)/Stackless/platf/slp_transfer.c - $(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c + $(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c .PHONY: teststackless teststackless: @DEF_MAKE_RULE@ platform - $(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py + $(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py ############################################################################ # Importlib From c5bea115985e4306bd137281882f2cc9a8c1493c Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Wed, 14 Jul 2021 08:11:48 +0200 Subject: [PATCH 15/18] Fix the usage of SLP_SAVE_STATE. --- Stackless/platf/switch_stackman.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Stackless/platf/switch_stackman.h b/Stackless/platf/switch_stackman.h index e6c1f3ca39523c..a358f150c90058 100644 --- a/Stackless/platf/switch_stackman.h +++ b/Stackless/platf/switch_stackman.h @@ -21,7 +21,8 @@ static int slp_stackman_cb_save(void *sp, intptr_t *pdiff) { intptr_t diff; - SLP_SAVE_STATE(sp, diff); + /* first argument must be a pointer to a "stack word", not a void* */ + SLP_SAVE_STATE(((intptr_t *)sp), diff); *pdiff = diff; return 1; } From ab0c29ae9ba2e2f246c3a71cc1e44d6bdc0cd99d Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Wed, 14 Jul 2021 11:57:34 +0200 Subject: [PATCH 16/18] Integrate Stackman into the windows build infrastructure. You can set the property stackmanDir=no to build without Stackman. You can set the property stackmanDir=C:\path\to\stackman to build with Stackman installed in C:\path\to\stackman. (To set a property add '"/p:name=value"' to the build.bat command line. --- PCbuild/python.props | 5 +++++ PCbuild/pythoncore.vcxproj | 31 +++++++++++++------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/PCbuild/python.props b/PCbuild/python.props index f83d4df0d59fa4..a828316f695884 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -66,6 +66,11 @@ $(BuildPath)python$(PyDebugExt).exe + + + $(PySourcePath)Stackless\stackman\ + $(stackmanDir)\ + stackman.lib; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1cab4c0cef4224..bdcbade7fc24cc 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -72,9 +72,13 @@ $(zlibDir);%(AdditionalIncludeDirectories) _USRDLL;Py_BUILD_CORE;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) + SLP_NO_STACKMAN;%(PreprocessorDefinitions) - version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies) + version.lib;shlwapi.lib;ws2_32.lib;$(stackmanLib)%(AdditionalDependencies) + $(stackmanDir)lib\win_x64;%(AdditionalLibraryDirectories) + $(stackmanDir)lib\win_x86;%(AdditionalLibraryDirectories) + false @@ -438,7 +442,9 @@ - + + $(stackmanDir);%(AdditionalIncludeDirectories) + @@ -470,23 +476,12 @@ - true - true - true - true + true + true Document - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) - ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" - Running ml64 - $(IntDir)%(Filename) .obj;%(Outputs) + ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)" + Running ml64 + $(IntDir)%(Filename) .obj;%(Outputs) From bb897f8ede0ab5f0abc021eb8473df6176f2f5e7 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Wed, 14 Jul 2021 17:21:18 +0200 Subject: [PATCH 17/18] Add documentation and build instructions. --- Stackless/changelog.txt | 5 +++++ Stackless/readme.txt | 27 ++++++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Stackless/changelog.txt b/Stackless/changelog.txt index 2558bdae8374b1..fc3ecbd6de693a 100644 --- a/Stackless/changelog.txt +++ b/Stackless/changelog.txt @@ -9,6 +9,11 @@ What's New in Stackless 3.X.X? *Release date: 20XX-XX-XX* +- https://github.com/stackless-dev/stackless/issues/278 + Stackless now uses Stackman https://github.com/stackless-dev/stackman for + stack switching / hard tasklet switching. The legacy stack switching code + is still available. See 'readme.txt' for details. + - https://github.com/stackless-dev/stackless/issues/288 Starting from commit b73166e building Stackless on Windows was broken. This has been fixed. diff --git a/Stackless/readme.txt b/Stackless/readme.txt index 35d08cdde701e2..10b1ab22f95b84 100644 --- a/Stackless/readme.txt +++ b/Stackless/readme.txt @@ -27,13 +27,26 @@ It is the ultimate truth. Read it at https://github.com/stackless-dev/stackless Building Stackless ------------------ -Just follow the build instructions for regular Python. If you define -the C-preprocessor symbol STACKLESS_OFF, you get a Python interpreter -without Stackless. It should behave exactly like the corresponding -version of regular Python. Any difference constitutes a bug. -(Windows 64bit only: if you define STACKLESS_OFF, you also need -to clear the content of the file Stackless\platf\switch_x64_masm.asm: - c:\> echo END > Stackless\platf\switch_x64_masm.asm ) +Just follow the build instructions for regular C-Python. + +To more easily support new platforms and CPU architectures, Stackless Python +now commonly uses Stackman (https://github.com/stackless-dev/stackman) +for stack switching / hard switching between tasklets. Stackman is a +lightweight library for stack manipulation. The Stackless source archive +already contains a copy of Stackman, which is normally used automatically. +However, the legacy Stackless switching code can still be used without +restrictions. To do this, Stackless must be configured with +`configure --without-stackman` or built under Windows with +`build.bat "/p:stackmanDir=no"`. + +It is also possible to use your own version of Stackman instead of the copy +bundled with Stackless. To do this configure Stackless with +`configure --with-stackman=/path/to/your/stackman` or built under Windows +with `build.bat "/p:stackmanDir=X:\path\to\your\stackman"`. + +If you define the C-preprocessor symbol STACKLESS_OFF in Include/stackless.h, +you get a Python interpreter without Stackless. It should behave exactly like +the corresponding version of regular Python. Any difference constitutes a bug. Contributing ------------ From ac479bde72f3b2db18b50e62d6f1f2964fb8c8c3 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Thu, 15 Jul 2021 11:13:12 +0200 Subject: [PATCH 18/18] Add support for ARM and ARM64 --- PCbuild/pythoncore.vcxproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index cecd7b52e708bf..052388c4bfb241 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -94,6 +94,8 @@ version.lib;shlwapi.lib;ws2_32.lib;$(stackmanLib)%(AdditionalDependencies) $(stackmanDir)lib\win_x64;%(AdditionalLibraryDirectories) $(stackmanDir)lib\win_x86;%(AdditionalLibraryDirectories) + $(stackmanDir)lib\win_arm;%(AdditionalLibraryDirectories) + $(stackmanDir)lib\win_arm64;%(AdditionalLibraryDirectories) false