Stackless issue #278: implement switching using the Stackman submodule

Stackless now uses Stackman https://github.com/stackless-dev/stackman for stack switching / hard tasklet switching. The legacy stack switching code is still available. See 'readme.txt' for details.

The Stackman code is in a git submodule located below Stackless/stackman.

You can control the usage of Stackman with the configure options "--with-stackman" and "--without-stackman". On Windows you can use the MSBuild property "stackmanDir". (To set a property add '"/p:name=value"' to the build.bat command line.)

Co-authored-by: Anselm Kruis <[email protected]>

Author: kristjanvalur

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "Stackless/platf/stackman"]
+	path = Stackless/stackman
+	url = https://github.com/stackless-dev/stackman.git

Makefile.pre.in

@@ -119,6 +119,8 @@ CFLAGS_ALIASING=@CFLAGS_ALIASING@
 
 # Special C flags for slp_transfer.c
 SLPFLAGS=	@SLPFLAGS@
+SLP_STACKMAN_LIB= @SLP_STACKMAN_LIB@
+SLP_STACKMAN_OBJS= @SLP_STACKMAN_OBJS@
 
 # Machine-dependent subdirectories
 MACHDEP=	@MACHDEP@
@@ -329,6 +331,7 @@ PYTHON_OBJS=	\
 		Stackless/module/taskletobject.o \
 		Stackless/pickling/prickelpit.o \
 		Stackless/pickling/safe_pickle.o \
+		$(SLP_STACKMAN_OBJS) \
 		Python/codecs.o \
 		Python/compile.o \
 		Python/coreconfig.o \
@@ -707,6 +710,17 @@ $(srcdir)/Include/cpython/slp_exttype.h: $(srcdir)/Objects/typeobject.c
 	$(PYTHON_FOR_REGEN) $(srcdir)/Stackless/Tools/extract_slp_info.py && \
 	mv $(srcdir)/Include/cpython/slp_exttype.h.new $(srcdir)/Include/cpython/slp_exttype.h
 
+$(SLP_STACKMAN_OBJS): $(SLP_STACKMAN_LIB)
+	cp $(SLP_STACKMAN_LIB) Stackless/platf/libstackman.a
+	cd Stackless/platf && $(AR) x libstackman.a && rm libstackman.a
+
+Stackless/platf/slp_transfer.o: $(srcdir)/Stackless/platf/slp_transfer.c
+	$(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c
+
+.PHONY: teststackless
+teststackless:	@DEF_MAKE_RULE@ platform
+	$(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py
+
 ############################################################################
 # Importlib
 
@@ -960,9 +974,6 @@ regen-typeslots:
 		$(srcdir)/Objects/typeslots.inc.new
 	$(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new
 
-Stackless/platf/slp_transfer.o: $(srcdir)/Stackless/platf/slp_transfer.c
-		$(CC) -c $(PY_CORE_CFLAGS) $(SLPFLAGS) -o $@ $(srcdir)/Stackless/platf/slp_transfer.c
-
 ############################################################################
 # Header files
 
@@ -1121,7 +1132,7 @@ TESTPYTHON=	$(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS)
 TESTRUNNER=	$(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
 TESTTIMEOUT=	1200
 
-.PHONY: test testall testuniversal buildbottest pythoninfo teststackless
+.PHONY: test testall testuniversal buildbottest pythoninfo
 
 # Run a basic set of regression tests.
 # This excludes some tests that are particularly resource-intensive.
@@ -1142,9 +1153,6 @@ testall:	@DEF_MAKE_RULE@ platform
 		-$(TESTRUNNER) -u all $(TESTOPTS)
 		$(TESTRUNNER) -u all $(TESTOPTS)
 
-teststackless:	@DEF_MAKE_RULE@ platform
-		$(TESTPYTHON) -E $(srcdir)/Stackless/unittests/runAll.py
-
 # Run the test suite for both architectures in a Universal build on OSX.
 # Must be run on an Intel box.
 testuniversal:	@DEF_MAKE_RULE@ platform
@@ -1762,7 +1770,7 @@ docclean:
 	-rm -rf Doc/tools/sphinx Doc/tools/pygments Doc/tools/docutils
 
 clean: pycremoval
-	find . -name '*.[oa]' -exec rm -f {} ';'
+	find . -path '*Stackless/stackman/*.a' -prune -o -name '*.[oa]' -exec rm -f {} ';'
 	find . -name '*.s[ol]' -exec rm -f {} ';'
 	find . -name '*.so.[0-9]*.[0-9]*' -exec rm -f {} ';'
 	find build -name 'fficonfig.h' -exec rm -f {} ';' || true

PCbuild/python.props

@@ -71,6 +71,11 @@
 
     <!-- Full path of the resulting python.exe binary -->
     <PythonExe Condition="'$(PythonExe)' == ''">$(BuildPath)python$(PyDebugExt).exe</PythonExe>
+
+    <!-- Stackless stackman directory. Set this property to 'no', to build without Stackman -->
+    <stackmanDir Condition="'$(stackmanDir)' == ''">$(PySourcePath)Stackless\stackman\</stackmanDir>
+    <stackmanDir Condition="'$(stackmanDir)' != 'no' And !HasTrailingSlash($(stackmanDir))">$(stackmanDir)\</stackmanDir>
+    <stackmanLib Condition="'$(stackmanDir)' != 'no'">stackman.lib;</stackmanLib>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Platform)'=='ARM'" Label="ArmConfiguration">

PCbuild/pythoncore.vcxproj

@@ -88,9 +88,15 @@
       <AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(stackmanDir)' == 'no'">SLP_NO_STACKMAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>version.lib;shlwapi.lib;ws2_32.lib;$(stackmanLib)%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories Condition="'$(stackmanLib)'!='' And '$(Platform)'=='x64'">$(stackmanDir)lib\win_x64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories Condition="'$(stackmanLib)'!='' And '$(Platform)'=='Win32'">$(stackmanDir)lib\win_x86;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories Condition="'$(stackmanLib)'!='' And '$(Platform)'=='ARM'">$(stackmanDir)lib\win_arm;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories Condition="'$(stackmanLib)'!='' And '$(Platform)'=='ARM64'">$(stackmanDir)lib\win_arm64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <ImageHasSafeExceptionHandlers Condition="'$(stackmanLib)'!='' And '$(Platform)'=='Win32'">false</ImageHasSafeExceptionHandlers>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
@@ -452,7 +458,9 @@
     <ClCompile Include="..\Python\thread.c" />
     <ClCompile Include="..\Python\traceback.c" />
     <ClCompile Include="..\Stackless\core\cframeobject.c" />
-    <ClCompile Include="..\Stackless\platf\slp_transfer.c" />
+    <ClCompile Include="..\Stackless\platf\slp_transfer.c">
+      <AdditionalIncludeDirectories Condition="'$(stackmanDir)'!='no'">$(stackmanDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
     <ClCompile Include="..\Stackless\core\stacklesseval.c" />
     <ClCompile Include="..\Stackless\core\stackless_util.c" />
     <ClCompile Include="..\Stackless\module\channelobject.c" />
@@ -484,23 +492,12 @@
   </ItemGroup>
   <ItemGroup>
     <CustomBuild Include="..\Stackless\platf\switch_x64_masm.asm">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='PGInstrument|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='PGUpdate|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Platform)'!='x64'">true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(Platform)'=='x64' And '$(stackmanDir)'!='no'">true</ExcludedFromBuild>
       <FileType>Document</FileType>
-      <Command Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)"</Command>
-      <Message Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Running ml64</Message>
-      <Outputs Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(IntDir)%(Filename) .obj;%(Outputs)</Outputs>
-      <Command Condition="'$(Configuration)|$(Platform)'=='PGInstrument|x64'">ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)"</Command>
-      <Message Condition="'$(Configuration)|$(Platform)'=='PGInstrument|x64'">Running ml64</Message>
-      <Outputs Condition="'$(Configuration)|$(Platform)'=='PGInstrument|x64'">$(IntDir)%(Filename) .obj;%(Outputs)</Outputs>
-      <Command Condition="'$(Configuration)|$(Platform)'=='PGUpdate|x64'">ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)"</Command>
-      <Message Condition="'$(Configuration)|$(Platform)'=='PGUpdate|x64'">Running ml64</Message>
-      <Outputs Condition="'$(Configuration)|$(Platform)'=='PGUpdate|x64'">$(IntDir)%(Filename) .obj;%(Outputs)</Outputs>
-      <Command Condition="'$(Configuration)|$(Platform)'=='Release|x64'">ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)"</Command>
-      <Message Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Running ml64</Message>
-      <Outputs Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(IntDir)%(Filename) .obj;%(Outputs)</Outputs>
+      <Command Condition="'$(Platform)'=='x64'">ml64 /nologo /c /Zi /Fo "$(IntDir)%(Filename) .obj" "%(FullPath)"</Command>
+      <Message Condition="'$(Platform)'=='x64'">Running ml64</Message>
+      <Outputs Condition="'$(Platform)'=='x64'">$(IntDir)%(Filename) .obj;%(Outputs)</Outputs>
     </CustomBuild>
     <None Include="..\Stackless\readme.txt" />
   </ItemGroup>

Stackless/Tools/create_source_archive.sh

@@ -14,7 +14,7 @@ if [ $# != 3 ] ; then
    exit 1
 fi
 
-: ${excludes:=.gitignore .git* .hg* .bzrignore .mention-bot .travis.yml .readthedocs.yml .azure-pipelines}
+: ${excludes:=.gitignore .git* .mention-bot .travis.yml .readthedocs.yml .azure-pipelines Stackless/stackman/.git* }
 
 git_python_dir="$1" ; shift
 if [ "x$1" = "x-v" ] ; then
@@ -42,6 +42,7 @@ mkdir "$tmpdir"
 cd "$git_python_dir"
 git archive --format=tar --prefix="${srcdir}/" "$tag" | \
   ( cd "${tmpdir}" && tar xf - )
+git submodule foreach --recursive "git archive --format=tar --prefix=${srcdir}/\$path/ HEAD | ( cd \"${tmpdir}\" && tar xf - )"
 cd "${tmpdir}"
 ( cd "${srcdir}" && set +f && rm -rf $excludes )
 

Stackless/changelog.txt

@@ -9,6 +9,11 @@ What's New in Stackless 3.X.X?
 
 *Release date: 20XX-XX-XX*
 
+- https://github.com/stackless-dev/stackless/issues/278
+  Stackless now uses Stackman https://github.com/stackless-dev/stackman for
+  stack switching / hard tasklet switching. The legacy stack switching code
+  is still available. See 'readme.txt' for details.
+
 - https://github.com/stackless-dev/stackless/issues/254
   The Stackless version is now "3.8".
 

Stackless/platf/slp_transfer.c

@@ -39,6 +39,16 @@
  */
 #define SLP_EVAL  /* enable code generation in the included header */
 
+#ifndef SLP_NO_STACKMAN  /* defined by configure --without-stackman */
+/* First, see if stackman an implementation without external
+ * assembler, use that if possible
+ */
+#define STACKMAN_OPTIONAL
+#include "stackman/stackman.h"
+#endif  /* #ifndef SLP_NO_STACKMAN */
+#if defined(STACKMAN_PLATFORM) && !defined(STACKMAN_EXTERNAL_ASM)
+#include "switch_stackman.h"
+#else  /* use traditional stackless switching */
 #if   defined(MS_WIN32) && !defined(MS_WIN64) && defined(_M_IX86)
 #include "switch_x86_msvc.h" /* MS Visual Studio on X86 */
 #elif defined(MS_WIN64) && defined(_M_X64)
@@ -74,6 +84,7 @@ Please provide an implementation of the switch_XXX.h
 or disable the STACKLESS flag.
 **********
 #endif
+#endif  /* use traditional stackless switching */
 
 /* default definitions if not defined in above files */
 

Stackless/platf/switch_stackman.h

@@ -0,0 +1,67 @@
+/*
+ * this is the internal transfer function, using
+ * the stackman platform library.
+ * We create a wrapper callback that employs the existing
+ * stack macros.
+ * At some later point in time, the callback could be
+ * written directly.
+ *
+ */
+
+#define SLP_STACK_REFPLUS 1
+
+#ifdef SLP_EVAL
+#define SLP_STACK_MAGIC 0
+
+
+/* need a special function arount SLP_SAVE_STATE() because
+ * the macro has a built-in return of 0 or -1.  Must catch
+ * that.
+ */
+static int slp_stackman_cb_save(void *sp, intptr_t *pdiff)
+{
+    intptr_t diff;
+    /* first argument must be a pointer to a "stack word", not a void* */
+    SLP_SAVE_STATE(((intptr_t *)sp), diff);
+    *pdiff = diff;
+    return 1;
+}
+
+static void *slp_stackman_cb(void *_ctxt, int opcode, void *sp)
+{
+    int *error = (int*)_ctxt;
+    intptr_t stsizediff;
+    if (opcode == STACKMAN_OP_SAVE)
+    {
+        int ret = slp_stackman_cb_save(sp, &stsizediff);
+        if (ret == 1) {
+            /* regular switch */
+            return (void*)((char*)sp + stsizediff);
+        } 
+        if (ret == -1)
+        {
+            *error = -1;
+        }
+        /* error or save only, no change in sp */
+        return sp;
+    }
+    else
+    {
+        if (*error != -1)
+            SLP_RESTORE_STATE();
+        return sp;
+    }
+}
+
+static int
+slp_switch(void)
+{
+    /* this can be on the stack, because if there isn't a switch
+     * then it is intact. (error or save only) */
+    int error = 0;
+    stackman_switch(&slp_stackman_cb, &error);
+    return error;
+}
+
+#include "stackman/stackman_impl.h"
+#endif

Stackless/readme.txt

@@ -27,13 +27,26 @@ It is the ultimate truth. Read it at https://github.com/stackless-dev/stackless
 Building Stackless
 ------------------
 
-Just follow the build instructions for regular Python. If you define
-the C-preprocessor symbol STACKLESS_OFF, you get a Python interpreter
-without Stackless. It should behave exactly like the corresponding
-version of regular Python. Any difference constitutes a bug.
-(Windows 64bit only: if you define STACKLESS_OFF, you also need
-to clear the content of the file Stackless\platf\switch_x64_masm.asm:
-  c:\> echo END > Stackless\platf\switch_x64_masm.asm )
+Just follow the build instructions for regular C-Python.
+
+To more easily support new platforms and CPU architectures, Stackless Python
+now commonly uses Stackman (https://github.com/stackless-dev/stackman)
+for stack switching / hard switching between tasklets. Stackman is a
+lightweight library for stack manipulation. The Stackless source archive
+already contains a copy of Stackman, which is normally used automatically.
+However, the legacy Stackless switching code can still be used without
+restrictions. To do this, Stackless must be configured with
+`configure --without-stackman` or built under Windows with
+`build.bat "/p:stackmanDir=no"`.
+
+It is also possible to use your own version of Stackman instead of the copy
+bundled with Stackless. To do this configure Stackless with
+`configure --with-stackman=/path/to/your/stackman` or built under Windows
+with `build.bat "/p:stackmanDir=X:\path\to\your\stackman"`.
+
+If you define the C-preprocessor symbol STACKLESS_OFF in Include/stackless.h,
+you get a Python interpreter without Stackless. It should behave exactly like
+the corresponding version of regular Python. Any difference constitutes a bug.
 
 Contributing
 ------------