mage: build with OpenSSL

3 build options via TT_CLI_BUILD_SSL environment variable have been
added:

- no     without OpenSSL;
- static with statically linked OpenSSL;
- shared with dynamically linked OpenSSL;

Closes #308

Author: oleg-jukovec

.github/actions/prepare-ce-test-env/action.yml

@@ -42,5 +42,7 @@ runs:
         tarantool-version: '${{ inputs.tarantool-version }}'
 
     - name: Build tt
+      env:
+        TT_CLI_BUILD_SSL: 'static'
       run: mage build
       shell: bash

.github/actions/prepare-ee-test-env/action.yml

@@ -64,5 +64,7 @@ runs:
         shell: bash
 
       - name: Build tt
+        env:
+          TT_CLI_BUILD: 'static'
         run: mage build
         shell: bash

.github/workflows/full-ci.yml

@@ -134,6 +134,8 @@ jobs:
         uses: docker-practice/actions-setup-docker@master
 
       - name: Build tt
+        env:
+          TT_CLI_BUILD_SSL: 'static'
         run: mage build
 
       - name: Install tarantool

.github/workflows/publish.yml

@@ -28,6 +28,14 @@ jobs:
           cd mage
           go run bootstrap.go
 
+      - name: Build OpenSSL 3.0
+        run: |
+          wget https://github.com/openssl/openssl/releases/download/openssl-3.0.8/openssl-3.0.8.tar.gz
+          tar -xvf openssl-3.0.8.tar.gz
+          cd openssl-3.0.8/
+          ./Configure --prefix=${GITHUB_WORKSPACE}/openssl no-shared
+          make && make install
+
       - name: Setup GoReleaser
         run: |
           curl -O -L https://github.com/goreleaser/goreleaser/releases/download/v1.12.3/goreleaser_1.12.3_amd64.deb
@@ -45,6 +53,8 @@ jobs:
 
       - name: Build packages
         env:
+          CGO_LDFLAGS: "-L${{ env.GITHUB_WORKSPACE }}/openssl/lib64"
+          CGO_CFLAGS: "-I${{ env.GITHUB_WORKSPACE }}/openssl/include"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           goreleaser release ${{ steps.set-goreleaser-flags.outputs.GORELEASER_FLAGS }}

.github/workflows/tests.yml

@@ -124,6 +124,8 @@ jobs:
           pip3 install pytest tarantool requests psutil pyyaml netifaces
 
       - name: Build tt
+        env:
+          TT_CLI_BUILD_SSL: 'static'
         run: mage build
 
       - name: Install tarantool

.goreleaser.yml

@@ -12,12 +12,15 @@ builds:
     dir: cli
 
     env:
-      - CGO_ENABLED=0
+      - CGO_ENABLED=1
 
-    flags:
-      - -tags=go_tarantool_ssl_disable
+    tags:
+      - netgo
+      - osusergo
+      - openssl_static
 
     ldflags:
+      - -linkmode=external -extldflags -static
       - -s -w
       - -X github.com/tarantool/tt/cli/version.gitTag={{ .Tag }}
       - -X github.com/tarantool/tt/cli/version.gitCommit={{ .ShortCommit }}

README.rst

@@ -37,8 +37,29 @@ Build
 
    git clone https://github.com/tarantool/tt --recursive
    cd tt
+
+You can build a binary without OpenSSL and TLS support for development
+purposes:
+
+.. code-block:: bash
+
+   TT_CLI_BUILD_SSL=no mage build
    mage build
 
+You can build a binary with statically linked OpenSSL. This build type is used
+for releases:
+
+.. code-block:: bash
+
+   TT_CLI_BUILD_SSL=static mage build
+
+Finally, you can build a binary with dynamically linked OpenSSL for development
+purposes:
+
+.. code-block:: bash
+
+   TT_CLI_BUILD_SSL=shared mage build
+
 ~~~~~~~~~~~~
 Dependencies
 ~~~~~~~~~~~~

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/spf13/cobra v1.3.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tarantool/cartridge-cli v0.0.0-20220605082730-53e6a5be9a61
-	github.com/tarantool/go-tarantool v1.9.0
+	github.com/tarantool/go-tarantool v1.10.1-0.20230309143354-e257ff30dd4d
 	github.com/vmihailenco/msgpack/v5 v5.3.5
 	github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64
 	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd
@@ -78,8 +78,8 @@ require (
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tarantool/go-openssl v0.0.8-0.20220711094538-d93c1eff4f49 // indirect
-	github.com/temoto/robotstxt v1.1.1 // indirect
+	github.com/tarantool/go-openssl v0.0.8-0.20230307065445-720eeb389195 // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.4 // indirect
 	github.com/tklauser/numcpus v0.2.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect

go.sum

@@ -704,6 +704,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/markphelps/optional v0.10.0/go.mod h1:Fvjs1vxcm7/wDqJPFGEiEM1RuxFl9GCyxQlj9M9YMAQ=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
@@ -1007,15 +1008,15 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/tarantool/go-openssl v0.0.8-0.20220711094538-d93c1eff4f49 h1:rZYYi1cI3QXZ3yRFZd2ItYM1XA2BaJqP0buDroMbjNo=
-github.com/tarantool/go-openssl v0.0.8-0.20220711094538-d93c1eff4f49/go.mod h1:M7H4xYSbzqpW/ZRBMyH0eyqQBsnhAMfsYk5mv0yid7A=
+github.com/tarantool/go-openssl v0.0.8-0.20230307065445-720eeb389195 h1:/AN3eUPsTlvF6W+Ng/8ZjnSU6o7L0H4Wb9GMks6RkzU=
+github.com/tarantool/go-openssl v0.0.8-0.20230307065445-720eeb389195/go.mod h1:M7H4xYSbzqpW/ZRBMyH0eyqQBsnhAMfsYk5mv0yid7A=
 github.com/tarantool/go-prompt v0.2.6-tarantool h1:/dYMRBuM5nE3mleka/mqJWPf8SrJ151U+OqDlTzvES0=
 github.com/tarantool/go-prompt v0.2.6-tarantool/go.mod h1:8enZKIgoGFEQu2XPBK79TguJG2XF3SR4QU2iYI28NSo=
-github.com/tarantool/go-tarantool v1.9.0 h1:qUotwiMNZhi9AvogF3uyswEN2NAFcKJOhcvXiE/P+Rs=
-github.com/tarantool/go-tarantool v1.9.0/go.mod h1:oPjvZNKaN4iKbf8YPo3pGpxzk6cRZF1neAxgq8WcBh8=
+github.com/tarantool/go-tarantool v1.10.1-0.20230309143354-e257ff30dd4d h1:qkKjWaxX8GPdHSP2WETv096vm/MH2iBPOZa6hnJ780s=
+github.com/tarantool/go-tarantool v1.10.1-0.20230309143354-e257ff30dd4d/go.mod h1:oMCTx0UAjEkHjdVdgnh3DjwUxs0xgyvIcaOBV8i4CF4=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
-github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
-github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
+github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
 github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=

magefile.go

@@ -17,6 +17,7 @@ import (
 )
 
 const (
+	buildTypeEnv = "TT_CLI_BUILD_SSL"
 	goPackageName = "github.com/tarantool/tt/cli"
 
 	asmflags = "all=-trimpath=${PWD}"
@@ -38,14 +39,25 @@ var (
 		"-X ${PACKAGE}/version.versionLabel=${VERSION_LABEL}",
 		"-X ${PACKAGE}/configure.defaultConfigPath=${CONFIG_PATH}",
 	}
-
+	staticLdflags = []string{
+		"-linkmode=external", "-extldflags", "-static",
+	}
 	goExecutableName     = "go"
 	pythonExecutableName = "python3"
 	ttExecutableName     = "tt"
 
 	generateModePath = filepath.Join(packagePath, "codegen", "generate_code.go")
 )
 
+type BuildType string
+
+const (
+	BuildTypeDefault BuildType = ""
+	BuildTypeNoCgo   BuildType = "no"
+	BuildTypeShared  BuildType = "shared"
+	BuildTypeStatic  BuildType = "static"
+)
+
 func init() {
 	var err error
 
@@ -60,7 +72,6 @@ func init() {
 			panic(err)
 		}
 	}
-
 	// We want to use Go 1.11 modules even if the source lives inside GOPATH.
 	// The default is "auto".
 	os.Setenv("GO111MODULE", "on")
@@ -143,18 +154,41 @@ func PatchCC() error {
 	return nil
 }
 
-// Building tt executable.
+// Building tt executable. Supported environment variables:
+// TT_CLI_BUILD_SSL=(no|static|shared)
 func Build() error {
 	fmt.Println("Building tt...")
 
 	mg.Deps(PatchCC)
 	mg.Deps(GenerateGoCode)
 
+	buildLdflags := make([]string, len(ldflags))
+	copy(buildLdflags, ldflags)
+	tags := "-tags=netgo,osusergo"
+
+	buildType := os.Getenv(buildTypeEnv)
+	switch BuildType(buildType) {
+	case BuildTypeDefault:
+		fallthrough
+	case BuildTypeNoCgo:
+		tags = tags + ",go_tarantool_ssl_disable"
+	case BuildTypeStatic:
+		if runtime.GOOS != "darwin" {
+			buildLdflags = append(buildLdflags, staticLdflags...)
+		}
+		tags = tags + ",openssl_static"
+	case BuildTypeShared:
+	default:
+		return fmt.Errorf("Unsupported build type: %s, supported: "+
+			"%s, %s, %s\n",
+			buildType, BuildTypeNoCgo, BuildTypeStatic, BuildTypeShared)
+	}
+
 	err := sh.RunWith(
 		getBuildEnvironment(), goExecutableName, "build",
 		"-o", ttExecutableName,
-		"-tags=go_tarantool_ssl_disable",
-		"-ldflags", strings.Join(ldflags, " "),
+		tags,
+		"-ldflags", strings.Join(buildLdflags, " "),
 		"-asmflags", asmflags,
 		"-gcflags", gcflags,
 		packagePath,
@@ -210,11 +244,11 @@ func Unit() error {
 	mg.Deps(GenerateGoCode)
 
 	if mg.Verbose() {
-		return sh.RunV(goExecutableName, "test", "-v", "-tags", "go_tarantool_ssl_disable",
+		return sh.RunV(goExecutableName, "test", "-v",
 			fmt.Sprintf("%s/...", packagePath))
 	}
 
-	return sh.RunV(goExecutableName, "test", "-tags", "go_tarantool_ssl_disable", fmt.Sprintf("%s/...", packagePath))
+	return sh.RunV(goExecutableName, "test", fmt.Sprintf("%s/...", packagePath))
 }
 
 // Run unit tests with a Tarantool instance integration.
@@ -225,11 +259,11 @@ func UnitFull() error {
 
 	if mg.Verbose() {
 		return sh.RunV(goExecutableName, "test", "-v", fmt.Sprintf("%s/...", packagePath),
-			"-tags", "integration,go_tarantool_ssl_disable")
+			"-tags", "integration")
 	}
 
 	return sh.RunV(goExecutableName, "test", fmt.Sprintf("%s/...", packagePath),
-		"-tags", "integration,go_tarantool_ssl_disable")
+		"-tags", "integration")
 }
 
 // Run integration tests, excluding slow tests.
@@ -320,6 +354,6 @@ func getBuildEnvironment() map[string]string {
 		"VERSION_LABEL": os.Getenv("VERSION_LABEL"),
 		"PWD":           currentDir,
 		"CONFIG_PATH":   getDefaultConfigPath(),
-		"CGO_ENABLED":   "0",
+		"CGO_ENABLED":   "1",
 	}
 }

test/conftest.py

@@ -40,6 +40,7 @@ def tt_cmd(session_tmpdir):
 
     build_env = os.environ.copy()
     build_env["TTEXE"] = tt_path
+    build_env["TT_CLI_BUILD_SSL"] = "static"
 
     process = subprocess.run(["mage", "-v", "build"], cwd=tt_base_path, env=build_env)
     assert process.returncode == 0, "Failed to build Tarantool CLI executable"