From c523797ce2ec89a090185048b0190d992572f547 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Fri, 28 Apr 2023 21:00:30 +0300 Subject: [PATCH 1/3] Autogenerate PlatformIO manifest file for prebuilt SDK libs - Add a special Python script that generates "package.json" with IDF revision from the "version.txt" according to SemVer --- build.sh | 8 ++- tools/gen_platformio_manifest.py | 95 ++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tools/gen_platformio_manifest.py diff --git a/build.sh b/build.sh index a5ea518e2..086554845 100755 --- a/build.sh +++ b/build.sh @@ -197,6 +197,12 @@ if [ "$BUILD_TYPE" = "all" ]; then if [ $? -ne 0 ]; then exit 1; fi fi +# Generate PlatformIO manifest file +if [ "$BUILD_TYPE" = "all" ]; then + python3 ./tools/gen_platformio_manifest.py -o "$TOOLS_JSON_OUT/" + if [ $? -ne 0 ]; then exit 1; fi +fi + # copy everything to arduino-esp32 installation if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then ./tools/copy-to-arduino.sh @@ -212,4 +218,4 @@ fi if [ "$BUILD_TYPE" = "all" ]; then ./tools/archive-build.sh if [ $? -ne 0 ]; then exit 1; fi -fi +fi diff --git a/tools/gen_platformio_manifest.py b/tools/gen_platformio_manifest.py new file mode 100644 index 000000000..a0a1d162b --- /dev/null +++ b/tools/gen_platformio_manifest.py @@ -0,0 +1,95 @@ +import argparse +import json +import os +import re +import sys + +MANIFEST_DATA = { + "name": "framework-arduinoespressif32-libs", + "description": "Precompiled libraries for Arduino Wiring-based Framework for the Espressif ESP32 series of SoCs", + "keywords": ["framework", "arduino", "espressif", "esp32"], + "license": "LGPL-2.1-or-later", + "repository": { + "type": "git", + "url": "https://github.com/espressif/esp32-arduino-libs", + }, +} + + +def convert_version(version_line): + """A helper function that converts a custom IDF version string + to a suitable SemVer alternative. For example: + 'release/v5.1 420ebd208a' becomes '5.1.0+sha.420ebd208a' + """ + + regex_pattern = r"^esp-idf:\s*release\/v(?P[\d\.]{3,5})\s*(?P[0-9a-f]{5,40})" + match = re.search(regex_pattern, version_line) + if not match: + sys.stderr.write( + f"Failed to find a regex match for '{regex_pattern}' in '{version_line}'\n" + ) + return "" + + version = match.group("IDF_VERSION") + commit = match.group("COMMIT_HASH") + + assert version, f"Failed to parse version value from '{version_line}'" + assert commit, f"Failed to parse commit hash value from '{version_line}'" + + if version.count(".") < 2: + # The most basic casting to a SemVer with three digits + version = version + ".0" + + return f"{version}+sha.{commit}" + + +def main(dst_dir): + # The "version.txt" file is expected to contain IDF version in the following format + # "esp-idf: release/v$VERSION COMMIT_HASH". + version_file = os.path.join("version.txt") + + if not os.path.isfile(version_file): + sys.stderr.write("Missing the 'version.txt' file.\n") + return -1 + + version_line = "" + with open(version_file, encoding="utf8") as fp: + for line in fp.readlines(): + if not line.startswith("esp-idf"): + continue + version_line = line.strip() + + if not version_line: + sys.stderr.write("Failed to find ESP-IDF version in the 'version.txt' file!\n") + return -1 + + converted_version = convert_version(version_line) + if not converted_version: + sys.stderr.write( + f"Failed to convert version '{version_line}' from version.txt\n" + ) + return -1 + + manifest_file_path = os.path.join(dst_dir, "package.json") + with open(manifest_file_path, "w", encoding="utf8") as fp: + MANIFEST_DATA["version"] = converted_version + json.dump(MANIFEST_DATA, fp, indent=2) + + print( + f"Generated PlatformIO manifest file '{manifest_file_path}' with '{converted_version}' version" + ) + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-o", + "--dst-dir", + dest="dst_dir", + required=True, + help="Destination folder where the 'package.json' manifest will be located", + ) + args = parser.parse_args() + + sys.exit(main(args.dst_dir)) From 87d3bfb0a1727fb0f840733ce2717a8778452074 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Sat, 29 Apr 2023 12:24:50 +0300 Subject: [PATCH 2/3] Tidy up --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 086554845..139503560 100755 --- a/build.sh +++ b/build.sh @@ -218,4 +218,4 @@ fi if [ "$BUILD_TYPE" = "all" ]; then ./tools/archive-build.sh if [ $? -ne 0 ]; then exit 1; fi -fi +fi From a69e4ea4f6412227a42bd854025734df3f72955e Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 2 May 2023 16:39:56 +0300 Subject: [PATCH 3/3] Refactor manifest generator Now IDF version and commit hash are passed directly from Git client instead of reading from a pregenerated "version.txt" file --- build.sh | 2 +- tools/gen_platformio_manifest.py | 73 ++++++++++++++------------------ 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/build.sh b/build.sh index 139503560..7465f04be 100755 --- a/build.sh +++ b/build.sh @@ -199,7 +199,7 @@ fi # Generate PlatformIO manifest file if [ "$BUILD_TYPE" = "all" ]; then - python3 ./tools/gen_platformio_manifest.py -o "$TOOLS_JSON_OUT/" + python3 ./tools/gen_platformio_manifest.py -o "$TOOLS_JSON_OUT/" -s $(git -C "$IDF_PATH" symbolic-ref --short HEAD || git -C "$IDF_PATH" tag --points-at HEAD) -c $(git -C "$IDF_PATH" rev-parse --short HEAD) if [ $? -ne 0 ]; then exit 1; fi fi diff --git a/tools/gen_platformio_manifest.py b/tools/gen_platformio_manifest.py index a0a1d162b..2d031b687 100644 --- a/tools/gen_platformio_manifest.py +++ b/tools/gen_platformio_manifest.py @@ -16,63 +16,40 @@ } -def convert_version(version_line): +def convert_version(version_string): """A helper function that converts a custom IDF version string - to a suitable SemVer alternative. For example: - 'release/v5.1 420ebd208a' becomes '5.1.0+sha.420ebd208a' + extracted from a Git repository to a suitable SemVer alternative. For example: + 'release/v5.1' becomes '5.1.0', + 'v7.7.7' becomes '7.7.7' """ - regex_pattern = r"^esp-idf:\s*release\/v(?P[\d\.]{3,5})\s*(?P[0-9a-f]{5,40})" - match = re.search(regex_pattern, version_line) + regex_pattern = ( + r"v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.*(?P0|[1-9]\d*)*" + ) + match = re.search(regex_pattern, version_string) if not match: sys.stderr.write( - f"Failed to find a regex match for '{regex_pattern}' in '{version_line}'\n" + f"Failed to find a regex match for '{regex_pattern}' in '{version_string}'\n" ) return "" - version = match.group("IDF_VERSION") - commit = match.group("COMMIT_HASH") - - assert version, f"Failed to parse version value from '{version_line}'" - assert commit, f"Failed to parse commit hash value from '{version_line}'" - - if version.count(".") < 2: - # The most basic casting to a SemVer with three digits - version = version + ".0" - - return f"{version}+sha.{commit}" + major, minor, patch = match.groups() + if not patch: + patch = "0" + return ".".join((major, minor, patch)) -def main(dst_dir): - # The "version.txt" file is expected to contain IDF version in the following format - # "esp-idf: release/v$VERSION COMMIT_HASH". - version_file = os.path.join("version.txt") - if not os.path.isfile(version_file): - sys.stderr.write("Missing the 'version.txt' file.\n") - return -1 - - version_line = "" - with open(version_file, encoding="utf8") as fp: - for line in fp.readlines(): - if not line.startswith("esp-idf"): - continue - version_line = line.strip() - - if not version_line: - sys.stderr.write("Failed to find ESP-IDF version in the 'version.txt' file!\n") - return -1 +def main(dst_dir, version_string, commit_hash): - converted_version = convert_version(version_line) + converted_version = convert_version(version_string) if not converted_version: - sys.stderr.write( - f"Failed to convert version '{version_line}' from version.txt\n" - ) + sys.stderr.write(f"Failed to convert version '{version_string}'\n") return -1 manifest_file_path = os.path.join(dst_dir, "package.json") with open(manifest_file_path, "w", encoding="utf8") as fp: - MANIFEST_DATA["version"] = converted_version + MANIFEST_DATA["version"] = f"{converted_version}+sha.{commit_hash}" json.dump(MANIFEST_DATA, fp, indent=2) print( @@ -90,6 +67,20 @@ def main(dst_dir): required=True, help="Destination folder where the 'package.json' manifest will be located", ) + parser.add_argument( + "-s", + "--version-string", + dest="version_string", + required=True, + help="ESP-IDF version string used for compiling libraries", + ) + parser.add_argument( + "-c", + "--commit-hash", + dest="commit_hash", + required=True, + help="ESP-IDF revision in form of a commit hash", + ) args = parser.parse_args() - sys.exit(main(args.dst_dir)) + sys.exit(main(args.dst_dir, args.version_string, args.commit_hash))