Use github releases for libs

lucasssvaz · lucasssvaz · commit 6d86a0be8fd8 · 2024-07-16T22:59:28.000-03:00
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -12,6 +12,7 @@ on:
 #             │ │ │ │ │
 #             * * * * *
     - cron:  '0 */6 * * *'
+  workflow_dispatch: # For testing
 
 defaults:
   run:
@@ -25,6 +26,7 @@ jobs:
       idf_branch: ${{ matrix.idf_branch }}
       lib_builder_branch: ${{ matrix.lib_builder_branch }}
       targets: ${{ matrix.targets }}
+    secrets: inherit
     strategy:
       fail-fast: false
       matrix:
diff --git a/.github/workflows/cron_build.yml b/.github/workflows/cron_build.yml
@@ -24,6 +24,20 @@ jobs:
     name: Check if deploy is needed for ${{ inputs.idf_branch }}
     runs-on: ubuntu-latest
     outputs:
+      idf_commit: ${{ steps.check.outputs.idf_commit }}
+      ar_branch: ${{ steps.check.outputs.ar_branch }}
+      ar_new_commit_message: ${{ steps.check.outputs.ar_new_commit_message }}
+      ar_new_branch_name: ${{ steps.check.outputs.ar_new_branch_name }}
+      ar_new_pr_title: ${{ steps.check.outputs.ar_new_pr_title }}
+      ar_has_commit: ${{ steps.check.outputs.ar_has_commit }}
+      ar_has_branch: ${{ steps.check.outputs.ar_has_branch }}
+      ar_has_pr: ${{ steps.check.outputs.ar_has_pr }}
+      libs_release_tag: ${{ steps.check.outputs.libs_release_tag }}
+      libs_version: ${{ steps.check.outputs.libs_version }}
+      libs_release_id: ${{ steps.check.outputs.libs_release_id }}
+      libs_has_release: ${{ steps.check.outputs.libs_has_release }}
+      libs_asset_id: ${{ steps.check.outputs.libs_asset_id }}
+      libs_has_asset: ${{ steps.check.outputs.libs_has_asset }}
       deploy_needed: ${{ steps.check.outputs.deploy_needed }}
       targets_list: ${{ steps.check.outputs.targets_list }}
     steps:
@@ -39,13 +53,12 @@ jobs:
           source ./tools/check-deploy-needed.sh
           targets_list=$(echo "${{ inputs.targets }}" | sed 's/ *, */,/g' | sed 's/^/["/' | sed 's/$/"]/' | sed 's/,/","/g')
           echo "Targets list: $targets_list"
-          echo "deploy_needed=$DEPLOY_NEEDED" >> $GITHUB_OUTPUT
           echo "targets_list=$targets_list" >> $GITHUB_OUTPUT
 
   build-libs:
     name: Build for ${{ matrix.target }} (${{ inputs.idf_branch }})
     runs-on: ubuntu-latest
-    if: needs.check-if-needed.outputs.deploy_needed == '1'
+    if: needs.check-if-needed.outputs.libs_has_asset == '0'
     needs: check-if-needed
     strategy:
       fail-fast: false
@@ -87,47 +100,69 @@ jobs:
         path: dist
 
   combine-artifacts:
-    name: Combine artifacts for IDF ${{ inputs.idf_branch }}
+    name: Combine artifacts and push changes for IDF ${{ inputs.idf_branch }}
     runs-on: ubuntu-latest
     needs: [check-if-needed, build-libs]
-    if: needs.check-if-needed.outputs.deploy_needed == '1'
+    if: |
+      always() &&
+      needs.check-if-needed.result == 'success' &&
+      needs.check-if-needed.outputs.deploy_needed == '1'
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.lib_builder_branch }}
 
       - name: Replace invalid characters in the artifact name
+        if: needs.check-if-needed.outputs.libs_has_asset == '0'
         run: |
           branch=${{ inputs.idf_branch }}
           echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV
 
       - name: Download artifacts
+        if: needs.check-if-needed.outputs.libs_has_asset == '0'
         uses: actions/download-artifact@v4
         with:
           path: dist
           pattern: libs-${{ env.libs_branch }}-*
           merge-multiple: true
 
       - name: Combine artifacts
+        if: needs.check-if-needed.outputs.libs_has_asset == '0'
         run: bash ./tools/combine-artifacts.sh
 
       - name: Upload full esp32-arduino-libs archive
+        if: needs.check-if-needed.outputs.libs_has_asset == '0'
         uses: actions/upload-artifact@v4
         with:
           name: esp32-arduino-libs-${{ env.libs_branch }}
-          path: dist/esp32-arduino-libs.tar.gz
-
-      - name: Upload package_esp32_index.template.json
-        uses: actions/upload-artifact@v4
-        with:
-          name: package-esp32-index-json-${{ env.libs_branch }}
-          path: dist/package_esp32_index.template.json
+          path: dist/esp32-arduino-libs.zip
+          compression-level: 0
 
       - name: Push changes
         if: github.repository == 'espressif/esp32-arduino-lib-builder'
         env:
           GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
           GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
           GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
+          IDF_COMMIT: ${{ needs.check-if-needed.outputs.idf_commit }}
+          AR_BRANCH: ${{ needs.check-if-needed.outputs.ar_branch }}
+          AR_NEW_COMMIT_MESSAGE: ${{ needs.check-if-needed.outputs.ar_new_commit_message }}
+          AR_NEW_BRANCH_NAME: ${{ needs.check-if-needed.outputs.ar_new_branch_name }}
+          AR_NEW_PR_TITLE: ${{ needs.check-if-needed.outputs.ar_new_pr_title }}
+          AR_HAS_COMMIT: ${{ needs.check-if-needed.outputs.ar_has_commit }}
+          AR_HAS_BRANCH: ${{ needs.check-if-needed.outputs.ar_has_branch }}
+          AR_HAS_PR: ${{ needs.check-if-needed.outputs.ar_has_pr }}
+          LIBS_RELEASE_TAG: ${{ needs.check-if-needed.outputs.libs_release_tag }}
+          LIBS_VERSION: ${{ needs.check-if-needed.outputs.libs_version }}
+          LIBS_RELEASE_ID: ${{ needs.check-if-needed.outputs.libs_release_id }}
+          LIBS_HAS_RELEASE: ${{ needs.check-if-needed.outputs.libs_has_release }}
+          LIBS_ASSET_ID: ${{ needs.check-if-needed.outputs.libs_asset_id }}
+          LIBS_HAS_ASSET: ${{ needs.check-if-needed.outputs.libs_has_asset }}
         run: |
           bash ./tools/push-to-arduino.sh
+
+      - name: Upload package_esp32_index.template.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-esp32-index-json-${{ env.libs_branch }}
+          path: out/package_esp32_index.template.json
diff --git a/tools/check-deploy-needed.sh b/tools/check-deploy-needed.sh
@@ -36,7 +36,8 @@ AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH"
 AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT"
 AR_NEW_PR_TITLE="IDF $IDF_BRANCH"
 
-LIBS_VERSION="idf-"${IDF_BRANCH//\//_}"-$IDF_COMMIT"
+LIBS_RELEASE_TAG="idf-"${IDF_BRANCH//\//_}""
+LIBS_VERSION="$LIBS_RELEASE_TAG-$IDF_COMMIT"
 
 AR_HAS_BRANCH=`github_branch_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME"`
 if [ "$AR_HAS_BRANCH" == "1" ]; then
@@ -46,8 +47,10 @@ else
 fi
 AR_HAS_PR=`github_pr_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME"`
 
-LIBS_HAS_BRANCH=`github_branch_exists "$AR_LIBS_REPO" "$AR_NEW_BRANCH_NAME"`
-LIBS_HAS_COMMIT=`github_commit_exists "$AR_LIBS_REPO" "$AR_NEW_BRANCH_NAME" "$IDF_COMMIT"`
+LIBS_RELEASE_ID=`github_release_id "$AR_LIBS_REPO" "$LIBS_RELEASE_TAG"`
+LIBS_HAS_RELEASE=`if [ -n "$LIBS_RELEASE_ID" ]; then echo "1"; else echo "0"; fi`
+LIBS_ASSET_ID=`github_release_asset_id "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "esp32-arduino-libs-$LIBS_VERSION.zip"`
+LIBS_HAS_ASSET=`if [ -n "$LIBS_ASSET_ID" ]; then echo "1"; else echo "0"; fi`
 
 export IDF_COMMIT
 
@@ -59,9 +62,25 @@ export AR_HAS_COMMIT
 export AR_HAS_BRANCH
 export AR_HAS_PR
 
+export LIBS_RELEASE_TAG
 export LIBS_VERSION
-export LIBS_HAS_COMMIT
-export LIBS_HAS_BRANCH
+export LIBS_RELEASE_ID
+export LIBS_HAS_RELEASE
+export LIBS_ASSET_ID
+export LIBS_HAS_ASSET
+
+if [ "$LIBS_HAS_RELEASE" == "1" ]; then
+	if [ "$LIBS_HAS_ASSET" == "0" ] || [ "$AR_HAS_COMMIT" == "0" ]; then
+		echo "Deploy needed"
+		export DEPLOY_NEEDED="1"
+	else
+		echo "Deploy not needed. Skipping..."
+		export DEPLOY_NEEDED="0"
+	fi
+else
+	echo "Release for tag \"$LIBS_RELEASE_TAG\" not found. Please create the release first."
+	exit 1
+fi
 
 echo "IDF_COMMIT: $IDF_COMMIT"
 echo "AR_BRANCH: $AR_BRANCH"
@@ -71,9 +90,13 @@ echo "AR_NEW_PR_TITLE: $AR_NEW_PR_TITLE"
 echo "AR_HAS_COMMIT: $AR_HAS_COMMIT"
 echo "AR_HAS_BRANCH: $AR_HAS_BRANCH"
 echo "AR_HAS_PR: $AR_HAS_PR"
+echo "LIBS_RELEASE_TAG: $LIBS_RELEASE_TAG"
 echo "LIBS_VERSION: $LIBS_VERSION"
-echo "LIBS_HAS_COMMIT: $LIBS_HAS_COMMIT"
-echo "LIBS_HAS_BRANCH: $LIBS_HAS_BRANCH"
+echo "LIBS_RELEASE_ID: $LIBS_RELEASE_ID"
+echo "LIBS_HAS_RELEASE: $LIBS_HAS_RELEASE"
+echo "LIBS_ASSET_ID: $LIBS_ASSET_ID"
+echo "LIBS_HAS_ASSET: $LIBS_HAS_ASSET"
+echo "DEPLOY_NEEDED: $DEPLOY_NEEDED"
 
 if [ ! -x $GITHUB_OUTPUT ]; then
 	echo "idf_commit=$IDF_COMMIT" >> "$GITHUB_OUTPUT"
@@ -84,15 +107,12 @@ if [ ! -x $GITHUB_OUTPUT ]; then
 	echo "ar_has_commit=$AR_HAS_COMMIT" >> "$GITHUB_OUTPUT"
 	echo "ar_has_branch=$AR_HAS_BRANCH" >> "$GITHUB_OUTPUT"
 	echo "ar_has_pr=$AR_HAS_PR" >> "$GITHUB_OUTPUT"
+	echo "libs_release_tag=$LIBS_RELEASE_TAG" >> "$GITHUB_OUTPUT"
 	echo "libs_version=$LIBS_VERSION" >> "$GITHUB_OUTPUT"
-	echo "libs_has_commit=$LIBS_HAS_COMMIT" >> "$GITHUB_OUTPUT"
-	echo "libs_has_branch=$LIBS_HAS_BRANCH" >> "$GITHUB_OUTPUT"
+	echo "libs_release_id=$LIBS_RELEASE_ID" >> "$GITHUB_OUTPUT"
+	echo "libs_has_release=$LIBS_HAS_RELEASE" >> "$GITHUB_OUTPUT"
+	echo "libs_asset_id=$LIBS_ASSET_ID" >> "$GITHUB_OUTPUT"
+	echo "libs_has_asset=$LIBS_HAS_ASSET" >> "$GITHUB_OUTPUT"
+	echo "deploy_needed=$DEPLOY_NEEDED" >> "$GITHUB_OUTPUT"
 fi
 
-if [ "$LIBS_HAS_COMMIT" == "0" ] || [ "$AR_HAS_COMMIT" == "0" ]; then
-	echo "Deploy needed"
-	export DEPLOY_NEEDED="1"
-else
-	echo "Deploy not needed. Skipping..."
-	export DEPLOY_NEEDED="0"
-fi
diff --git a/tools/combine-artifacts.sh b/tools/combine-artifacts.sh
@@ -16,5 +16,4 @@ done
 awk -i inplace '!seen[$0]++' $libs_folder/versions_full.txt
 mv -f $libs_folder/versions_full.txt $libs_folder/versions.txt
 
-cd $libs_folder && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
-cp out/package_esp32_index.template.json dist/package_esp32_index.template.json
+cd out/tools && zip -r ../../dist/esp32-arduino-libs.zip * && cd ../..
diff --git a/tools/config.sh b/tools/config.sh
@@ -50,7 +50,6 @@ AR_GEN_PART_PY="$AR_TOOLS/gen_esp32part.py"
 AR_SDK="$AR_TOOLS/esp32-arduino-libs/$IDF_TARGET"
 PIO_SDK="FRAMEWORK_SDK_DIR, \"$IDF_TARGET\""
 TOOLS_JSON_OUT="$AR_TOOLS/esp32-arduino-libs"
-IDF_LIBS_DIR="$AR_ROOT/../esp32-arduino-libs"
 
 if [ -d "$IDF_PATH" ]; then
     export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
@@ -132,6 +131,37 @@ function github_pr_exists(){ # github_pr_exists <repo-path> <branch-name>
     if [ ! "$pr_num" == "" ] && [ ! "$pr_num" == "null" ]; then echo 1; else echo 0; fi
 }
 
+function github_release_id(){ # github_release_id <repo-path> <release-tag>
+    local repo_path="$1"
+    local release_tag="$2"
+    local release=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/releases" | jq --arg release_tag "$release_tag" -r '.[] | select(.tag_name == $release_tag) | .id'`
+    if [ ! "$release" == "" ] && [ ! "$release" == "null" ]; then echo "$release"; else echo ""; fi
+}
+
+function github_release_asset_id(){ # github_release_asset_id <repo-path> <release-id> <release-file>
+    local repo_path="$1"
+    local release_id="$2"
+    local release_file="$3"
+    local release_asset=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/releases/$release_id/assets" | jq --arg release_file "$release_file" -r '.[] | select(.name == $release_file) | .id'`
+    if [ ! "$release_asset" == "" ] && [ ! "$release_asset" == "null" ]; then echo "$release_asset"; else echo ""; fi
+}
+
+function github_release_asset_upload(){ # github_release_asset_upload <repo-path> <release-id> <release-file-name> <release-file-path>
+    local repo_path="$1"
+    local release_id="$2"
+    local release_file_name="$3"
+    local release_file_path="$4"
+    local file_extension="${release_file_name##*.}"
+    local release_asset=`curl -s -k -X POST -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -H "Content-Type: application/$file_extension" --data-binary "@$release_file_path" "https://uploads.github.com/repos/$repo_path/releases/$release_id/assets?name=$release_file_name" | jq -r '.id'`
+    if [ ! "$release_asset" == "" ] && [ ! "$release_asset" == "null" ]; then echo "$release_asset"; else echo ""; fi
+}
+
+function github_release_asset_delete(){ # github_release_asset_delete <repo-path> <release-asset-id>
+    local repo_path="$1"
+    local release_asset_id="$2"
+    local res=$(curl -s -k -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/releases/assets/$release_asset_id")
+    if [ "$res" -eq 204 ]; then echo 1; else echo 0; fi
+}
 
 
 function git_branch_exists(){ # git_branch_exists <repo-path> <branch-name>
diff --git a/tools/install-arduino.sh b/tools/install-arduino.sh
@@ -46,17 +46,3 @@ if [ "$AR_BRANCH" ]; then
 	git -C "$AR_COMPS/arduino" pull --ff-only
 fi
 if [ $? -ne 0 ]; then exit 1; fi
-
-#
-# CLONE/UPDATE ESP32-ARDUINO-LIBS
-#
-if [ ! -d "$IDF_LIBS_DIR" ]; then
-	echo "Cloning esp32-arduino-libs..."
-	git clone "$AR_LIBS_REPO_URL" "$IDF_LIBS_DIR"
-else
-	echo "Updating esp32-arduino-libs..."
-	git -C "$IDF_LIBS_DIR" fetch && \
-	git -C "$IDF_LIBS_DIR" pull --ff-only
-fi
-if [ $? -ne 0 ]; then exit 1; fi
-
diff --git a/tools/push-to-arduino.sh b/tools/push-to-arduino.sh
