From a49c1b1187b3f25d714986b5e0d7c415bd429937 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 2 May 2019 00:35:34 +0300 Subject: [PATCH 01/11] Fix values of mismatch detection variables carrying over to the next iteration Happened in https://travis-ci.org/skvark/opencv-python/builds/518836739 S1 jobs --- travis_osx_brew_cache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 43625c41..6a270678 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -106,8 +106,8 @@ function brew_add_local_bottles { local FORMULA_HAS_BOTTLE; [ -n "$FORMULA_BOTTLE_HASH" ] && FORMULA_HAS_BOTTLE=1 || true - local BOTTLE_LINK BOTTLE; BOTTLE_LINK="${JSON}.bottle.lnk"; - local BOTTLE_EXISTS BOTTLE_MISMATCH VERSION_MISMATCH + local BOTTLE_LINK BOTTLE=""; BOTTLE_LINK="${JSON}.bottle.lnk"; + local BOTTLE_EXISTS= BOTTLE_MISMATCH= VERSION_MISMATCH= # Check that the bottle file exists and is still appropriate for the formula From 3844f7aa834518c2eb2a7b3cde357b220ac1adda Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 2 May 2019 00:46:02 +0300 Subject: [PATCH 02/11] Fix removing old built opencv_ffmpeg when a new upstream formula is available Happened in https://travis-ci.org/skvark/opencv-python/builds/518179810 S1 jobs --- config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.sh b/config.sh index c31def77..2a7e8ec1 100644 --- a/config.sh +++ b/config.sh @@ -92,6 +92,7 @@ function pre_build { #after the cache stage, all bottles and Homebrew metadata should be already cached locally if [ -n "$CACHE_STAGE" ]; then brew update + generate_ffmpeg_formula brew_add_local_bottles fi @@ -107,7 +108,6 @@ function pre_build { echo 'Installing FFmpeg' if [ -n "$CACHE_STAGE" ]; then - generate_ffmpeg_formula brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } else brew install ffmpeg_opencv From 144e61c15251c09f596f842e5d25825aa100670d Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 2 May 2019 01:19:18 +0300 Subject: [PATCH 03/11] Don't print bogus error message when cache warmup is complete --- travis_osx_brew_cache.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 6a270678..eb15618c 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -223,12 +223,19 @@ function brew_go_bootstrap_mode { #Can't just `exit` because that would terminate the build without saving the cache #Have to replace further actions with no-ops + local MESSAGE=""; if [ "$EXIT_CODE" -ne 0 ]; then + MESSAGE='Building dependencies took too long. Restart the build in Travis UI to continue from cache.'; + fi + eval ' function '"$cmd"' { return 0; } function repair_wheelhouse { return 0; } - function install_run { - echo -e "\nBuilding dependencies took too long. Restart the build in Travis UI to continue from cache.\n" - + function install_run {'\ + "$(if [ -n "$MESSAGE" ]; then + echo \ + ' echo -e "\n'"$MESSAGE"'\n"' + fi)"\ + ' # Travis runs user scripts via `eval` i.e. in the same shell process. # So have to unset errexit in order to get to cache save stage set +e; return '"$EXIT_CODE"' From 583d1266b47ac5527ba34d9118add325577a333e Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 2 May 2019 04:12:14 +0300 Subject: [PATCH 04/11] Fix "Error: broken pipe" if `brew info` gives lots of output --- travis_osx_brew_cache.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index eb15618c..65d4f0a9 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -307,7 +307,9 @@ function _brew_is_bottle_available { local PACKAGE;PACKAGE="${1:?}" local VAR_KEG_ONLY="$2" - local INFO;INFO="$(brew info "$PACKAGE" | head -n 1)" + # `brew info` prints "Error: Broken pipe" if piped directly to `head` and the info is long + # 141 = 128 + SIGPIPE + local INFO;INFO="$((brew info "$PACKAGE" | cat || test $? -eq 141) | head -n 1)" if [ -n "$VAR_KEG_ONLY" ]; then if grep -qwF '[keg-only]' <<<"$INFO"; then eval "${VAR_KEG_ONLY}=1" From d0dbb0513b28122fc6cfdf126e344064f2077de8 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 2 May 2019 05:22:08 +0300 Subject: [PATCH 05/11] CentOS 5 ccache doesn't have --show-stats --- config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.sh b/config.sh index 2a7e8ec1..af888179 100644 --- a/config.sh +++ b/config.sh @@ -16,7 +16,7 @@ function bdist_wheel_cmd { local abs_wheelhouse=$1 python setup.py bdist_wheel $BDIST_PARAMS cp dist/*.whl $abs_wheelhouse - if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache --show-stats; fi + if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache -s; fi } if [ -n "$IS_OSX" ]; then From d4a51981676eacd6763fac5b183e6267a4305610 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 2 May 2019 02:29:59 +0300 Subject: [PATCH 06/11] Use latest cmake for the main build, too, with better Python support --- config.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config.sh b/config.sh index af888179..3370d221 100644 --- a/config.sh +++ b/config.sh @@ -111,6 +111,10 @@ function pre_build { brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } else brew install ffmpeg_opencv + #cmake is a build dependency of ffmpeg so a bottle should be present in cache + if (brew outdated | grep -qxF cmake) || ! (brew list | grep -qxF cmake); then + _brew_install_and_cache cmake 1 0 + fi fi if [ -n "$CACHE_STAGE" ]; then From e1523c845ed4f17029d7a97671ca49b630d3c6c7 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 2 May 2019 23:55:12 +0300 Subject: [PATCH 07/11] JSON format change for formula without any bottles --- travis_osx_brew_cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 65d4f0a9..5eaefe42 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -285,7 +285,7 @@ function _brew_parse_package_info { revision=data["revision"] # in bottle''s json, revision is included into version; here, they are separate print data["versions"]["stable"]+("_"+str(revision) if revision else "") - bottle_data=data["bottle"]["stable"] + bottle_data=data["bottle"].get("stable",{"rebuild":"","files":{}}) print bottle_data["rebuild"] print bottle_data["files"].get(sys.argv[2],{"sha256":"!?"})["sha256"] #prevent losing trailing blank line to command substitution ' \ From c4286d26c3f9763a502e17c852855a4303e4c771 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Fri, 3 May 2019 22:34:38 +0300 Subject: [PATCH 08/11] Speed up repeat checks for the same deps --- travis_osx_brew_cache.sh | 85 +++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 5eaefe42..e3a69cc3 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -27,7 +27,8 @@ export HOMEBREW_NO_INSTALL_CLEANUP=1 # see https://docs.brew.sh/Manpage , "info formula" section export HOMEBREW_NO_GITHUB_API=1 - +#Packages already installed in the current session to avoid checking them again +_BREW_ALREADY_INSTALLED='$' #$ = illegal package name; a blank line would cause macos grep to swallow everything #Public functions @@ -37,39 +38,8 @@ function brew_install_and_cache_within_time_limit { # use bottle if available, build and cache bottle if not. # Terminate and exit with status 1 if this takes too long. # Exit with status 2 on any other error. - ( set -eE -o pipefail; trap '{ sleep 3; exit 2; }' ERR - - local PACKAGE TIME_LIMIT TIME_HARD_LIMIT TIME_START - PACKAGE="${1:?}" || exit 2 - TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || exit 2 - TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || exit 2 - TIME_START=${4:-$BREW_TIME_START} || exit 2 - - local BUILD_FROM_SOURCE INCLUDE_BUILD KEG_ONLY - - if brew list --versions "$PACKAGE" >/dev/null && ! (brew outdated | grep -qxF "$PACKAGE"); then - echo "Already installed and the latest version: $PACKAGE" - return 0 - fi - - - _brew_is_bottle_available "$PACKAGE" KEG_ONLY || BUILD_FROM_SOURCE=1 - [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true - - # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay. - DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || exit 2 - for dep in $DEPS; do - #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation - #While we don't know that yet, we can make better use of Travis-given time with a laxer limit - #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache - brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || exit $? - done - - _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || exit $? - _brew_install_and_cache "$PACKAGE" "$([[ -z "$BUILD_FROM_SOURCE" ]] && echo 1 || echo 0)" "$KEG_ONLY" || exit 2 - _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || exit $? - ) \ - || if test $? -eq 1; then brew_go_bootstrap_mode; return 1; else return 2; fi #must run this in current process + _brew_install_and_cache_within_time_limit $@ \ + || if test $? -eq 1; then brew_go_bootstrap_mode; return 1; else return 2; fi } function brew_add_local_bottles { @@ -246,6 +216,46 @@ function brew_go_bootstrap_mode { #Internal functions +function _brew_install_and_cache_within_time_limit { + # This fn is run with || so errexit can't be enabled + + local PACKAGE TIME_LIMIT TIME_HARD_LIMIT TIME_START MARKED_INSTALLED + PACKAGE="${1:?}" || return 2 + TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || return 2 + TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || return 2 + TIME_START=${4:-$BREW_TIME_START} || return 2 + + if grep -qxFf <(cat <<<"$_BREW_ALREADY_INSTALLED") <<<"$PACKAGE"; then + MARKED_INSTALLED=1 + fi + + if [ -n "$MARKED_INSTALLED" ] || (brew list --versions "$PACKAGE" >/dev/null && ! (brew outdated | grep -qxF "$PACKAGE")); then + echo "Already installed and the latest version: $PACKAGE" + if [ -z "$MARKED_INSTALLED" ]; then _brew_mark_installed "$PACKAGE"; fi + return 0 + fi + + local BUILD_FROM_SOURCE INCLUDE_BUILD KEG_ONLY + + _brew_is_bottle_available "$PACKAGE" KEG_ONLY || BUILD_FROM_SOURCE=1 + [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true + + # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay. + DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || return 2 + DEPS=`grep -vxF <(cat <<<"$_BREW_ALREADY_INSTALLED") <<<"$DEPS"` || test $? -eq 1 || return 2 + for dep in $DEPS; do + #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation + #While we don't know that yet, we can make better use of Travis-given time with a laxer limit + #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache + _brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || return $? + done + + _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || return $? + _brew_install_and_cache "$PACKAGE" "$([[ -z "$BUILD_FROM_SOURCE" ]] && echo 1 || echo 0)" "$KEG_ONLY" || return 2 + _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $? +} + + function _brew_parse_bottle_json { # Parse JSON file resulting from `brew bottle --json` # and save data into specified variables @@ -386,10 +396,13 @@ function _brew_install_and_cache { echo "$CACHED_BOTTLE" >"$BOTTLE_LINK" fi + + _brew_mark_installed "$PACKAGE" } - - +function _brew_mark_installed { + _BREW_ALREADY_INSTALLED="$_BREW_ALREADY_INSTALLED"$'\n'"${1:?}" +} function _brew_check_elapsed_build_time { # If time limit has been reached, From 42b35191fae5c41cf028ae627ca47033e06ff1ea Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 4 May 2019 11:02:49 +0300 Subject: [PATCH 09/11] Add prefix to all files related to CI rather than project --- .travis.yml | 3 ++- config.sh => travis_config.sh | 0 multibuild_customize.sh => travis_multibuild_customize.sh | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename config.sh => travis_config.sh (100%) rename multibuild_customize.sh => travis_multibuild_customize.sh (100%) diff --git a/.travis.yml b/.travis.yml index 34d989f8..81d2331e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ env: - TEST_DEPENDS="numpy==1.11.1" # params to bdist_wheel. used to set osx build target. - BDIST_PARAMS="" + - CONFIG_PATH="travis_config.sh" - USE_CCACHE=1 - PLAT=x86_64 - UNICODE_WIDTH=32 @@ -663,7 +664,7 @@ before_install: | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi source multibuild/travis_steps.sh # This sets -x - source multibuild_customize.sh + source travis_multibuild_customize.sh echo $ENABLE_CONTRIB > contrib.enabled echo $ENABLE_HEADLESS > headless.enabled diff --git a/config.sh b/travis_config.sh similarity index 100% rename from config.sh rename to travis_config.sh diff --git a/multibuild_customize.sh b/travis_multibuild_customize.sh similarity index 100% rename from multibuild_customize.sh rename to travis_multibuild_customize.sh From d6a45ba2ae322562e8731c27695b6e00528862f7 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 9 May 2019 01:14:29 +0300 Subject: [PATCH 10/11] (dummy commit to rerun checks) From fac2703ffcc7ad810ce9a0af119d95bd03048fb9 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 13 Jun 2019 19:57:44 +0300 Subject: [PATCH 11/11] Revert "Use latest cmake for the main build, too, with better Python support This reverts commit d4a51981676eacd6763fac5b183e6267a4305610. cmake 3.14 can't find py3 installation when "python" points to Python 3 while cmake 3.9 can --- travis_config.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/travis_config.sh b/travis_config.sh index 3370d221..af888179 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -111,10 +111,6 @@ function pre_build { brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } else brew install ffmpeg_opencv - #cmake is a build dependency of ffmpeg so a bottle should be present in cache - if (brew outdated | grep -qxF cmake) || ! (brew list | grep -qxF cmake); then - _brew_install_and_cache cmake 1 0 - fi fi if [ -n "$CACHE_STAGE" ]; then