Skip to content

Commit 9871401

Browse files
adriaanmadriaanm
committed
WIP: New PR validation
1 parent db52438 commit 9871401

File tree

4 files changed

+217
-71
lines changed

4 files changed

+217
-71
lines changed

‎scripts/common

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# This is for forcibly stopping the job from a subshell (see test
2+
# below).
3+
trap "exit 1" TERM
4+
export TOP_PID=$$
5+
set -e
6+
7+
# Known problems : does not fare well with interrupted, partial
8+
# compilations. We should perhaps have a multi-dependency version
9+
# of do_i_have below
10+
11+
LOGGINGDIR="$WORKSPACE/logs"
12+
mkdir -p $LOGGINGDIR
13+
14+
unset SBT_HOME
15+
SBT_HOME="$WORKSPACE/.sbt"
16+
mkdir -p $SBT_HOME
17+
IVY_CACHE="$WORKSPACE/.ivy2"
18+
mkdir -p $IVY_CACHE
19+
rm -rf $IVY_CACHE/cache/org.scala-lang
20+
21+
# temp dir where all 'non-build' operation are performed
22+
TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX)
23+
TMP_DIR="${TMP_ROOT_DIR}/tmp"
24+
mkdir "${TMP_DIR}"
25+
26+
27+
# detect sed version and how to enable extended regexes
28+
SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)"
29+
30+
31+
32+
# :docstring test:
33+
# Usage: test <argument ..>
34+
# Executes <argument ..>, logging the launch of the command to the
35+
# main log file, and kills global script execution with the TERM
36+
# signal if the commands ends up failing.
37+
# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES,
38+
# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST
39+
# :end docstring:
40+
41+
function test() {
42+
echo "### $@"
43+
"$@"
44+
status=$?
45+
if [ $status -ne 0 ]; then
46+
say "### ERROR with $1"
47+
kill -s TERM $TOP_PID
48+
fi
49+
}
50+
51+
# :docstring say:
52+
# Usage: say <argument ..>
53+
# Prints <argument ..> to both console and the main log file.
54+
# :end docstring:
55+
56+
function say(){
57+
(echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log
58+
}
59+
60+
# General debug logging
61+
# $* - message
62+
function debug () {
63+
echo "----- $*"
64+
}
65+
66+
function parseScalaProperties(){
67+
propFile="$baseDir/$1"
68+
if [ ! -f $propFile ]; then
69+
echo "Property file $propFile not found."
70+
exit 1
71+
else
72+
awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh"
73+
. "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again)
74+
fi
75+
}
76+
77+
78+
## TAKEN FROM UBER-BUILD, except that it "returns" (via $RES) true/false
79+
# Check if an artifact is available
80+
# $1 - groupId
81+
# $2 - artifacId
82+
# $3 - version
83+
# $4 - extra repository to look in (optional)
84+
# return value in $RES
85+
function checkAvailability () {
86+
pushd "${TMP_DIR}"
87+
rm -rf *
88+
89+
# pom file for the test project
90+
cat > pom.xml << EOF
91+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
92+
<modelVersion>4.0.0</modelVersion>
93+
<groupId>com.typesafe</groupId>
94+
<artifactId>typesafeDummy</artifactId>
95+
<packaging>war</packaging>
96+
<version>1.0-SNAPSHOT</version>
97+
<name>Dummy</name>
98+
<url>http://127.0.0.1</url>
99+
<dependencies>
100+
<dependency>
101+
<groupId>$1</groupId>
102+
<artifactId>$2</artifactId>
103+
<version>$3</version>
104+
</dependency>
105+
</dependencies>
106+
<repositories>
107+
<repository>
108+
<id>sonatype.snapshot</id>
109+
<name>Sonatype maven snapshot repository</name>
110+
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
111+
<snapshots>
112+
<updatePolicy>daily</updatePolicy>
113+
</snapshots>
114+
</repository>
115+
EOF
116+
117+
if [ -n "$4" ]
118+
then
119+
# adds the extra repository
120+
cat >> pom.xml << EOF
121+
<repository>
122+
<id>extrarepo</id>
123+
<name>extra repository</name>
124+
<url>$4</url>
125+
</repository>
126+
EOF
127+
fi
128+
129+
cat >> pom.xml << EOF
130+
</repositories>
131+
</project>
132+
EOF
133+
134+
set +e
135+
mvn "${MAVEN_ARGS[@]}" compile &> "${TMP_DIR}/mvn.log"
136+
RES=$?
137+
# Quiet the maven, but allow diagnosing problems.
138+
grep -i downloading "${TMP_DIR}/mvn.log"
139+
grep -i exception "${TMP_DIR}/mvn.log"
140+
grep -i error "${TMP_DIR}/mvn.log"
141+
set -e
142+
143+
# log the result
144+
if [ ${RES} == 0 ]
145+
then
146+
debug "$1:$2:jar:$3 found !"
147+
RES=true
148+
else
149+
debug "$1:$2:jar:$3 not found !"
150+
RES=false
151+
fi
152+
popd
153+
}

‎scripts/jobs/integrate/bootstrap

Lines changed: 3 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@
7171
# set to something besides the default to build nightly snapshots of the modules instead of some tagged version
7272
moduleVersioning=${moduleVersioning-"versions.properties"}
7373

74-
baseDir=${WORKSPACE-`pwd`}
7574
publishPrivateTask=${publishPrivateTask-"publish"}
7675
publishSonatypeTaskCore=${publishSonatypeTaskCore-"publish-signed"}
7776
publishSonatypeTaskModules=${publishSonatypeTaskModules-"publish-signed"}
@@ -93,67 +92,11 @@ forceRebuild=${forceRebuild-no}
9392
antBuildTask="${antBuildTask-nightly}" # TESTING leave empty to avoid the sanity check (don't set it to "init" because ant will croak)
9493
clean="clean" # TESTING leave empty to speed up testing
9594

96-
scriptsDir="$WORKSPACE/scripts"
97-
98-
# This is for forcibly stopping the job from a subshell (see test
99-
# below).
100-
trap "exit 1" TERM
101-
export TOP_PID=$$
102-
set -e
103-
104-
# Known problems : does not fare well with interrupted, partial
105-
# compilations. We should perhaps have a multi-dependency version
106-
# of do_i_have below
107-
108-
LOGGINGDIR="$WORKSPACE/logs"
109-
mkdir -p $LOGGINGDIR
110-
111-
unset SBT_HOME
112-
SBT_HOME="$WORKSPACE/.sbt"
113-
mkdir -p $SBT_HOME
114-
IVY_CACHE="$WORKSPACE/.ivy2"
115-
mkdir -p $IVY_CACHE
116-
rm -rf $IVY_CACHE/cache/org.scala-lang
117-
118-
# temp dir where all 'non-build' operation are performed
119-
TMP_ROOT_DIR=$(mktemp -d -t pr-scala.XXXX)
120-
TMP_DIR="${TMP_ROOT_DIR}/tmp"
121-
mkdir "${TMP_DIR}"
12295

12396

124-
# detect sed version and how to enable extended regexes
125-
SEDARGS="-n$(if (echo "a" | sed -nE "s/a/b/" &> /dev/null); then echo E; else echo r; fi)"
126-
127-
128-
129-
# :docstring test:
130-
# Usage: test <argument ..>
131-
# Executes <argument ..>, logging the launch of the command to the
132-
# main log file, and kills global script execution with the TERM
133-
# signal if the commands ends up failing.
134-
# DO NOT USE ON FUNCTIONS THAT DECLARE VARIABLES,
135-
# AS YOU'LL BE RUNNING IN A SUBSHELL AND VARIABLE DECLARATIONS WILL BE LOST
136-
# :end docstring:
137-
138-
function test() {
139-
echo "### $@"
140-
"$@"
141-
status=$?
142-
if [ $status -ne 0 ]; then
143-
say "### ERROR with $1"
144-
kill -s TERM $TOP_PID
145-
fi
146-
}
147-
148-
# :docstring say:
149-
# Usage: say <argument ..>
150-
# Prints <argument ..> to both console and the main log file.
151-
# :end docstring:
152-
153-
function say(){
154-
(echo "$@") | tee -a $LOGGINGDIR/compilation-$SCALADATE-$SCALAHASH.log
155-
}
156-
97+
baseDir=${WORKSPACE-`pwd`}
98+
scriptsDir="$baseDir/scripts"
99+
. $scriptsDir/common
157100

158101
# we must change ivy home to get a fresh ivy cache, otherwise we get half-bootstrapped scala
159102
# rm it in case it existed (and there's no ivy2-shadow, which indicates we're running in a TESTING environment)...
@@ -168,17 +111,6 @@ mkdir -p $baseDir/resolutionScratch_
168111
privateCred="private-repo"
169112
privateRepo="http://private-repo.typesafe.com/typesafe/scala-release-temp/"
170113

171-
function parseScalaProperties(){
172-
propFile="$baseDir/$1"
173-
if [ ! -f $propFile ]; then
174-
echo "Property file $propFile not found."
175-
exit 1
176-
else
177-
awk -f "$scriptsDir/readproperties.awk" "$propFile" > "$propFile.sh"
178-
. "$propFile.sh" # yeah yeah, not that secure, improvements welcome (I tried, but bash made me cry again)
179-
fi
180-
}
181-
182114
##### git
183115
gfxd() {
184116
git clean -fxd # TESTING

‎scripts/jobs/validate/publish-core

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash -e
2+
# This script publishes the core of Scala to maven for use as locker downstream,
3+
# and saves the relevant properties used in its build artifacts, versions.properties.
4+
# (This means we'll use locker instead of quick downstream in dbuild.
5+
# The only downside is that backend improvements don't improve compiler performance itself until they are in STARR).
6+
# The version is suffixed with "-${sha:0:7}-SNAPSHOT"
7+
8+
baseDir=${WORKSPACE-`pwd`}
9+
scriptsDir="$baseDir/scripts"
10+
. $scriptsDir/common
11+
12+
case $prDryRun in
13+
yep)
14+
echo "DRY RUN"
15+
mkdir -p build/pack ; mkdir -p dists/maven/latest
16+
;;
17+
*)
18+
sha=$(git rev-parse HEAD) # TODO: warn if $repo_ref != $sha (we shouldn't do PR validation using symbolic gitrefs)
19+
echo "sha/repo_ref == $sha/$repo_ref ?"
20+
21+
parseScalaProperties build.number
22+
23+
./pull-binary-libs.sh
24+
# "noyoudont" is there juuuust in case
25+
antDeployArgs="-Dmaven.version.suffix=\"-${sha:0:7}-SNAPSHOT\" -Dremote.snapshot.repository=$prRepoUrl -Drepository.credentials.id=pr-scala -Dremote.release.repository=noyoudont"
26+
27+
echo ">>> Getting Scala version number."
28+
ant -q $antDeployArgs init
29+
parseScalaProperties buildcharacter.properties # produce maven_version_number
30+
31+
echo ">>> Checking availability of Scala ${maven_version_number} in $prRepoUrl."
32+
checkAvailability "org.scala-lang" "scala-library" "${maven_version_number}" $prRepoUrl; libraryAvailable=$RES
33+
checkAvailability "org.scala-lang" "scala-reflect" "${maven_version_number}" $prRepoUrl; reflectAvailable=$RES
34+
checkAvailability "org.scala-lang" "scala-compiler" "${maven_version_number}" $prRepoUrl; compilerAvailable=$RES
35+
36+
if $libraryAvailable && $reflectAvailable && $compilerAvailable; then
37+
echo "Scala core already built!"
38+
else
39+
ant $antDeployArgs $antBuildArgs publish-opt-nodocs
40+
fi
41+
42+
mv buildcharacter.properties jenkins.properties # parsed by the jenkins job
43+
;;
44+
esac

‎scripts/jobs/validate/test

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash -e
2+
3+
case $prDryRun in
4+
yep)
5+
echo "DRY RUN"
6+
;;
7+
*)
8+
./pull-binary-libs.sh
9+
10+
# build quick using STARR built upstream, as specified by maven_version_number
11+
# (in that sense it's locker, since it was built with starr by that upstream job)
12+
ant -Dstarr.version=$maven_version_number \
13+
-Dscalac.args.optimise=-optimise \
14+
-Dlocker.skip=1 -Dstarr.use.released=1 -Dextra.repo.url=$prRepoUrl \
15+
$testExtraArgs ${testTarget-test.core docs.done}
16+
;;
17+
esac

0 commit comments

Comments
 (0)