openjdk team mailing list archive
-
openjdk team
-
Mailing list archive
-
Message #13855
[Merge] ~tdaitx/ubuntu/+source/openjdk:fix-autopkgtests into ~openjdk/ubuntu/+source/openjdk/+git/openjdk:openjdk-11
Tiago Stürmer Daitx has proposed merging ~tdaitx/ubuntu/+source/openjdk:fix-autopkgtests into ~openjdk/ubuntu/+source/openjdk/+git/openjdk:openjdk-11.
Commit message:
Improve and fix build tests and autopkgtests
* Improve and fix build tests and autopkgtests:
- Update debian/tests/hotspot,jdk,langtools to ignore
jtreg-autopkgtest.sh return code.
- Create debian/tests/jtdiff-autopkgtest.in as it depends
on debian/rules variables.
- debian/tests/jtreg-autopkgtest.sh:
+ Enable retry of failed tests to trim out flaky tests.
+ Fix unbound variable.
+ Force JT_JAVA otherwise jtreg will look for
/usr/lib/jvm/default-java which might not be installed
or be the right JVM to use.
+ Keep .jtr files from failed tests only.
- debian/tests/jtdiff-autopkgtest.sh:
+ Fail only if an actual regression is detected.
+ Force JT_JAVA otherwise jtreg will look for
/usr/lib/jvm/default-java which might not be installed
or be the right JVM to use.
+ Add the super-diff comparison from jtdiff.
- debian/rules:
+ Preserve all JTreport directories in the test output
directory.
+ Use JDK_DIR instead of JDK_TO_TEST for autopkgtest
generation.
+ Package all .jtr files from JTwork as jtreg-autopkgtest.sh
makes sure it contains only failing tests.
Requested reviews:
Matthias Klose (doko)
For more details, see:
https://code.launchpad.net/~tdaitx/ubuntu/+source/openjdk/+git/openjdk/+merge/371605
--
Your team OpenJDK is subscribed to branch ~openjdk/ubuntu/+source/openjdk/+git/openjdk:openjdk-11.
diff --git a/debian/changelog b/debian/changelog
index 61e525c..087efdb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,31 @@ openjdk-11 (11.0.4+11-2) UNRELEASED; urgency=medium
[ Tiago Stürmer Daitx ]
* Properly generate Breaks: rules for bionic (fix typo).
+ * Improve and fix build tests and autopkgtests:
+ - Update debian/tests/hotspot,jdk,langtools to ignore
+ jtreg-autopkgtest.sh return code.
+ - Create debian/tests/jtdiff-autopkgtest.in as it depends
+ on debian/rules variables.
+ - debian/tests/jtreg-autopkgtest.sh:
+ + Enable retry of failed tests to trim out flaky tests.
+ + Fix unbound variable.
+ + Force JT_JAVA otherwise jtreg will look for
+ /usr/lib/jvm/default-java which might not be installed
+ or be the right JVM to use.
+ + Keep .jtr files from failed tests only.
+ - debian/tests/jtdiff-autopkgtest.sh:
+ + Fail only if an actual regression is detected.
+ + Force JT_JAVA otherwise jtreg will look for
+ /usr/lib/jvm/default-java which might not be installed
+ or be the right JVM to use.
+ + Add the super-diff comparison from jtdiff.
+ - debian/rules:
+ + Preserve all JTreport directories in the test output
+ directory.
+ + Use JDK_DIR instead of JDK_TO_TEST for autopkgtest
+ generation.
+ + Package all .jtr files from JTwork as jtreg-autopkgtest.sh
+ makes sure it contains only failing tests.
-- Matthias Klose <doko@xxxxxxxxxx> Wed, 17 Jul 2019 14:49:36 +0200
diff --git a/debian/rules b/debian/rules
index 0966d64..28f45a5 100755
--- a/debian/rules
+++ b/debian/rules
@@ -809,12 +809,13 @@ debian/control: debian/control.in debian/rules
debian/tests/%.sh: debian/tests/%.in debian/rules
sed \
- -e 's,@JDK_TO_TEST@,/$(basedir),g' \
+ -e 's,@JDK_DIR@,/$(basedir),g' \
-e 's/@jtreg_archs@/$(jtreg_archs)/g' \
+ -e 's:@doc_dir@:/usr/share/doc/$(p_jrehl)/:g' \
$< > $@;
chmod +x $@
-gen-autopkgtests: debian/tests/jtreg-autopkgtest.sh
+gen-autopkgtests: debian/tests/jtdiff-autopkgtest.sh debian/tests/jtreg-autopkgtest.sh
packaging-files:
for f in debian/*.in; do \
@@ -1046,13 +1047,10 @@ ifneq (,$(filter $(VMNAME), $(with_jtreg_check)))
@echo "END jtreg-summary-$(VMNAME)"
-for i in hotspot langtools jaxp jdk; do \
- test -f jtreg-test-output/check-$$i-$(VMNAME).log || continue; \
- for t in $$(egrep '^(FAILED|Error)' jtreg-test-output/check-$$i-$(VMNAME).log | grep -v '^Error: Some tests failed or other problems occurred.$$' | sed -e 's/.* \(.*\)\.[^.#]\+\(#id[0-9]*\)\?$$/\1\2/' -e 's/#id/_id/'); do \
- echo jtreg-test-output/$$i/JTwork/$$t.jtr; \
- done; \
+ find jtreg-test-output/$$i/JTwork/ -name '*.jtr'; \
done | sort -u > jtreg-test-output/failed_tests-$(VMNAME).list; \
GZIP=-9vn tar --ignore-failed-read -C . -c -z -f jtreg-test-output/failed_tests-$(VMNAME).tar.gz -T jtreg-test-output/failed_tests-$(VMNAME).list
- GZIP=-9vn tar -C . -c -z -f jtreg-test-output/jtreport-$(VMNAME).tar.gz jtreg-test-output/*/JTreport/
+ GZIP=-9vn tar -C . -c -z -f jtreg-test-output/jtreport-$(VMNAME).tar.gz $$(find jtreg-test-output -name JTreport)
else
echo "jtreg harness not run for this build" > jtreg-test-output/jtreg_output-$(VMNAME)
endif
@@ -1065,26 +1063,26 @@ TESTS_TO_RUN = $(addprefix check-,$(TEST_SUITES))
check-hotspot: stamps/build
mkdir -p jtreg-test-output/hotspot
JTREG_OPTIONS=$(JTREG_OPTIONS) VMNAME=$(VMNAME) JDK_TO_TEST=$(builddir)/$(sdkimg) \
- AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/hotspot/ \
+ AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/ \
$(TIME) debian/tests/hotspot | tee jtreg-test-output/$@.log
check-langtools: stamps/build
mkdir -p jtreg-test-output/langtools
JTREG_OPTIONS=$(JTREG_OPTIONS) VMNAME=$(VMNAME) JDK_TO_TEST=$(builddir)/$(sdkimg) \
- AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/langtools/ \
+ AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/ \
$(TIME) debian/tests/langtools | tee jtreg-test-output/$@.log
check-jaxp: stamps/build
mkdir -p jtreg-test-output/jaxp
JTREG_OPTIONS=$(JTREG_OPTIONS) VMNAME=$(VMNAME) JDK_TO_TEST=$(builddir)/$(sdkimg) \
- AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/langtools/ \
+ AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/ \
$(TIME) debian/tests/jaxp | tee jtreg-test-output/$@.log
check-jdk: stamps/build
mkdir -p jtreg-test-output/jdk
JTREG_OPTIONS=$(JTREG_OPTIONS) VMNAME=$(VMNAME) JDK_TO_TEST=$(builddir)/$(sdkimg) \
- AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/jdk/ \
+ AUTOPKGTEST_TMP=/tmp/ AUTOPKGTEST_ARTIFACTS=jtreg-test-output/ \
$(TIME) debian/tests/jdk | tee jtreg-test-output/$@.log
jtregcheck: $(TESTS_TO_RUN)
diff --git a/debian/tests/hotspot b/debian/tests/hotspot
index 9b5b467..21e9e83 100755
--- a/debian/tests/hotspot
+++ b/debian/tests/hotspot
@@ -4,9 +4,9 @@ set -o errtrace
set -o pipefail
set -o nounset
-debian/tests/jtreg-autopkgtest.sh \
+debian/tests/jtreg-autopkgtest.sh hotspot \
-exclude:test/hotspot/jtreg/ProblemList.txt \
-dir:test/hotspot/jtreg \
- :hotspot_compiler :hotspot_gc :hotspot_runtime :hotspot_serviceability
+ :hotspot_compiler :hotspot_gc :hotspot_runtime :hotspot_serviceability || true
debian/tests/jtdiff-autopkgtest.sh hotspot
diff --git a/debian/tests/jaxp b/debian/tests/jaxp
index 4111bfc..567b103 100755
--- a/debian/tests/jaxp
+++ b/debian/tests/jaxp
@@ -4,5 +4,5 @@ set -o errtrace
set -o pipefail
set -o nounset
-debian/tests/jtreg-autopkgtest.sh -exclude:test/jaxp/ProblemList.txt test/jaxp
+debian/tests/jtreg-autopkgtest.sh jaxp -exclude:test/jaxp/ProblemList.txt test/jaxp || true
debian/tests/jtdiff-autopkgtest.sh jaxp
diff --git a/debian/tests/jdk b/debian/tests/jdk
index 1f5b3ab..53086fc 100755
--- a/debian/tests/jdk
+++ b/debian/tests/jdk
@@ -22,5 +22,5 @@ export DISPLAY=:10
debian/tests/start-xvfb.sh 10 &
sleep 3
-debian/tests/jtreg-autopkgtest.sh -exclude:test/jdk/ProblemList.txt test/jdk:jdk_stable
+debian/tests/jtreg-autopkgtest.sh jdk -exclude:test/jdk/ProblemList.txt test/jdk:jdk_stable || true
debian/tests/jtdiff-autopkgtest.sh jdk
diff --git a/debian/tests/jtdiff-autopkgtest.in b/debian/tests/jtdiff-autopkgtest.in
new file mode 100755
index 0000000..94dae09
--- /dev/null
+++ b/debian/tests/jtdiff-autopkgtest.in
@@ -0,0 +1,55 @@
+#!/bin/bash
+set -o errexit
+set -o errtrace
+set -o pipefail
+set -o nounset
+
+testsuite=$1
+shift
+
+if [ -z "${AUTOPKGTEST_TMP+x}" ] || [ -z "${AUTOPKGTEST_ARTIFACTS+x}" ]; then
+ echo "Environment variables AUTOPKGTEST_TMP and AUTOPKGTEST_ARTIFACTS must be set" >&2
+ exit 1
+fi
+
+host_arch=${DEB_HOST_ARCH:-$(dpkg --print-architecture)}
+
+# force jtreg to use the JDK we depend on instead of default-java
+export JT_JAVA=$(echo @JDK_DIR@ | sed "s/-[^-]*$/-$host_arch/")
+
+vmname=${VMNAME:-hotspot}
+
+jt_report_tb="@doc_dir@/test-${host_arch}/jtreport-${vmname}.tar.gz"
+
+if [ ! -f "${jt_report_tb}" ]; then
+ echo "Unable to compare jtreg results: no build jtreport found for ${vmname}/${host_arch}."
+ echo "Reason: '${jt_report_tb}' does not exist."
+ exit 77
+fi
+
+# create directories to hold the results
+mkdir -p "${AUTOPKGTEST_ARTIFACTS}/${testsuite}"
+mkdir -p "${AUTOPKGTEST_TMP}/openjdk-pkg-jtreg-report"
+
+current_report_dir="${AUTOPKGTEST_ARTIFACTS}/${testsuite}"
+previous_report_dir="${AUTOPKGTEST_TMP}/openjdk-pkg-jtreg-report/${testsuite}"
+
+# extract testsuite results from openjdk package
+[ -d "${previous_report_dir}" ] || \
+ tar -xf "${jt_report_tb}" --strip-components=1 -C "${AUTOPKGTEST_TMP}/openjdk-pkg-jtreg-report"
+
+
+jtdiff -o "${current_report_dir}/jtdiff.html" "${previous_report_dir}/JTreport" "${current_report_dir}/JTreport" || true
+jtdiff "${previous_report_dir}/JTreport" "${current_report_dir}/JTreport" | tee "${current_report_dir}/jtdiff.txt" || true
+
+# create jdiff super-diff structure
+jtdiff_dir="${AUTOPKGTEST_TMP}/jtdiff-${testsuite}/${host_arch}"
+mkdir -p "${jtdiff_dir}/"{1,2} "${current_report_dir}/jtdiff-super"
+ln -sf "${previous_report_dir}/"[0-9] "${jtdiff_dir}/1/"
+ln -sf "${current_report_dir}/"[0-9] "${jtdiff_dir}/2/"
+
+# run jtdiff super-diff
+jtdiff -o "${current_report_dir}/jtdiff-super/" -s "${AUTOPKGTEST_TMP}/jtdiff-${testsuite}/" || true
+
+# fail if we detect a regression
+egrep '^pass +[^-]{3}' "${current_report_dir}/jtdiff.txt" && exit 1
diff --git a/debian/tests/jtdiff-autopkgtest.sh b/debian/tests/jtdiff-autopkgtest.sh
index 7d73c83..54cbaa1 100755
--- a/debian/tests/jtdiff-autopkgtest.sh
+++ b/debian/tests/jtdiff-autopkgtest.sh
@@ -14,18 +14,12 @@ fi
host_arch=${DEB_HOST_ARCH:-$(dpkg --print-architecture)}
-# don't mess around with JT_* env vars. If JDK_TO_TEST is set, then the
-# script is called from the build, if not, from the autopkg tests
-if [ -z "$JDK_TO_TEST" ]; then
- JDK_TO_TEST=@JDK_TO_TEST@
-fi
-JVERSION=$($JDK_TO_TEST/bin/java -version 2>&1| sed -n '1s/.*"\(.*\)".*/\1/p')
-JMAJOR=$(echo $JVERSION | sed 's/\..*//')
+# force jtreg to use the JDK we depend on instead of default-java
+export JT_JAVA=$(echo /usr/lib/jvm/java-11-openjdk-amd64 | sed "s/-[^-]*$/-$host_arch/")
vmname=${VMNAME:-hotspot}
-jt_report_tb="/usr/share/doc/openjdk-$JMAJOR-jdk/test-${host_arch}/jtreport-${vmname}.tar.gz"
-build_report_dir="${AUTOPKGTEST_TMP}/jtreg-test-output/${testsuite}/JTreport"
+jt_report_tb="/usr/share/doc/openjdk-11-jre-headless//test-${host_arch}/jtreport-${vmname}.tar.gz"
if [ ! -f "${jt_report_tb}" ]; then
echo "Unable to compare jtreg results: no build jtreport found for ${vmname}/${host_arch}."
@@ -33,9 +27,29 @@ if [ ! -f "${jt_report_tb}" ]; then
exit 77
fi
-# extract testsuite results from original openjdk build
-[ -d "${build_report_dir}" ] || \
- tar -xf "${jt_report_tb}" -C "${AUTOPKGTEST_TMP}"
+# create directories to hold the results
+mkdir -p "${AUTOPKGTEST_ARTIFACTS}/${testsuite}"
+mkdir -p "${AUTOPKGTEST_TMP}/openjdk-pkg-jtreg-report"
+
+current_report_dir="${AUTOPKGTEST_ARTIFACTS}/${testsuite}"
+previous_report_dir="${AUTOPKGTEST_TMP}/openjdk-pkg-jtreg-report/${testsuite}"
+
+# extract testsuite results from openjdk package
+[ -d "${previous_report_dir}" ] || \
+ tar -xf "${jt_report_tb}" --strip-components=1 -C "${AUTOPKGTEST_TMP}/openjdk-pkg-jtreg-report"
+
+
+jtdiff -o "${current_report_dir}/jtdiff.html" "${previous_report_dir}/JTreport" "${current_report_dir}/JTreport" || true
+jtdiff "${previous_report_dir}/JTreport" "${current_report_dir}/JTreport" | tee "${current_report_dir}/jtdiff.txt" || true
+
+# create jdiff super-diff structure
+jtdiff_dir="${AUTOPKGTEST_TMP}/jtdiff-${testsuite}/${host_arch}"
+mkdir -p "${jtdiff_dir}/"{1,2} "${current_report_dir}/jtdiff-super"
+ln -sf "${previous_report_dir}/"[0-9] "${jtdiff_dir}/1/"
+ln -sf "${current_report_dir}/"[0-9] "${jtdiff_dir}/2/"
+
+# run jtdiff super-diff
+jtdiff -o "${current_report_dir}/jtdiff-super/" -s "${AUTOPKGTEST_TMP}/jtdiff-${testsuite}/" || true
-jtdiff -o "${AUTOPKGTEST_ARTIFACTS}/jtdiff.html" "${build_report_dir}" "${AUTOPKGTEST_ARTIFACTS}/JTreport" || true
-jtdiff "${build_report_dir}" "${AUTOPKGTEST_ARTIFACTS}/JTreport" | tee "${AUTOPKGTEST_ARTIFACTS}/jtdiff.txt"
+# fail if we detect a regression
+egrep '^pass +[^-]{3}' "${current_report_dir}/jtdiff.txt" && exit 1
diff --git a/debian/tests/jtreg-autopkgtest.in b/debian/tests/jtreg-autopkgtest.in
index 53e59e6..fb032ca 100755
--- a/debian/tests/jtreg-autopkgtest.in
+++ b/debian/tests/jtreg-autopkgtest.in
@@ -4,17 +4,20 @@ set -o errtrace
set -o pipefail
set -o nounset
+testsuite=$1
+shift
+
if [ -z "${AUTOPKGTEST_TMP+x}" ] || [ -z "${AUTOPKGTEST_ARTIFACTS+x}" ]; then
echo "Environment variables AUTOPKGTEST_TMP and AUTOPKGTEST_ARTIFACTS must be set" >&2
exit 1
fi
-host_arch=${DEB_HOST_ARCH:-$(dpkg --print-architecture)}
+host_arch="${DEB_HOST_ARCH:-$(dpkg --print-architecture)}"
-# don't mess around with JT_* env vars. If JDK_TO_TEST is set, then the
-# script is called from the build, if not, from the autopkg tests
-if [ -z "$JDK_TO_TEST" ]; then
- JDK_TO_TEST=$(echo @JDK_TO_TEST@ | sed "s/-[^-]*$/-$host_arch/")
+# force jtreg to use the JDK we depend on instead of default-java
+export JT_JAVA=$(echo @JDK_DIR@ | sed "s/-[^-]*$/-$host_arch/")
+if [ -z "${JDK_TO_TEST+x}" ]; then
+ JDK_TO_TEST=$JT_JAVA
fi
jtreg_version="$(dpkg-query -W jtreg | cut -f2)"
@@ -35,7 +38,7 @@ if [ ! -x "${JDK_TO_TEST}/bin/java" ]; then
fi
# restrict the tests to a few archs (set from debian/rules)
-if ! echo "${host_arch}" | grep -E "^($(echo @jtreg_archs@ | tr ' ' '|'))$"; then
+if ! echo "${host_arch}" | grep -qE "^($(echo @jtreg_archs@ | tr ' ' '|'))$"; then
echo "Error: ${host_arch} is not on the jtreg_archs list, ignoring it."
exit 77
fi
@@ -43,7 +46,7 @@ fi
jtreg_processes() {
ps x -ww -o pid,ppid,args \
| awk '$2 == 1 && $3 ~ /^\/scratch/' \
- | sed "s,${JDK_DIR},<sdkimg>,g;s,$(pwd),<pwd>,g"
+ | sed "s,${JDK_TO_TEST},<sdkimg>,g;s,$(pwd),<pwd>,g"
}
jtreg_pids() {
@@ -78,14 +81,43 @@ cleanup() {
trap "cleanup" EXIT INT TERM ERR
-jtreg ${jt_options} \
- -verbose:summary \
- -automatic \
- -retain:none \
- -ignore:quiet \
- -agentvm \
- -timeout:5 \
- -workDir:"${AUTOPKGTEST_ARTIFACTS}/JTwork" \
- -reportDir:"${AUTOPKGTEST_ARTIFACTS}/JTreport" \
- -jdk:${JDK_TO_TEST} \
- $@
+jtwork_dir="${AUTOPKGTEST_TMP}/${testsuite}/JTwork"
+output_dir="${AUTOPKGTEST_ARTIFACTS}/${testsuite}/"
+
+# retry tests with "fail" or "error" status at most 3 times
+for i in 0 1 2 3; do
+ # save each try under its own folder to preserve history
+ report_path="${i}/JTreport"
+ report_dir="${output_dir}/${report_path}"
+ jtreg ${jt_options} \
+ -verbose:summary \
+ -automatic \
+ -retain:none \
+ -ignore:quiet \
+ -agentvm \
+ -timeout:5 \
+ -workDir:"${jtwork_dir}" \
+ -reportDir:"${report_dir}" \
+ -jdk:${JDK_TO_TEST} \
+ ${on_retry:-} $@ \
+ && exit_code=0 && break || exit_code=$?
+
+ # break if jtdiff reports no difference from previous run
+ # also deletes the just created JTreport
+ # DISABLED: don't use it for now as flaky tests could still pass given more retries
+ #jtdiff "${output_dir}/JTreport" "$report_dir" >/dev/null 2>&1 \
+ # && rm -rf "${report_dir}" && break
+
+ # link latest JTreport to output_dir
+ ln -sf -t "${output_dir}" "$report_path"
+ on_retry="-status:fail,error"
+done
+
+# copy .jtr files from failed tests for latter debugging
+find "${jtwork_dir}" -name '*.jtr' -exec egrep -q '^execStatus=[^Pass]' {} \; -printf "%P\n" \
+ | while IF= read -r jtr; do
+ mkdir -p "$(dirname "${output_dir}/JTwork/${jtr}")"
+ cp "${jtwork_dir}/${jtr}" "${output_dir}/JTwork/$jtr"
+ done
+
+exit $exit_code
diff --git a/debian/tests/jtreg-autopkgtest.sh b/debian/tests/jtreg-autopkgtest.sh
index 9c956fd..b4c462a 100755
--- a/debian/tests/jtreg-autopkgtest.sh
+++ b/debian/tests/jtreg-autopkgtest.sh
@@ -4,17 +4,20 @@ set -o errtrace
set -o pipefail
set -o nounset
+testsuite=$1
+shift
+
if [ -z "${AUTOPKGTEST_TMP+x}" ] || [ -z "${AUTOPKGTEST_ARTIFACTS+x}" ]; then
echo "Environment variables AUTOPKGTEST_TMP and AUTOPKGTEST_ARTIFACTS must be set" >&2
exit 1
fi
-host_arch=${DEB_HOST_ARCH:-$(dpkg --print-architecture)}
+host_arch="${DEB_HOST_ARCH:-$(dpkg --print-architecture)}"
-# don't mess around with JT_* env vars. If JDK_TO_TEST is set, then the
-# script is called from the build, if not, from the autopkg tests
-if [ -z "$JDK_TO_TEST" ]; then
- JDK_TO_TEST=$(echo /usr/lib/jvm/java-13-openjdk-amd64 | sed "s/-[^-]*$/-$host_arch/")
+# force jtreg to use the JDK we depend on instead of default-java
+export JT_JAVA=$(echo /usr/lib/jvm/java-11-openjdk-amd64 | sed "s/-[^-]*$/-$host_arch/")
+if [ -z "${JDK_TO_TEST+x}" ]; then
+ JDK_TO_TEST=$JT_JAVA
fi
jtreg_version="$(dpkg-query -W jtreg | cut -f2)"
@@ -35,7 +38,7 @@ if [ ! -x "${JDK_TO_TEST}/bin/java" ]; then
fi
# restrict the tests to a few archs (set from debian/rules)
-if ! echo "${host_arch}" | grep -E "^($(echo amd64 i386 arm64 armhf ppc64 ppc64el sparc64 s390x kfreebsd-amd64 kfreebsd-i386 alpha ia64 powerpc powerpcspe ppc64 sh4 x32 | tr ' ' '|'))$"; then
+if ! echo "${host_arch}" | grep -qE "^($(echo amd64 i386 arm64 armhf ppc64 ppc64el sparc64 s390x kfreebsd-amd64 kfreebsd-i386 alpha ia64 powerpc powerpcspe ppc64 sh4 x32 | tr ' ' '|'))$"; then
echo "Error: ${host_arch} is not on the jtreg_archs list, ignoring it."
exit 77
fi
@@ -43,7 +46,7 @@ fi
jtreg_processes() {
ps x -ww -o pid,ppid,args \
| awk '$2 == 1 && $3 ~ /^\/scratch/' \
- | sed "s,${JDK_DIR},<sdkimg>,g;s,$(pwd),<pwd>,g"
+ | sed "s,${JDK_TO_TEST},<sdkimg>,g;s,$(pwd),<pwd>,g"
}
jtreg_pids() {
@@ -78,14 +81,43 @@ cleanup() {
trap "cleanup" EXIT INT TERM ERR
-jtreg ${jt_options} \
- -verbose:summary \
- -automatic \
- -retain:none \
- -ignore:quiet \
- -agentvm \
- -timeout:5 \
- -workDir:"${AUTOPKGTEST_ARTIFACTS}/JTwork" \
- -reportDir:"${AUTOPKGTEST_ARTIFACTS}/JTreport" \
- -jdk:${JDK_TO_TEST} \
- $@
+jtwork_dir="${AUTOPKGTEST_TMP}/${testsuite}/JTwork"
+output_dir="${AUTOPKGTEST_ARTIFACTS}/${testsuite}/"
+
+# retry tests with "fail" or "error" status at most 3 times
+for i in 0 1 2 3; do
+ # save each try under its own folder to preserve history
+ report_path="${i}/JTreport"
+ report_dir="${output_dir}/${report_path}"
+ jtreg ${jt_options} \
+ -verbose:summary \
+ -automatic \
+ -retain:none \
+ -ignore:quiet \
+ -agentvm \
+ -timeout:5 \
+ -workDir:"${jtwork_dir}" \
+ -reportDir:"${report_dir}" \
+ -jdk:${JDK_TO_TEST} \
+ ${on_retry:-} $@ \
+ && exit_code=0 && break || exit_code=$?
+
+ # break if jtdiff reports no difference from previous run
+ # also deletes the just created JTreport
+ # DISABLED: don't use it for now as flaky tests could still pass given more retries
+ #jtdiff "${output_dir}/JTreport" "$report_dir" >/dev/null 2>&1 \
+ # && rm -rf "${report_dir}" && break
+
+ # link latest JTreport to output_dir
+ ln -sf -t "${output_dir}" "$report_path"
+ on_retry="-status:fail,error"
+done
+
+# copy .jtr files from failed tests for latter debugging
+find "${jtwork_dir}" -name '*.jtr' -exec egrep -q '^execStatus=[^Pass]' {} \; -printf "%P\n" \
+ | while IF= read -r jtr; do
+ mkdir -p "$(dirname "${output_dir}/JTwork/${jtr}")"
+ cp "${jtwork_dir}/${jtr}" "${output_dir}/JTwork/$jtr"
+ done
+
+exit $exit_code
diff --git a/debian/tests/langtools b/debian/tests/langtools
index fd0cbf3..04637ea 100755
--- a/debian/tests/langtools
+++ b/debian/tests/langtools
@@ -4,5 +4,5 @@ set -o errtrace
set -o pipefail
set -o nounset
-debian/tests/jtreg-autopkgtest.sh -exclude:test/langtools/ProblemList.txt -dir:test/langtools jdk tools lib
+debian/tests/jtreg-autopkgtest.sh langtools -exclude:test/langtools/ProblemList.txt -dir:test/langtools jdk tools lib || true
debian/tests/jtdiff-autopkgtest.sh langtools
Follow ups
