[Fuego] [PATCH 1/1] jenkins: upgrade to latest LTS release 2.164.1
Fathi Boudra
fathi.boudra at linaro.org
Tue Apr 9 14:46:00 UTC 2019
* Bump JENKINS_VERSION from 2.32.1 to 2.164.1 release.
* Update JENKINS_SHA accordingly.
* Upgrade to latest install-plugins.sh and jenkins-support coming from
https://github.com/jenkinsci/docker. It brings the feature to resolve
plugin dependencies.
* Due to the newer install-plugins.sh/jenkins-support scripts, setting
JENKINS_UC and REF environment variable is required. They were
previously set in the scripts but not anymore.
* Update python-jenkins from 0.4.14 to 1.4.0 release.
* Add -Dhudson.model.DirectoryBrowserSupport.allowSymlinkEscape=true to
JAVA_ARGS. It's required to allow to follow fuego logs symlinked under
userContent/ and caused by a change in behavior in Jenkins.
* Add -remoting argument to jenkins-cli.jar client. It's required in order
to install a local plugin and caused by a change in Jenkins.
* Upgrade Jenkins plugins to their latest release.
Signed-off-by: Fathi Boudra <fathi.boudra at linaro.org>
---
Dockerfile | 42 ++++++++++--------
frontend-install/install-plugins.sh | 69 +++++++++++++++++------------
frontend-install/jenkins-support | 50 ++++++++++++++++++++-
install-debian.sh | 42 ++++++++++--------
4 files changed, 136 insertions(+), 67 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 364ac55..2015b06 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -93,16 +93,18 @@ ARG group=jenkins
ARG uid=1000
ARG gid=${uid}
ARG JENKINS_PORT=8080
-ARG JENKINS_VERSION=2.32.1
-ARG JENKINS_SHA=bfc226aabe2bb089623772950c4cc13aee613af1
+ARG JENKINS_VERSION=2.164.1
+ARG JENKINS_SHA=969df594d1958800cd7da55e19ca75cf65f7fbf0
ARG JENKINS_URL=https://pkg.jenkins.io/debian-stable/binary/jenkins_${JENKINS_VERSION}_all.deb
+ARG JENKINS_UC=https://updates.jenkins.io
+ARG REF=/var/lib/jenkins/plugins
ENV JENKINS_HOME=/var/lib/jenkins
ENV JENKINS_PORT=$JENKINS_PORT
# Jenkins dependencies
RUN apt-get -q=2 -V --no-install-recommends install \
default-jdk daemon psmisc adduser procps unzip
-RUN pip install python-jenkins==0.4.14
+RUN pip install python-jenkins==1.4.0
RUN echo -e "JENKINS_PORT=$JENKINS_PORT" >> /etc/environment
RUN getent group ${gid} >/dev/null || groupadd -g ${gid} ${group}
@@ -144,7 +146,7 @@ RUN source /etc/default/jenkins && \
sed -i -e "s#JENKINS_ARGS.*#JENKINS_ARGS\=\"${JENKINS_ARGS}\"#g" /etc/default/jenkins
RUN source /etc/default/jenkins && \
- JAVA_ARGS="$JAVA_ARGS -Djenkins.install.runSetupWizard=false" && \
+ JAVA_ARGS="$JAVA_ARGS -Djenkins.install.runSetupWizard=false -Dhudson.model.DirectoryBrowserSupport.allowSymlinkEscape=true" && \
if [ -n "$HTTP_PROXY" ]; then \
PROXYSERVER=$(echo $http_proxy | sed -E 's/^http://' | sed -E 's/\///g' | sed -E 's/(.*):(.*)/\1/') && \
PROXYPORT=$(echo $http_proxy | sed -E 's/^http://' | sed -E 's/\///g' | sed -E 's/(.*):(.*)/\2/') && \
@@ -164,32 +166,34 @@ COPY frontend-install/install-plugins.sh \
# install flot.hpi manually from local file
RUN service jenkins start && \
sleep 30 && \
- sudo -u jenkins java -jar /var/cache/jenkins/war/WEB-INF/jenkins-cli.jar -s http://localhost:$JENKINS_PORT/fuego install-plugin /tmp/flot.hpi && \
+ sudo -u jenkins java -jar /var/cache/jenkins/war/WEB-INF/jenkins-cli.jar -remoting -s http://localhost:$JENKINS_PORT/fuego install-plugin /tmp/flot.hpi && \
sleep 10 && \
service jenkins stop
# install other plugins from Jenkins update center
# NOTE: not sure all of these are needed, but keep list
# compatible with 1.2.1 release for now
-RUN /usr/local/bin/install-plugins.sh ant:1.7 \
- bouncycastle-api:2.16.2 \
+RUN /usr/local/bin/install-plugins.sh \
+ ant:1.9 \
+ antisamy-markup-formatter:1.5 \
+ bouncycastle-api:2.17 \
+ command-launcher:1.3 \
description-setter:1.10 \
- display-url-api:2.1.0 \
+ display-url-api:2.3.1 \
external-monitor-job:1.7 \
greenballs:1.15 \
icon-shim:2.0.3 \
- javadoc:1.4 \
- junit:1.21 \
- ldap:1.17 \
- mailer:1.20 \
- matrix-auth:1.7 \
- matrix-project:1.12 \
- antisamy-markup-formatter:1.5 \
- pam-auth:1.3 \
+ javadoc:1.5 \
+ jdk-tool:1.2 \
+ junit:1.27 \
+ ldap:1.20 \
+ mailer:1.23 \
+ matrix-auth:2.3 \
+ matrix-project:1.14 \
+ pam-auth:1.5 \
pegdown-formatter:1.3 \
- script-security:1.35 \
- structs:1.10 \
- windows-slaves:1.3.1
+ structs:1.17 \
+ windows-slaves:1.4
# make the mod.js symlink well after flot is installed
RUN service jenkins start && sleep 30 && \
diff --git a/frontend-install/install-plugins.sh b/frontend-install/install-plugins.sh
index 2b5d78c..86c4e43 100755
--- a/frontend-install/install-plugins.sh
+++ b/frontend-install/install-plugins.sh
@@ -5,13 +5,10 @@
#
# Copied from: https://github.com/jenkinsci/docker
#
-# Download plugins given on the command line
+# Resolve dependencies and download plugins given on the command line
#
# FROM jenkins
-# COPY install-plugins.sh /usr/local/bin/
-# RUN install-plugins.sh greenballs description-setter:1.10
-
-JENKINS_UC=https://updates.jenkins.io
+# RUN install-plugins.sh docker-slaves github-branch-source
set -o pipefail
@@ -29,14 +26,15 @@ getArchiveFilename() {
}
download() {
- local plugin originalPlugin version lock ignoreLockFile
+ local plugin originalPlugin version lock ignoreLockFile url
plugin="$1"
version="${2:-latest}"
ignoreLockFile="${3:-}"
+ url="${4:-}"
lock="$(getLockFile "$plugin")"
if [[ $ignoreLockFile ]] || mkdir "$lock" &>/dev/null; then
- if ! doDownload "$plugin" "$version"; then
+ if ! doDownload "$plugin" "$version" "$url"; then
# some plugin don't follow the rules about artifact ID
# typically: docker-plugin
originalPlugin="$plugin"
@@ -54,8 +52,7 @@ download() {
return 1
fi
- # TRB - don't resolve deps - only load exactly what is specified
- #resolveDependencies "$plugin"
+ resolveDependencies "$plugin"
fi
}
@@ -63,6 +60,7 @@ doDownload() {
local plugin version url jpi
plugin="$1"
version="$2"
+ url="$3"
jpi="$(getArchiveFilename "$plugin")"
# If plugin already exists and is the same version do not download
@@ -71,22 +69,33 @@ doDownload() {
return 0
fi
- if [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then
+ if [[ -n $url ]] ; then
+ echo "Will use url=$url"
+ elif [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then
# If version-specific Update Center is available, which is the case for LTS versions,
# use it to resolve latest versions.
url="$JENKINS_UC_LATEST/latest/${plugin}.hpi"
elif [[ "$version" == "experimental" && -n "$JENKINS_UC_EXPERIMENTAL" ]]; then
# Download from the experimental update center
url="$JENKINS_UC_EXPERIMENTAL/latest/${plugin}.hpi"
+ elif [[ "$version" == incrementals* ]] ; then
+ # Download from Incrementals repo: https://jenkins.io/blog/2018/05/15/incremental-deployment/
+ # Example URL: https://repo.jenkins-ci.org/incrementals/org/jenkins-ci/plugins/workflow/workflow-support/2.19-rc289.d09828a05a74/workflow-support-2.19-rc289.d09828a05a74.hpi
+ local groupId incrementalsVersion
+ arrIN=("${version//;/ }")
+ groupId=${arrIN[1]}
+ incrementalsVersion=${arrIN[2]}
+ url="${JENKINS_INCREMENTALS_REPO_MIRROR}/$(echo "${groupId}" | tr '.' '/')/${plugin}/${incrementalsVersion}/${plugin}-${incrementalsVersion}.hpi"
else
JENKINS_UC_DOWNLOAD=${JENKINS_UC_DOWNLOAD:-"$JENKINS_UC/download"}
url="$JENKINS_UC_DOWNLOAD/plugins/$plugin/$version/${plugin}.hpi"
fi
echo "Downloading plugin: $plugin from $url"
- curl --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-5}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" -s -f -L "$url" -o "$jpi"
- chown jenkins.jenkins "$jpi"
- cp -p "$jpi" /var/lib/jenkins/plugins
+ # We actually want to allow variable value to be split into multiple options passed to curl.
+ # This is needed to allow long options and any options that take value.
+ # shellcheck disable=SC2086
+ retry_command curl ${CURL_OPTIONS:--sSfL} --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-3}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" "$url" -o "$jpi"
return $?
}
@@ -156,9 +165,7 @@ bundledPlugins() {
done
rm -fr $TEMP_PLUGIN_DIR
else
- rm -f "$TEMP_ALREADY_INSTALLED"
- echo "ERROR file not found: $JENKINS_WAR"
- exit 1
+ echo "war not found, installing all plugins: $JENKINS_WAR"
fi
}
@@ -169,6 +176,7 @@ versionFromPlugin() {
else
echo "latest"
fi
+
}
installedPlugins() {
@@ -182,21 +190,21 @@ jenkinsMajorMinorVersion() {
JENKINS_WAR=/usr/share/jenkins/jenkins.war
if [[ -f "$JENKINS_WAR" ]]; then
local version major minor
- version="$(java -jar /usr/share/jenkins/jenkins.war --version)"
+ version="$(java -jar $JENKINS_WAR --version)"
major="$(echo "$version" | cut -d '.' -f 1)"
minor="$(echo "$version" | cut -d '.' -f 2)"
echo "$major.$minor"
else
- echo "ERROR file not found: $JENKINS_WAR"
- return 1
+ echo ""
fi
}
main() {
- local plugin pluginVersion jenkinsVersion
+ local plugin jenkinsVersion
local plugins=()
mkdir -p "$REF_DIR" || exit 1
+ rm -f "$FAILED"
# Read plugins from stdin or from the command line arguments
if [[ ($# -eq 0) ]]; then
@@ -236,14 +244,16 @@ main() {
echo "Downloading plugins..."
for plugin in "${plugins[@]}"; do
- pluginVersion=""
-
- if [[ $plugin =~ .*:.* ]]; then
- pluginVersion=$(versionFromPlugin "${plugin}")
- plugin="${plugin%%:*}"
+ local reg='^([^:]+):?([^:]+)?:?([^:]+)?:?(http.+)?'
+ if [[ $plugin =~ $reg ]]; then
+ local pluginId="${BASH_REMATCH[1]}"
+ local version="${BASH_REMATCH[2]}"
+ local lock="${BASH_REMATCH[3]}"
+ local url="${BASH_REMATCH[4]}"
+ download "$pluginId" "$version" "${lock:-true}" "${url}" &
+ else
+ echo "Skipping the line '${plugin}' as it does not look like a reference to a plugin"
fi
-
- download "$plugin" "$pluginVersion" "true" &
done
wait
@@ -260,7 +270,10 @@ main() {
fi
echo "Cleaning up locks"
- rm -r "$REF_DIR"/*.lock
+ find "$REF_DIR" -regex ".*.lock" | while read -r filepath; do
+ rm -r "$filepath"
+ done
+
}
main "$@"
diff --git a/frontend-install/jenkins-support b/frontend-install/jenkins-support
index 9ce7937..900e594 100755
--- a/frontend-install/jenkins-support
+++ b/frontend-install/jenkins-support
@@ -119,7 +119,7 @@ copy_reference_file() {
action="INSTALLED"
log=true
mkdir -p "$JENKINS_HOME/${dir:23}"
- cp -pr "${f}" "$JENKINS_HOME/${rel}";
+ cp -pr "$(realpath "${f}")" "$JENKINS_HOME/${rel}";
else
action="SKIPPED"
fi
@@ -132,3 +132,51 @@ copy_reference_file() {
fi
fi
}
+
+# Retries a command a configurable number of times with backoff.
+#
+# The retry count is given by ATTEMPTS (default 60), the initial backoff
+# timeout is given by TIMEOUT in seconds (default 1.)
+#
+function retry_command() {
+ local max_attempts=${ATTEMPTS-3}
+ local timeout=${TIMEOUT-1}
+ local success_timeout=${SUCCESS_TIMEOUT-1}
+ local max_success_attempt=${SUCCESS_ATTEMPTS-1}
+ local attempt=0
+ local success_attempt=0
+ local exitCode=0
+
+ while (( attempt < max_attempts ))
+ do
+ set +e
+ "$@"
+ exitCode=$?
+ set -e
+
+ if [[ $exitCode == 0 ]]
+ then
+ success_attempt=$(( success_attempt + 1 ))
+ if (( success_attempt >= max_success_attempt))
+ then
+ break
+ else
+ sleep "$success_timeout"
+ continue
+ fi
+ fi
+
+ echo "$(date -u '+%T') Failure ($exitCode) Retrying in $timeout seconds..." 1>&2
+ sleep "$timeout"
+ success_attempt=0
+ attempt=$(( attempt + 1 ))
+ timeout=$(( timeout ))
+ done
+
+ if [[ $exitCode != 0 ]]
+ then
+ echo "$(date -u '+%T') Failed in the last attempt ($*)" 1>&2
+ fi
+
+ return $exitCode
+}
diff --git a/install-debian.sh b/install-debian.sh
index bd03af9..6e706af 100755
--- a/install-debian.sh
+++ b/install-debian.sh
@@ -90,16 +90,18 @@ fi
# ==============================================================================
if [ $nojenkins -eq 0 ]; then
- JENKINS_VERSION=2.32.1
- JENKINS_SHA=bfc226aabe2bb089623772950c4cc13aee613af1
+ JENKINS_VERSION=2.164.1
+ JENKINS_SHA=969df594d1958800cd7da55e19ca75cf65f7fbf0
JENKINS_URL=https://pkg.jenkins.io/debian-stable/binary/jenkins_${JENKINS_VERSION}_all.deb
+ JENKINS_UC=https://updates.jenkins.io
+ REF=/var/lib/jenkins/plugins
JENKINS_HOME=/var/lib/jenkins
JENKINS_PORT=$port
# Jenkins dependencies
apt-get -q=2 -V --no-install-recommends install \
default-jdk daemon psmisc adduser procps unzip
- pip install python-jenkins==0.4.14
+ pip install python-jenkins==1.4.0
echo -e "JENKINS_PORT=$JENKINS_PORT" >> /etc/environment
groupadd jenkins
@@ -124,7 +126,7 @@ if [ $nojenkins -eq 0 ]; then
sed -i -e "s#JENKINS_ARGS.*#JENKINS_ARGS\=\"${JENKINS_ARGS}\"#g" /etc/default/jenkins
source /etc/default/jenkins && \
- JAVA_ARGS="$JAVA_ARGS -Djenkins.install.runSetupWizard=false" && \
+ JAVA_ARGS="$JAVA_ARGS -Djenkins.install.runSetupWizard=false -Dhudson.model.DirectoryBrowserSupport.allowSymlinkEscape=true" && \
if [ -n "$http_proxy" ]; then \
PROXYSERVER=$(echo $http_proxy | sed -E 's/^http://' | sed -E 's/\///g' | sed -E 's/(.*):(.*)/\1/') && \
PROXYPORT=$(echo $http_proxy | sed -E 's/^http://' | sed -E 's/\///g' | sed -E 's/(.*):(.*)/\2/') && \
@@ -151,7 +153,7 @@ if [ $nojenkins -eq 0 ]; then
service jenkins start && \
sleep 30 && \
sudo -u jenkins java -jar /var/cache/jenkins/war/WEB-INF/jenkins-cli.jar \
- -s http://localhost:$JENKINS_PORT/fuego install-plugin \
+ -remoting -s http://localhost:$JENKINS_PORT/fuego install-plugin \
/fuego/frontend-install/plugins/flot-plotter-plugin/flot.hpi && \
sleep 10 && \
service jenkins stop
@@ -159,25 +161,27 @@ if [ $nojenkins -eq 0 ]; then
# install other plugins from Jenkins update center
# NOTE: not sure all of these are needed, but keep list
# compatible with 1.2.1 release for now
- /usr/local/bin/install-plugins.sh ant:1.7 \
- bouncycastle-api:2.16.2 \
+ /usr/local/bin/install-plugins.sh \
+ ant:1.9 \
+ antisamy-markup-formatter:1.5 \
+ bouncycastle-api:2.17 \
+ command-launcher:1.3 \
description-setter:1.10 \
- display-url-api:2.1.0 \
+ display-url-api:2.3.1 \
external-monitor-job:1.7 \
greenballs:1.15 \
icon-shim:2.0.3 \
- javadoc:1.4 \
- junit:1.21 \
- ldap:1.17 \
- mailer:1.20 \
- matrix-auth:1.7 \
- matrix-project:1.12 \
- antisamy-markup-formatter:1.5 \
- pam-auth:1.3 \
+ javadoc:1.5 \
+ jdk-tool:1.2 \
+ junit:1.27 \
+ ldap:1.20 \
+ mailer:1.23 \
+ matrix-auth:2.3 \
+ matrix-project:1.14 \
+ pam-auth:1.4 \
pegdown-formatter:1.3 \
- script-security:1.35 \
- structs:1.10 \
- windows-slaves:1.3.1
+ structs:1.17 \
+ windows-slaves:1.4
# make the mod.js symlink well after flot is installed
service jenkins start && sleep 30 && \
--
2.20.1
More information about the Fuego
mailing list