data-platform team mailing list archive
-
data-platform team
-
Mailing list archive
-
Message #00065
[Merge] soss/+source/charmed-spark:dpe-1966-build-from-source-new into soss/+source/charmed-spark:lp-3.5.0
Enrico Deusebio has proposed merging soss/+source/charmed-spark:dpe-1966-build-from-source-new into soss/+source/charmed-spark:lp-3.5.0.
Requested reviews:
Mehdi B. (medib)
Canonical Data Platform (data-platform)
For more details, see:
https://code.launchpad.net/~data-platform/soss/+source/charmed-spark/+git/charmed-spark/+merge/455623
First release of Spark 3.5.0-ubuntu0
--
Your team Canonical Data Platform is requested to review the proposed merge of soss/+source/charmed-spark:dpe-1966-build-from-source-new into soss/+source/charmed-spark:lp-3.5.0.
diff --git a/.launchpad.yaml b/.launchpad.yaml
new file mode 100644
index 0000000..ba0eced
--- /dev/null
+++ b/.launchpad.yaml
@@ -0,0 +1,109 @@
+pipeline:
+- build
+
+jobs:
+ build:
+ series: jammy
+ architectures: amd64
+ packages:
+ - wget
+ - openjdk-8-jdk
+ - maven
+ - git
+ - python3
+ - python3-pip
+ - python3-setuptools
+ - zip
+ snaps:
+ - name: yq
+ classic: true
+ environment:
+ JAVA_OPTS: "-Dhttp.proxyHost=10.10.10.1 -Dhttp.proxyPort=8222 -Dhttps.proxyHost=10.10.10.1 -Dhttps.proxyPort=8222 -Dhttp.nonProxyHosts"
+ NO_PROXY: localhost
+ ARTIFACTORY_BUILDING_URL: "https://canonical.jfrog.io/artifactory/dataplatform-spark/";
+ ARTIFACTORY_STAGING_URL: "https://canonical.jfrog.io/artifactory/dataplatform-spark-staging/";
+ RELEASE_NAME: "k8s"
+ run: |-
+ # try to read branch name (works only locally)
+
+ # We need to copy files under home due to the confined nature of the snap
+ cp pom.xml ~/pom.xml
+ SPARK_VERSION=$(yq .project.version ~/pom.xml)
+ rm ~/pom.xml
+ echo "Spark version: $SPARK_VERSION"
+
+ BRANCH_NAME=$(git branch --show-current)
+ echo "branch_name: $BRANCH_NAME"
+ # check if branch name is valid
+ if [ -z "$BRANCH_NAME" ]
+ then
+ # get branch revision id from git HEAD file
+ echo "No branch name given from git command! Try to get it from .git folder"
+ git_rev=$(cat .git/HEAD)
+ while read line; do
+ current_rev=$( echo $line | awk -F ' ' '{print $1}' )
+ branch_name=$( echo $line | awk -F ' ' '{print $2}' | awk -F '/' '{print $NF}' )
+ if [[ $current_rev = $git_rev ]]
+ then
+ echo "Branch name: $branch_name"
+ export BRANCH_NAME=$branch_name
+ fi
+ done < .git/packed-refs
+ else
+ export JAVA_OPTS=""
+ sed -i 's/<proxies>/<!-- <proxies>/' settings.xml
+ sed -i 's/<\/proxies>/<\/proxies> -->/' settings.xml
+ fi
+ [ ! -z "$BRANCH_NAME" ] && echo "Current branch: $BRANCH_NAME"
+ if [[ "$BRANCH_NAME" != "lp-"* ]]; then
+ export ARTIFACTORY_URL=$ARTIFACTORY_STAGING_URL
+ export RELEASE=false
+ else
+ export ARTIFACTORY_URL=$ARTIFACTORY_BUILDING_URL
+ export RELEASE=true
+ fi
+ echo "Selected artifactory: $ARTIFACTORY_URL"
+ echo "Release artifact: $RELEASE"
+ # check artifactory credentials
+ [ -z "PIP_INDEX_URL" ] && exit 1
+ [ ! -z "$PIP_INDEX_URL" ] && echo "Env variable exists :) "
+ [ ! -z "$PIP_INDEX_URL" ] && export USERNAME=$(echo "${PIP_INDEX_URL#https://}"; | awk -F '@' '{print $1}' | awk -F ':' '{print $1}')
+ [ ! -z "$PIP_INDEX_URL" ] && export PASSWORD=$(echo "${PIP_INDEX_URL#https://}"; | awk -F '@' '{print $1}' | awk -F ':' '{print $2}')
+ echo "ARTIFACTORY TO BE USED: $ARTIFACTORY_URL"
+ echo "ARTIFACTORY USERNAME: $USERNAME"
+ # check release name
+ current_date=$(date '+%Y%m%d%H%M%S')
+ CANONICAL_PATCH_VERSION=$(cat PATCH_VERSION )
+ CANONICAL_PATCH_VERSION+="-$current_date"
+ export CANONICAL_PATCH_VERSION=$CANONICAL_PATCH_VERSION
+ echo "Canonical patch: $RELEASE_NAME"
+ # set java version to Java 8
+ update-java-alternatives -s $(update-java-alternatives -l | grep '1\.8' | cut -d " " -f1) || echo '.'
+ java -version
+ # configure setting for maven repository
+ mkdir ~/.m2
+ mv settings.xml ~/.m2/settings.xml
+ cat ~/.m2/settings.xml
+ # start building process
+ echo "Start building Spark..."
+ # build spark
+ ./dev/make-distribution.sh --pip --tgz --name $RELEASE_NAME -Pkubernetes -Phive -Phive-thriftserver
+ FILE=spark-$SPARK_VERSION-$CANONICAL_PATCH_VERSION-bin-$RELEASE_NAME.tgz
+ if [ -f "$FILE" ]; then
+ echo "$FILE exists. Spark is correctly built."
+ else
+ echo "$FILE does not exist. Exit..."
+ exit 1
+ fi
+ # compute checksum
+ sha512sum spark-$SPARK_VERSION-$CANONICAL_PATCH_VERSION-bin-$RELEASE_NAME.tgz > spark-$SPARK_VERSION-$CANONICAL_PATCH_VERSION-bin-$RELEASE_NAME.tgz.sha512
+ cd ~/.m2
+ # compress local maven repository
+ zip -rq repository.zip repository
+ cp repository.zip /build/lpci/project/
+ cd /build/lpci/project/
+ output:
+ paths:
+ - spark-*-bin-*.tgz
+ - spark-*-bin-*.tgz.sha512
+ - repository.zip
diff --git a/PATCH_VERSION b/PATCH_VERSION
new file mode 100644
index 0000000..852dc9e
--- /dev/null
+++ b/PATCH_VERSION
@@ -0,0 +1 @@
+ubuntu0
diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
index ef7c010..3fa53ee 100755
--- a/dev/make-distribution.sh
+++ b/dev/make-distribution.sh
@@ -36,7 +36,7 @@ MAKE_TGZ=false
MAKE_PIP=false
MAKE_R=false
NAME=none
-MVN="$SPARK_HOME/build/mvn"
+MVN="mvn"
function exit_with_usage {
set +x
@@ -153,7 +153,7 @@ fi
echo "Spark version is $VERSION"
if [ "$MAKE_TGZ" == "true" ]; then
- echo "Making spark-$VERSION-bin-$NAME.tgz"
+ echo "Making spark-$VERSION-$CANONICAL_PATCH_VERSION-bin-$NAME.tgz"
else
echo "Making distribution for Spark $VERSION in '$DISTDIR'..."
fi
@@ -289,7 +289,7 @@ if [ -d "$SPARK_HOME/R/lib/SparkR" ]; then
fi
if [ "$MAKE_TGZ" == "true" ]; then
- TARDIR_NAME=spark-$VERSION-bin-$NAME
+ TARDIR_NAME=spark-$VERSION-$CANONICAL_PATCH_VERSION-bin-$NAME
TARDIR="$SPARK_HOME/$TARDIR_NAME"
rm -rf "$TARDIR"
cp -r "$DISTDIR" "$TARDIR"
@@ -297,6 +297,6 @@ if [ "$MAKE_TGZ" == "true" ]; then
if [ "$(uname -s)" = "Darwin" ]; then
TAR="tar --no-mac-metadata --no-xattrs --no-fflags"
fi
- $TAR -czf "spark-$VERSION-bin-$NAME.tgz" -C "$SPARK_HOME" "$TARDIR_NAME"
+ $TAR -czf "spark-$VERSION-$CANONICAL_PATCH_VERSION-bin-$NAME.tgz" -C "$SPARK_HOME" "$TARDIR_NAME"
rm -rf "$TARDIR"
fi
diff --git a/pom.xml b/pom.xml
index 93d696d..245d38a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,7 +115,7 @@
<java.version>1.8</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
- <maven.version>3.8.8</maven.version>
+ <maven.version>3.6.3</maven.version>
<exec-maven-plugin.version>3.1.0</exec-maven-plugin.version>
<sbt.project.name>spark</sbt.project.name>
<asm.version>9.5</asm.version>
@@ -175,7 +175,7 @@
<scala.version>2.12.18</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
- <!-- dont update scala-maven-plugin to version 4.8.1 SPARK-42809 and SPARK-43595 -->
+ <!-- dont update scala-maven-plugin to version 4.8.1 SPARK-42809 and SPARK-43595 -->
<scala-maven-plugin.version>4.8.0</scala-maven-plugin.version>
<maven.scaladoc.skip>false</maven.scaladoc.skip>
<versions-maven-plugin.version>2.16.0</versions-maven-plugin.version>
diff --git a/settings.xml b/settings.xml
new file mode 100644
index 0000000..dcb5cf7
--- /dev/null
+++ b/settings.xml
@@ -0,0 +1,81 @@
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd";>
+ <servers>
+ <server>
+ <username>${env.USERNAME}</username>
+ <password>${env.PASSWORD}</password>
+ <id>central</id>
+ </server>
+ <server>
+ <username>${env.USERNAME}</username>
+ <password>${env.PASSWORD}</password>
+ <id>snapshots</id>
+ </server>
+ </servers>
+ <profiles>
+ <profile>
+ <repositories>
+ <repository>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <id>central</id>
+ <name>dataplatform-spark</name>
+ <url>${env.ARTIFACTORY_URL}</url>
+ </repository>
+ <repository>
+ <snapshots />
+ <id>snapshots</id>
+ <name>dataplatform-spark</name>
+ <url>${env.ARTIFACTORY_URL}</url>
+ </repository>
+ </repositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <id>central</id>
+ <name>dataplatform-spark</name>
+ <url>${env.ARTIFACTORY_URL}</url>
+ </pluginRepository>
+ <pluginRepository>
+ <snapshots />
+ <id>snapshots</id>
+ <name>dataplatform-spark</name>
+ <url>${env.ARTIFACTORY_URL}</url>
+ </pluginRepository>
+ </pluginRepositories>
+ <id>artifactory</id>
+ </profile>
+ </profiles>
+ <activeProfiles>
+ <activeProfile>artifactory</activeProfile>
+ </activeProfiles>
+ <proxies>
+ <proxy>
+ <id>http_proxy</id>
+ <active>true</active>
+ <protocol>http</protocol>
+ <host>10.10.10.1</host>
+ <port>8222</port>
+ <nonProxyHosts>localhost</nonProxyHosts>
+ </proxy>
+ <proxy>
+ <id>https_proxy</id>
+ <active>true</active>
+ <protocol>https</protocol>
+ <host>10.10.10.1</host>
+ <port>8222</port>
+ <nonProxyHosts>localhost</nonProxyHosts>
+ </proxy>
+ </proxies>
+ <mirrors>
+ <mirror>
+ <id>central</id>
+ <name>Maven Repository Manager running on canonical.jfrog.io</name>
+ <url>${env.ARTIFACTORY_URL}</url>
+ <mirrorOf>*</mirrorOf>
+ </mirror>
+ </mirrors>
+</settings>
