data-platform team mailing list archive

Thread
Date

[Merge] soss/+source/charmed-spark:release-4.0.1-ubuntu0 into soss/+source/charmed-spark:lp-4.0.1

To: mp+492227@xxxxxxxxxxxxxxxxxx
From: Alex Batisse <mp+492227@xxxxxxxxxxxxxxxxxx>
Date: Tue, 09 Sep 2025 16:52:30 -0000
Reply-to: mp+492227@xxxxxxxxxxxxxxxxxx
Sender: noreply@xxxxxxxxxxxxx

Alex Batisse has proposed merging soss/+source/charmed-spark:release-4.0.1-ubuntu0 into soss/+source/charmed-spark:lp-4.0.1.

Commit message:
[DPE-8297] Release spark 4.0.1-ubuntu0

Requested reviews:
  Canonical Data Platform (data-platform)

For more details, see:
https://code.launchpad.net/~data-platform/soss/+source/charmed-spark/+git/charmed-spark/+merge/492227
-- 
Your team Canonical Data Platform is requested to review the proposed merge of soss/+source/charmed-spark:release-4.0.1-ubuntu0 into soss/+source/charmed-spark:lp-4.0.1.

diff --git a/.launchpad.yaml b/.launchpad.yaml
new file mode 100644
index 0000000..19ba057
--- /dev/null
+++ b/.launchpad.yaml
@@ -0,0 +1,296 @@
+pipeline:
+  - prepare
+  - build
+  - test
+  - release
+
+jobs:
+  prepare:
+    series: noble
+    architectures: amd64
+    environment:
+      BASE_BUILD_DIR: /home/ubuntu/charmed-spark
+      ARTIFACTORY_BUILDING_URL: "https://canonical.jfrog.io/artifactory/dataplatform-spark/";
+      ARTIFACTORY_STAGING_URL: "https://canonical.jfrog.io/artifactory/dataplatform-spark-staging/";
+    packages:
+      - git
+      - yq
+      - xmlstarlet
+    run: |-
+      echo -e "##################################"
+      echo -e "PREPARE"
+      echo -e "##################################"
+      
+      # try to read branch name (works only locally)
+      BRANCH_NAME=$(git branch --show-current)
+      COMMIT_ID=$(git rev-parse HEAD)
+      MODE="dev"
+      # check if branch name is valid
+      if [ -z "$BRANCH_NAME" ]
+      then
+        # get branch revision id from git HEAD file
+        echo "No branch name given from git command! Try to get it from .git folder"
+        git_rev=$(cat .git/HEAD)
+        while read line; do
+          current_rev=$( echo $line | awk -F ' ' '{print $1}' )
+          branch_name=$( echo $line | awk -F ' ' '{print $2}' | awk -F '/' '{print $NF}' )
+          if [[ $current_rev = $git_rev ]]
+          then
+            export BRANCH_NAME=$branch_name
+          fi
+        done < .git/packed-refs
+        MODE="release"
+      fi
+      
+      # create spark-build
+      mkdir -p ${BASE_BUILD_DIR}
+      
+      # move current project to the home dir 
+      cp -r * "${BASE_BUILD_DIR}/."
+      
+      cd ${BASE_BUILD_DIR}
+
+      # create release name
+      BUILD_VERSION=$(date '+%Y%m%d%H%M%S')
+      cp pom.xml ~/pom.xml
+      SPARK_VERSION=$(xmlstarlet sel -N x="http://maven.apache.org/POM/4.0.0"; -t -m "//x:project" -v "_:version" ~/pom.xml)
+      rm ~/pom.xml
+      
+      if [ ! -f "./lp/PATCH_VERSION" ]; 
+      then 
+        CANONICAL_PATCH_VERSION="ubuntu0"
+      else
+        CANONICAL_PATCH_VERSION=$(cat ./lp/PATCH_VERSION)
+      fi
+
+      VERSION="$CANONICAL_PATCH_VERSION-$BUILD_VERSION"
+           
+      [ ! -z "$BRANCH_NAME" ] && echo "Current branch: $BRANCH_NAME"
+      if [ -z "$RELEASE" ]; then
+        if [[ "$BRANCH_NAME" != "lp-"* ]]; then
+          RELEASE=false
+        else
+          RELEASE=true
+        fi
+      fi
+
+      mkdir -p logs
+      METADATA_FILE=${BASE_BUILD_DIR}/logs/metadata.yaml
+      echo -e "version: $VERSION\nrelease: $RELEASE\nbranch: $BRANCH_NAME\ncommit_id: $COMMIT_ID\nmode: $MODE\nbuild: $BUILD_VERSION\nspark_version: $SPARK_VERSION" > $METADATA_FILE
+      
+      echo "********"
+      echo "Metadata"
+      echo "********"
+      cat $METADATA_FILE
+      
+      # Setup settings.xml
+      mkdir -p ~/.m2
+      cp ./lp/settings.xml ~/.m2
+
+      CREDENTIALS_FILE="credentials.yaml"
+
+      if [[ "$RELEASE" = "true" ]]; then
+        export ARTIFACTORY_URL="${ARTIFACTORY_BUILDING_URL}"
+      else
+        export ARTIFACTORY_URL="${ARTIFACTORY_STAGING_URL}"
+      fi
+            
+      echo "Selected artifactory: $ARTIFACTORY_URL"
+      echo "Release artifact: $RELEASE"
+      
+      # check artifactory credentials
+      [ -z "PIP_INDEX_URL" ] && exit 1 
+      [ ! -z "$PIP_INDEX_URL" ] && echo "Env variable exists :) "
+      [ ! -z "$PIP_INDEX_URL" ] && export ARTIFACTORY_USER=$(echo "${PIP_INDEX_URL#https://}"; | awk -F '@' '{print $1}' | awk -F ':' '{print $1}')
+      [ ! -z "$PIP_INDEX_URL" ] && export ARTIFACTORY_PASSWORD=$(echo "${PIP_INDEX_URL#https://}"; | awk -F '@' '{print $1}' | awk -F ':' '{print $2}')
+
+      echo -e "url: ${ARTIFACTORY_URL}\nuser: ${ARTIFACTORY_USER}\npassword: ${ARTIFACTORY_PASSWORD}" > $CREDENTIALS_FILE
+
+  build:
+    series: noble
+    architectures: amd64
+    environment:
+      BASE_BUILD_DIR: /home/ubuntu/charmed-spark
+      HTTPS_PROXY: 10.10.10.1:8222
+      HTTP_PROXY: 10.10.10.1:8222
+      JAVA_OPTS: "-Dhttp.proxyHost=10.10.10.1 -Dhttp.proxyPort=8222 -Dhttps.proxyHost=10.10.10.1 -Dhttps.proxyPort=8222 -Dhttp.nonProxyHosts=localhost"
+      NO_PROXY: localhost
+    packages:
+      - build-essential
+      - wget
+      - maven
+      - openjdk-17-jdk
+      - tcpdump
+      - yq
+    run: |-
+      echo -e "##################################"
+      echo -e "BUILD"
+      echo -e "##################################"
+
+      METADATA_FILE=${BASE_BUILD_DIR}/logs/metadata.yaml
+      CREDENTIALS_FILE=${BASE_BUILD_DIR}/credentials.yaml
+
+      RELEASE=$(cat $METADATA_FILE | yq .release | tr -d '"')
+      BRANCH=$(cat $METADATA_FILE | yq .branch | tr -d '"')
+      COMMIT_ID=$(cat $METADATA_FILE | yq .commit_id | tr -d '"')
+      MODE=$(cat $METADATA_FILE | yq .mode | tr -d '"')
+      VERSION=$(cat $METADATA_FILE | yq .version | tr -d '"')
+      SPARK_VERSION=$(cat $METADATA_FILE | yq .spark_version | tr -d '"')
+      
+      export ARTIFACTORY_URL=$(cat $CREDENTIALS_FILE | yq .url | tr -d '"')
+      export ARTIFACTORY_USER=$(cat $CREDENTIALS_FILE | yq .user | tr -d '"')
+      export ARTIFACTORY_PASSWORD=$(cat $CREDENTIALS_FILE | yq .password | tr -d '"')
+        
+      if [ -n "$MODE" ] &&  [ "$MODE" = "dev" ];
+      then 
+        export NO_PROXY="*"
+        export JAVA_OPTS=
+        tcpdump -U -i any -w ${BASE_BUILD_DIR}/logs/dump.pcap &
+        sleep 5
+        TCPDUMP_PID=$(ps -e | pgrep tcpdump)
+        echo "TCPDUMP RUNNING AT: ${TCPDUMP_PID}"
+      fi 
+    
+      mkdir -p ~/.m2/repository  
+      
+      # set java version to Java 17
+      update-java-alternatives -s $(update-java-alternatives -l | grep '17' | cut -d " " -f1) || echo '.'
+      echo "Print java version"
+      java -version
+      echo "Print mnv version"
+      mvn -version
+
+      echo "Start building process."
+      pushd $BASE_BUILD_DIR
+      ./dev/make-distribution.sh --pip --tgz --mvn /usr/bin/mvn -Pkubernetes -Phive -Phive-thriftserver -Pyarn -Pvolcano -Phadoop-cloud
+      popd
+
+      BUILD_TAR="$BASE_BUILD_DIR/spark-$SPARK_VERSION-bin-3.4.1.tgz" # 3.4.1 is the hadoop version.
+      if [ ! -f "$BUILD_TAR" ]; then
+        echo "$BUILD_TAR does not exist."
+        exit 1
+      fi
+
+      # cp $BUILD_TAR ${BASE_BUILD_DIR}/.
+      mv ~/.m2 ${BASE_BUILD_DIR}/.
+      
+      if [ -n "$TCPDUMP_PID" ];
+      then
+        sleep 5
+        echo "Killing tcpdump process: ${TCPDUMP_PID}"
+        kill -9 "${TCPDUMP_PID}"
+      fi
+
+  test:
+    series: noble
+    architectures: amd64
+    environment:
+      BASE_BUILD_DIR: /home/ubuntu/charmed-spark
+      HTTPS_PROXY: 10.10.10.1:8222
+      HTTP_PROXY: 10.10.10.1:8222
+      JAVA_OPTS: "-Dhttp.proxyHost=10.10.10.1 -Dhttp.proxyPort=8222 -Dhttps.proxyHost=10.10.10.1 -Dhttps.proxyPort=8222 -Dhttp.nonProxyHosts=localhost"
+      NO_PROXY: localhost
+    packages:
+      - build-essential
+      - wget
+      - maven
+      - openjdk-17-jdk
+      - tcpdump
+      - yq
+    run: |-
+      echo -e "##################################"
+      echo -e "TEST"
+      echo -e "##################################"
+
+      METADATA_FILE=${BASE_BUILD_DIR}/logs/metadata.yaml
+      CREDENTIALS_FILE=${BASE_BUILD_DIR}/credentials.yaml
+
+      RELEASE=$(cat $METADATA_FILE | yq .release | tr -d '"')
+      BRANCH=$(cat $METADATA_FILE | yq .branch | tr -d '"')
+      COMMIT_ID=$(cat $METADATA_FILE | yq .commit_id | tr -d '"')
+      MODE=$(cat $METADATA_FILE | yq .mode | tr -d '"')
+      VERSION=$(cat $METADATA_FILE | yq .version | tr -d '"')
+      SPARK_VERSION=$(cat $METADATA_FILE | yq .spark_version | tr -d '"')
+
+      # set java version to Java 17
+      update-java-alternatives -s $(update-java-alternatives -l | grep '17' | cut -d " " -f1) || echo '.'
+      echo "Print java version"
+
+      mkdir test
+      cd test
+      mkdir spark
+      cp ${BASE_BUILD_DIR}/spark-*.tgz .
+      TARBALL=$(find . -name "spark-${SPARK_VERSION}-bin-3.4.1.tgz" | tail -n1 )
+      echo "TARBALL: $TARBALL"
+      tar -xvzf $TARBALL -C spark --strip-components=1
+
+      SPARK_PARENT=$(pwd)
+      SPARK_RELEASE=$SPARK_PARENT/spark
+      echo "Start Spark job!"
+      /bin/bash $SPARK_RELEASE/bin/spark-submit --master=local[1] --class org.apache.spark.examples.SparkPi $SPARK_RELEASE/examples/jars/spark-examples_2.13-$SPARK_VERSION.jar 1000 &> log.txt
+      echo "Spark job logs:"
+      tail -n 100 log.txt
+      if [ $(grep -Ri "Pi is roughly" log.txt | wc -l) -lt 1 ]; then
+        echo "Something went wrong with the spark job."
+        exit 1
+      fi
+      # remove test folder
+      cd ..
+      rm -rf test
+  
+  release:
+    series: noble
+    architectures: amd64
+    environment:
+      BASE_BUILD_DIR: /home/ubuntu/charmed-spark
+    packages:
+      - zip
+      - yq
+    run: |-
+      echo -e "##################################"
+      echo -e "PACKAGE RELEASE ARTIFACTS"
+      echo -e "##################################"
+
+      METADATA_FILE=${BASE_BUILD_DIR}/logs/metadata.yaml
+
+      VERSION=$(cat $METADATA_FILE | yq .version | tr -d '"')
+      BUILD_VERSION=$(cat $METADATA_FILE | yq .build | tr -d '"')
+      SPARK_VERSION=$(cat $METADATA_FILE | yq .spark_version | tr -d '"')
+
+      # copy the tarball to the launchpad build directory
+      cp ${BASE_BUILD_DIR}/spark-*.tgz ./
+      
+
+      # compute the checksum
+      FILE=$(find . -name "spark-${SPARK_VERSION}-bin-3.4.1.tgz" | tail -n1 )
+      # FILENAME=${FILE##*/apache-}
+      TARBALL="${FILE%bin-3.4.1.tgz}${VERSION}-bin-k8s.tgz"
+
+      echo "File: $FILE"
+      echo "Tarball: $TARBALL"
+
+      mv $FILE $TARBALL
+  
+      echo "Distribution: ${TARBALL}"
+      sha512sum "${TARBALL}" > "${TARBALL}.sha512"
+  
+      PROJECT_ROOT="${PWD}"
+  
+      # .m2 repos
+      pushd $BASE_BUILD_DIR/.m2
+      zip -r repository.zip repository
+      mv repository.zip ${PROJECT_ROOT}/.
+      popd
+  
+      # Logs
+      pushd $BASE_BUILD_DIR
+      zip -r logs.zip logs
+      mv logs.zip ${PROJECT_ROOT}/logs_$VERSION.zip
+      popd
+
+    output:
+      paths:
+        - spark-*.tgz
+        - spark-*.tgz.sha512
+        - repository.zip
+        - logs_*.zip
\ No newline at end of file
diff --git a/lp/PATCH_VERSION b/lp/PATCH_VERSION
new file mode 100644
index 0000000..faee25c
--- /dev/null
+++ b/lp/PATCH_VERSION
@@ -0,0 +1 @@
+ubuntu0
\ No newline at end of file
diff --git a/lp/settings.xml b/lp/settings.xml
new file mode 100644
index 0000000..ee4c784
--- /dev/null
+++ b/lp/settings.xml
@@ -0,0 +1,81 @@
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd";>
+  <servers>
+    <server>
+      <username>${env.ARTIFACTORY_USER}</username>
+      <password>${env.ARTIFACTORY_PASSWORD}</password>
+      <id>central</id>
+    </server>
+    <server>
+      <username>${env.ARTIFACTORY_USER}</username>
+      <password>${env.ARTIFACTORY_PASSWORD}</password>
+      <id>snapshots</id>
+    </server>
+  </servers>
+  <profiles>
+    <profile>
+      <repositories>
+        <repository>
+          <snapshots>
+            <enabled>false</enabled>
+          </snapshots>
+          <id>central</id>
+          <name>dataplatform-spark</name>
+          <url>${env.ARTIFACTORY_URL}</url>
+        </repository>
+	<repository>
+          <snapshots />
+          <id>snapshots</id>
+          <name>dataplatform-spark</name>
+          <url>${env.ARTIFACTORY_URL}</url>
+        </repository>
+      </repositories>
+      <pluginRepositories>
+        <pluginRepository>
+          <snapshots>
+            <enabled>false</enabled>
+          </snapshots>
+          <id>central</id>
+          <name>dataplatform-spark</name>
+          <url>${env.ARTIFACTORY_URL}</url>
+        </pluginRepository>
+        <pluginRepository>
+          <snapshots />
+          <id>snapshots</id>
+          <name>dataplatform-spark</name>
+          <url>${env.ARTIFACTORY_URL}</url>
+        </pluginRepository>
+      </pluginRepositories>
+      <id>artifactory</id>
+    </profile>
+  </profiles>
+  <activeProfiles>
+    <activeProfile>artifactory</activeProfile>
+  </activeProfiles>
+  <proxies>
+    <proxy>
+      <id>http_proxy</id>
+      <active>true</active>
+      <protocol>http</protocol>
+      <host>10.10.10.1</host>
+      <port>8222</port>
+      <nonProxyHosts>${env.NO_PROXY}</nonProxyHosts>
+    </proxy>
+    <proxy>
+      <id>https_proxy</id>
+      <active>true</active>
+      <protocol>https</protocol>
+      <host>10.10.10.1</host>
+      <port>8222</port>
+      <nonProxyHosts>${env.NO_PROXY}</nonProxyHosts>
+    </proxy>
+  </proxies>
+  <mirrors>
+    <mirror>
+      <id>central</id>
+      <name>Maven Repository Manager running on canonical.jfrog.io</name>
+      <url>${env.ARTIFACTORY_URL}</url>
+      <mirrorOf>*</mirrorOf>
+    </mirror>
+  </mirrors>
+</settings>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 2292214..a82667b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,7 +116,7 @@
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <java.version>17</java.version>
     <maven.compiler.release>${java.version}</maven.compiler.release>
-    <maven.version>3.9.9</maven.version>
+    <maven.version>3.8.7</maven.version>
     <exec-maven-plugin.version>3.2.0</exec-maven-plugin.version>
     <sbt.project.name>spark</sbt.project.name>
     <asm.version>9.7.1</asm.version>

Follow ups

Re: [Merge] soss/+source/charmed-spark:release-4.0.1-ubuntu0 into soss/+source/charmed-spark:lp-4.0.1
From: Alex Batisse, 2025-09-10