launchpad-reviewers team mailing list archive

[Scott Moser <smoser@xxxxxxxxxx>]

Scott Moser has proposed merging lp:~smoser/maas/maas-import-ephemeral into lp:maas.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~smoser/maas/maas-import-ephemeral/+merge/101155

This adds support for creating and managing iscsi targets that can
then be booted from via cobbler.  It sets up a new distro and profile
that should then boot instances off of the read-only iscsi disks.

Notes:
 * creates in cobbler:
   * new distro named <release>-<arch>-maas-ephemeral
   * new profile named maas-<release>-<arch>-commissioning
 * adds maas-commissioning preseed that does nothing
   more than render MAAS_PRESEED
 * adds script maas-import-ephemeral
   * This needs to be run at some point after installation, and
     can be run for updates similar to how maas-import-isos is.
     Failure to run its update will not result in failed use like
     import-isos does (ie, archive moving doesn't hurt this)
 * adds script maas-cloudimg2ephemeral
   This converts the cloudimg from cloud-images.ubuntu.com into
   something suitable for use here, and generates new initramfs
   that uses overlay

Still to do:
 * cleanup of old images (currently doees not do this)
 * replace cloud-images and repacking code with maas-ephemeral server
   maas-commissioning.preseed
 * it seems the interaction with tgt needs some work


Things needed in packaging:
 * depend on tgt
 * get the /var/lib/maas/ephemeral/tgt.conf file included
   in tgt's config. (so its iscsi targets are started on boot).
   I suggest doing this by:
   * append (if not present) to /etc/tgt/targets.conf this:
     include /etc/tgt/conf.d/*.conf
   * create /etc/tgt/conf.d
   * symlink /var/lib/maas/ephemeral/tgt.conf /etc/tgt/conf.d
-- 
https://code.launchpad.net/~smoser/maas/maas-import-ephemeral/+merge/101155
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~smoser/maas/maas-import-ephemeral into lp:maas.

=== added file 'contrib/preseeds/maas-commissioning.preseed'
--- contrib/preseeds/maas-commissioning.preseed	1970-01-01 00:00:00 +0000
+++ contrib/preseeds/maas-commissioning.preseed	2012-04-06 20:31:19 +0000
@@ -0,0 +1,2 @@
+# maas_preseed
+$SNIPPET('maas_preseed')

=== added file 'etc/maas/import_ephemerals'
--- etc/maas/import_ephemerals	1970-01-01 00:00:00 +0000
+++ etc/maas/import_ephemerals	2012-04-06 20:31:19 +0000
@@ -0,0 +1,17 @@
+## get default settings from maas_import_iso 
+[ ! -f /etc/maas/maas_import_iso ] || . /etc/maas/maas_import_iso
+
+#REMOTE_IMAGES_MIRROR="http://cloud-images.ubuntu.com";
+#ISCSI_TARGET_IP="" # defaults to cobbler server setting
+#EPH_KOPTS_CONSOLE="console=${CONSOLE:-ttyS0,9600n8}"
+#EPH_KOPTS_ISCSI="ip=dhcp iscsi_target_name=@@iscsi_target@@ iscsi_target_ip=@@iscsi_target_ip@@ iscsi_target_port=3260"
+#EPH_KOPTS_ROOT="root=cloudimg-rootfs ro"
+#EPH_KOPTS_LOGGING="log_host=@@server_ip@@ log_port=514"
+#EPH_UPDATE_CMD="maas-cloudimg2ephemeral"
+#TARGET_NAME_PREFIX="iqn.2004-05.com.ubuntu:maas:"
+#DATA_DIR="/var/lib/maas/ephemeral"
+#RELEASES="precise"
+#ARCHES="amd64 i386"
+#KSDIR="/var/lib/cobbler/kickstarts"
+#KICKSTART="$KSDIR/maas-commissioning.preseed"
+#TARBALL_CACHE_D="" # set to cache downloaded content

=== added file 'scripts/maas-cloudimg2ephemeral'
--- scripts/maas-cloudimg2ephemeral	1970-01-01 00:00:00 +0000
+++ scripts/maas-cloudimg2ephemeral	2012-04-06 20:31:19 +0000
@@ -0,0 +1,493 @@
+#!/bin/bash
+#
+# maas-cloudimg2ephemeral - update a cloud image to make it sufficient
+#                           for use as a maas ephemeral image
+#
+# Copyright (C) 2011-2012 Canonical
+#
+# Authors:
+#    Scott Moser <scott.moser@xxxxxxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+VERBOSITY=0
+
+error() { echo "$@" 1>&2; }
+errorp() { printf "$@" 1>&2; }
+fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
+failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
+
+Usage() {
+	cat <<EOF
+Usage: ${0##*/} [ options ] disk kernel initrd
+
+   Update the image 'disk', kernel 'kernel' and initrd 'initrd'
+   
+   Expects to receive in a cloudimg disk image with no partition
+   table, and will update it for maas ephemeral use.
+
+   options:
+      -v | --verbose      increase verbosity
+EOF
+}
+
+bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
+cleanup() {
+	[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || {
+		unmount_under "${TEMP_D}" &&
+		rm -Rf "${TEMP_D}"
+	}
+}
+
+debug() {
+	local level=${1}; shift;
+	[ "${level}" -gt "${VERBOSITY}" ] && return
+	error "${@}"
+}
+
+unmount_under() {
+	# unmount_under(dir)
+	# unmount all mounts under 'dir'
+	[ -f /proc/mounts ] ||
+		{ error "/proc/mounts not a file"; return 1; }
+	tac /proc/mounts | sh -c '
+		under=$1
+   		while read s mp t opt a b ; do 
+      		[ "${mp#${under}}" != "${mp}" ] || continue; 
+      		umount $mp || 
+				{ echo "failed umount $mp, waiting, trying again" 1>&2; 
+				  sleep 10;
+				  umount $mp || exit 1; }
+   		done' -- "$1"
+}
+
+loop_mount() {
+	# Create more loop nodes, if necessary
+	local mounts=$(grep -c /dev/loop /proc/mounts) || mounts=0
+	local loops=$(ls /dev/loop* | wc -l) || loops=0
+	if [ $mounts -ge $loops ]; then
+		mknod -m 660 /dev/loop$loops b 7 $loops &&
+		chown root:disk /dev/loop$loops ||
+			return 1
+	fi
+	# Do the loop mount
+	mount -o loop "$1" "$2"
+}
+
+mount_callback_umount() {
+	# mount_callback_umount(img_or_device, func, args)
+	# mount the image given, call function with args,
+	# umount the image, return function's exit value
+	local device="$1" cb="$2" mp="" opts="" ret=0 m=""
+	shift 2;
+	mp=$(mktemp -d "$TEMP_D/mp.XXXXXX")
+	if [ -b "$device" ]; then
+		mount $opts "$device" "$mp" || return 1
+	else
+		loop_mount "$device" "$mp" || return 1
+	fi
+	for m in "/proc" "/sys"; do
+		[ -d "$mp/$m" ] || continue
+		mount --bind "$m" "$mp/$m" || {
+			error "failed to mount $mp/$m";
+			unmount_under "$mp";
+			return 1;
+		}
+	done
+	"$cb" "$mp" "$@"
+	ret=$?
+	unmount_under "$mp" && rmdir "$mp" ||
+		{ error "WARN! failed to umount $device from $mp"; return 2; }
+	return $ret
+}
+
+add_initramfs_hooks() {
+	local dir="$1" idir="" hook="" script=""
+	idir="$dir/etc/initramfs-tools"
+	mkdir -p "$idir/hooks" "$idir/scripts/init-bottom" ||
+		return 1
+	hook="$idir/hooks/overlay-ro"
+	cat > "$hook" <<"ENDEND"
+#!/bin/sh
+set -e
+
+PREREQS=""
+case $1 in
+	prereqs) echo "${PREREQS}"; exit 0;;
+esac
+
+. /usr/share/initramfs-tools/hook-functions
+
+##
+manual_add_modules overlayfs
+force_load overlayfs
+ 
+# vi: ts=4 noexpandtab
+ENDEND
+
+	[ $? -eq 0 ] || { error "failed to write $hook"; return 1; }
+
+	script="$idir/scripts/init-bottom/root-ro"
+	cat > "$script" <<"ENDEND"
+#!/bin/sh
+#  Copyright, 2012 Axel Heider
+#
+#  Based on scrpts from
+#    Sebastian P.
+#    Nicholas A. Schembri State College PA USA
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see
+#    <http://www.gnu.org/licenses/>.
+#
+#
+# Tested with Ubuntu 11.10
+#
+# Notes:
+#   * no changes to the root fs are made by this script. 
+#   * if /home/[user] is on the RO root fs, files are in ram and not saved.
+#
+# Install:
+#  put this file in /etc/initramfs-tools/scripts/init-bottom/root-ro
+#  chmod 0755 root-ro
+#  optional: clean up menu.lst, update-grub
+#  update-initramfs -u
+#
+# Disable read-only root fs
+#   * option 1: kernel boot parameter "disable-root-ro=true"
+#   * option 2: create file "/disable-root-ro"
+#
+# ROOT_RO_DRIVER variable controls which driver isused for the ro/rw layering
+#   Supported drivers are: overlayfs, aufs
+#  the kernel parameter "root-ro-driver=[driver]" can be used to initialize
+#  the variable ROOT_RO_DRIVER. If nothing is given, overlayfs is used.
+#
+
+# no pre requirement
+PREREQ=""
+
+prereqs()
+{
+    echo "${PREREQ}"
+}
+
+case "$1" in
+    prereqs)
+    prereqs
+    exit 0
+    ;;
+esac
+
+. /scripts/functions
+
+MYTAG="root-ro"
+DISABLE_MAGIC_FILE="/disable-root-ro"
+
+# parse kernel boot command line 
+ROOT_RO_DRIVER=
+DISABLE_ROOT_RO=
+for CMD_PARAM in $(cat /proc/cmdline); do 
+    case ${CMD_PARAM} in 
+        disable-root-ro=*)
+            DISABLE_ROOT_RO=${CMD_PARAM#disable-root-ro=}
+            ;;
+        root-ro-driver=*)
+            ROOT_RO_DRIVER=${CMD_PARAM#root-ro-driver=}
+            ;;
+    esac
+done
+
+# check if read-only root fs is disabled
+if [ ! -z "${DISABLE_ROOT_RO}" ]; then
+    log_warning_msg "${MYTAG}: disabled, found boot parameter disable-root-ro=${DISABLE_ROOT_RO}"
+    exit 0
+fi
+if [ -e "${rootmnt}${DISABLE_MAGIC_FILE}" ]; then
+    log_warning_msg "${MYTAG}: disabled, found file ${rootmnt}${DISABLE_MAGIC_FILE}"
+    exit 0
+fi
+
+# generic settings 
+# ${ROOT} and ${rootmnt} are predefined by caller of this script. Note that
+# the root fs ${rootmnt} it mounted readonly on the initrams, which fits nicely
+# for our purposes.
+ROOT_RW=/mnt/root-rw
+ROOT_RO=/mnt/root-ro
+
+# check if ${ROOT_RO_DRIVER} is defined, otherwise set default 
+if [ -z "${ROOT_RO_DRIVER}" ]; then
+    ROOT_RO_DRIVER=overlayfs
+fi
+# settings based in ${ROOT_RO_DRIVER}, stop here if unsupported. 
+case ${ROOT_RO_DRIVER} in
+    overlayfs)
+        MOUNT_PARMS="-t overlayfs -o lowerdir=${ROOT_RO},upperdir=${ROOT_RW} overlayfs-root ${rootmnt}"
+        ;;
+    aufs)
+        MOUNT_PARMS="-t aufs -o dirs=${ROOT_RW}:${ROOT_RO}=ro aufs-root ${rootmnt}"
+        ;;
+    *)
+        panic "${MYTAG} ERROR: invalide ROOT_RO_DRIVER ${ROOT_RO_DRIVER}"
+        ;;
+esac
+
+
+# check if kernel module exists 
+modprobe -qb ${ROOT_RO_DRIVER}
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: missing kernel module ${ROOT_RO_DRIVER}"
+    exit 0
+fi
+
+# make the mount point on the init root fs ${ROOT_RW}
+[ -d ${ROOT_RW} ] || mkdir -p ${ROOT_RW}
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to create ${ROOT_RW}"
+    exit 0
+fi
+
+# make the mount point on the init root fs ${ROOT_RO}
+[ -d ${ROOT_RO} ] || mkdir -p ${ROOT_RO}
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to create ${ROOT_RO}"
+    exit 0
+fi
+
+# mount a tempfs using the device name tmpfs-root at ${ROOT_RW}
+mount -t tmpfs tmpfs-root ${ROOT_RW}
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to create tmpfs"
+    exit 0
+fi
+
+
+# root is mounted on ${rootmnt}, move it to ${ROOT_RO}.
+mount --move ${rootmnt} ${ROOT_RO}
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to move root away from ${rootmnt} to ${ROOT_RO}"
+    exit 0
+fi
+
+# there is nothing left at ${rootmnt} now. So for any error we get we should
+# either do recovery to restore ${rootmnt} for drop to a initramfs shell using
+# "panic". Otherwise the boot process is very likely to fail with even more 
+# errors and leave the system in a wired state. 
+
+# mount virtual fs ${rootmnt} with rw-fs ${ROOT_RW} on top or ro-fs ${ROOT_RO}.
+mount ${MOUNT_PARMS}
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to create new ro/rw layerd ${rootmnt}"
+    # do recovery and try resoring the mount for ${rootmnt}
+    mount --move ${ROOT_RO} ${rootmnt}
+    if [ $? -ne 0 ]; then
+       # thats badm, drpo to s shell to let the user try fixing this
+       panic "${MYTAG} RECOVERY ERROR: failed to move ${ROOT_RO} back to ${rootmnt}"
+    fi
+    exit 0
+fi
+
+# now the real root fs is on ${ROOT_RO} of the init file system, our layered
+# root fs is set up at ${rootmnt}. So we can write anywhere in {rootmnt} and the
+# changes will end up in ${ROOT_RW} while ${ROOT_RO} it not touched. However 
+# ${ROOT_RO} and ${ROOT_RW} are on the initramfs root fs, which will be removed
+# an replaced by ${rootmnt}. Thus we must move ${ROOT_RO} and ${ROOT_RW} to the
+# rootfs visible later, ie. ${rootmnt}${ROOT_RO} and ${rootmnt}${ROOT_RO}.
+# Since the layered ro/rw is already up, these changes also end up on 
+# ${ROOT_RW} while ${ROOT_RO} is not touched.
+
+# move mount from ${ROOT_RO} to ${rootmnt}${ROOT_RO} 
+[ -d ${rootmnt}${ROOT_RO} ] || mkdir -p ${rootmnt}${ROOT_RO}
+mount --move ${ROOT_RO} ${rootmnt}${ROOT_RO}
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to move ${ROOT_RO} to ${rootmnt}${ROOT_RO}"
+    exit 0
+fi
+
+# move mount from ${ROOT_RW} to ${rootmnt}${ROOT_RW} 
+[ -d ${rootmnt}${ROOT_RW} ] || mkdir -p ${rootmnt}${ROOT_RW}
+mount --move ${ROOT_RW} ${rootmnt}${ROOT_RW}
+if [ $? -ne 0 ]; then
+    s "${MYTAG}: ERROR: failed to move ${ROOT_RW} to ${rootmnt}${ROOT_RW}"
+    exit 0
+fi
+
+# technically, everything is set up nicely now. Since ${rootmnt} had beend 
+# mounted read-only on the initfamfs already, ${rootmnt}${ROOT_RO} is it, too.
+# Now we init process could run - but unfortunately, we may have to prepare 
+# some more things here. 
+# Basically, there are two ways to deal with the read-only root fs. If the 
+# system is made aware of this, things can be simplified a lot.
+# If it is not, things need to be done to our best knowledge. 
+#
+# So we assume here, the system does not really know about our read-only root fs.
+#
+# Let's deal with /etc/fstab first. It usually contains an entry for the root 
+# fs, which is no longer valid now. We have to remove it and add our new 
+# ${ROOT_RO} entry. 
+# Remember we are still on the initramfs root fs here, so we have to work on
+# ${rootmnt}/etc/fstab. The original fstab is ${rootmnt}${ROOT_RO}/etc/fstab.
+ROOT_TYPE=$(cat /proc/mounts | grep ${ROOT} | cut -d' ' -f3)
+ROOT_OPTIONS=$(cat /proc/mounts | grep ${ROOT} | cut -d' ' -f4)
+cat <<EOF >${rootmnt}/etc/fstab
+#
+#  This fstab is in RAM, the real one can be found at ${ROOT_RO}/etc/fstab
+#  The original entry for '/' and all swap files have been removed.  The new 
+#  entry for the read-only the real root fs follows. Write access can be 
+#  enabled using:
+#    sudo mount -o remount,rw ${ROOT_RO}
+#  re-mounting it read-only is done using:
+#    sudo mount -o remount,ro ${ROOT_RO}
+#
+
+${ROOT} ${ROOT_RO} ${ROOT_TYPE} ${ROOT_OPTIONS} 0 0
+
+#
+#  remaining entries from the original ${ROOT_RO}/etc/fstab follow.
+#
+EOF
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to modify /etc/fstab (step 1)"
+    #exit 0
+fi
+
+#remove root entry and swap from fstab
+cat ${rootmnt}${ROOT_RO}/etc/fstab | grep -v ' / ' | grep -v swap >>${rootmnt}/etc/fstab
+if [ $? -ne 0 ]; then
+    log_failure_msg "${MYTAG} ERROR: failed to modify etc/fstab (step 2)"
+    #exit 0
+fi
+
+# now we are done. Additinal steps may be necessary depending on the actualy
+# distribution and/or its configuration. 
+
+log_success_msg "${MYTAG} sucessfully set up ro/tmpfs-rw layered root fs using ${ROOT_RO_DRIVER}"
+
+exit 0
+ENDEND
+	[ $? -eq 0 ] || { error "failed to write $script"; return 1; }
+	chmod 755 "$hook" "$script" ||
+		{ error "failed to chmod $hook, $script"; return 1; }
+}
+
+apply_updates() {
+	# apply_updates(dir, kernel_out, initramfs_out)
+	# update directory given, and pull out kernel and initramfs
+	# to given locations
+	local dir=$1 kernel_out=$2 initrd_out=$3
+	if [ -f "$dir/etc/resolv.conf" ]; then
+		mv "$dir/etc/resolv.conf" "$dir/etc/resolv.conf.dist" || return 1
+	fi
+	cp "/etc/resolv.conf" "$dir/etc/resolv.conf" ||
+		return 1
+
+	cat > "$dir/usr/sbin/policy-rc.d" <<"EOF"
+#!/bin/sh
+while true; do
+    case "$1" in
+        -*) shift ;;
+        makedev) exit 0 ;;
+        x11-common) exit 0 ;;
+        *) exit 101 ;;
+    esac
+done
+EOF
+	[ $? -eq 0 ] && chmod 755 "$dir/usr/sbin/policy-rc.d" ||
+		{ error "failed to write policy-rc.d"; return 1; }
+	
+	add_initramfs_hooks "$dir" || return
+
+	local prox="" apt_opts=""
+    out=$(apt-config shell prox Acquire::HTTP::Proxy) &&
+    	eval $out && [ -n "$prox" ] &&
+		apt_opts="--option=Acquire::HTTP::Proxy=${prox}"
+
+	apt_opts="${apt_opts} --option=Dpkg::Options::=--force-confold"
+	[ -n "${apt_opts}" ] &&
+		debug 1 "using apt options ${apt_opts} for install"
+
+	LC_ALL=C DEBIAN_FRONTEND=noninteractive \
+		apt_opts="${apt_opts}" chroot "$dir" sh -c '
+		mkdir -p /etc/iscsi && touch /etc/iscsi/iscsi.initramfs &&
+		apt-get -q ${apt_opts} update &&
+		apt-get remove "linux.*virtual" ${apt_opts} --assume-yes &&
+		apt-get ${apt_opts} install -q -y linux-server open-iscsi ||
+			exit
+		k=""
+		for i in /boot/vmlinuz-*; do
+			[ "${i%-virtual}" = "${i}" ] && k=${i}; done
+		ver=${k##*/vmlinuz-}
+		mkinitramfs -o /tmp/initrd.img $ver &&
+		cp $k /tmp/kernel.img && chmod ugo+r /tmp/kernel.img' </dev/null
+	[ $? -eq 0 ] || {
+		error "failed to install packages or mkinitramfs in chroot";
+		return 1;
+	}
+
+	mv "$dir/tmp/kernel.img" "$kernel_out" &&
+		mv "$dir/tmp/initrd.img" "$initrd_out" ||
+		{ error "failed to copy kernels out"; return 1; }
+	
+	rm -f "$dir/etc/resolv.conf"
+	[ ! -e "$dir/etc/resolv.conf.dist" ] ||
+		mv "$dir/etc/resolv.conf.dist" "$dir/etc/resolv.conf" ||
+		{ error "failed to replace resolv.conf"; return 1; }
+
+	return 0
+}
+
+short_opts="hv"
+long_opts="help,verbose"
+getopt_out=$(getopt --name "${0##*/}" \
+	--options "${short_opts}" --long "${long_opts}" -- "$@") &&
+	eval set -- "${getopt_out}" ||
+	bad_Usage
+
+while [ $# -ne 0 ]; do
+	cur=${1}; next=${2};
+	case "$cur" in
+		-h|--help) Usage ; exit 0;;
+		-v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
+		--) shift; break;;
+	esac
+	shift;
+done
+
+[ $# -eq 3 ] ||
+	bad_Usage "expected image, kernel, ramdisk"
+
+img="$1"
+kernel="$2"
+initrd="$3"
+
+TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
+   fail "failed to make tempdir"
+
+trap cleanup EXIT
+
+mount_callback_umount "$img" apply_updates \
+	"$kernel" "$initrd" ||
+	fail "failed to apply updates to $img"
+
+exit 0
+
+# vi: ts=4 noexpandtab

=== added file 'scripts/maas-import-ephemerals'
--- scripts/maas-import-ephemerals	1970-01-01 00:00:00 +0000
+++ scripts/maas-import-ephemerals	2012-04-06 20:31:19 +0000
@@ -0,0 +1,484 @@
+#!/bin/bash
+#
+# maas-import-ephemerals - sync and import ephemeral images
+#
+# Copyright (C) 2011-2012 Canonical
+#
+# Authors:
+#    Scott Moser <scott.moser@xxxxxxxxxxxxx>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+VERBOSITY=0
+REMOTE_IMAGES_MIRROR="http://cloud-images.ubuntu.com";
+CONSOLE="ttyS0,9600n8"
+EPH_KOPTS_CONSOLE="console=$CONSOLE"
+EPH_KOPTS_ISCSI="ip=dhcp iscsi_target_name=@@iscsi_target@@ iscsi_target_ip=@@iscsi_target_ip@@ iscsi_target_port=3260"
+EPH_KOPTS_ROOT="root=cloudimg-rootfs ro"
+EPH_KOPTS_LOGGING="log_host=@@server_ip@@ log_port=514"
+EPH_UPDATE_CMD="maas-cloudimg2ephemeral"
+TARGET_NAME_PREFIX="iqn.2004-05.com.ubuntu:maas:"
+DATA_DIR="/var/lib/maas/ephemeral"
+CONFIG="/etc/maas/import_ephemerals"
+RELEASES="precise"
+ARCHES="amd64 i386"
+STREAM="server"
+KSDIR="/var/lib/cobbler/kickstarts"
+KICKSTART="$KSDIR/maas-commissioning.preseed"
+SYS_TGT_CONF="/etc/tgt/targets.conf""
+
+# DATA_DIR layout is like:
+#   tgt.conf
+#   tgt.conf.d/
+#     <name>.conf ->
+#        ../release/stream/arch/serial.conf
+#   release/
+#     stream/
+#       arch/
+#         serial/
+#           kernel
+#           disk.img
+#           initrd
+#           my.conf
+
+error() { echo "$@" 1>&2; }
+errorp() { printf "$@" 1>&2; }
+fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
+failp() { [ $# -eq 0 ] || errorp "$@"; exit 1; }
+
+Usage() {
+	cat <<EOF
+Usage: ${0##*/} [ options ] <<ARGUMENTS>>
+
+   Import ephemeral (commissioning) images into maas
+   Settings are read from /etc/maas/maas_import_ephemerals
+
+   options:
+      -i | --import       initial import or freshen the images
+      -c | --update-check check existing imported data versus available
+                          in mirror.  exits 0 if an update is needed or
+                          an initial import is needed.
+      -u | --update       update parameters on cobbler profiles per config
+EOF
+}
+
+bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; }
+cleanup() {
+	[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
+}
+
+debug() {
+	local level=${1}; shift;
+	[ "${level}" -gt "${VERBOSITY}" ] && return
+	error "${@}"
+}
+arch2u() {
+	# arch2ubuntu
+	_RET=$1
+	case "$1" in
+		i?86) _RET=i386;;
+		x86_64) _RET=amd64;;
+	esac
+}
+arch2cob() {
+	# arch 2 cobbler arch
+	_RET=$1
+	case "$1" in
+		i?86) _RET=i386;;
+		amd64) _RET=x86_64;;
+	esac
+}
+query_remote() {
+	# query /query data at REMOTE_IMAGES_MIRROR
+	# returns 7 values prefixed with 'r_'
+	local iarch=$1 irelease=$2 istream=$3 out=""
+	local burl="${REMOTE_IMAGES_MIRROR}/query"
+	local url="$burl/$irelease/$istream/released-dl.current.txt"
+	mkdir -p "$TEMP_D/query"
+	local target="$TEMP_D/query/$release.$stream"
+	if [ ! -f "$TEMP_D/query/$release.$stream" ]; then
+		wget -q "$url" -O "$target.tmp" && mv "$target.tmp" "$target" ||
+			{ error "failed to get $url"; return 1; }
+	fi
+
+	r_release=""; r_stream=""; r_label=""; r_serial="";
+	r_arch=""; r_url=""; r_name=""
+
+	out=$(awk '-F\t' '$1 == release && $2 == stream && $5 == arch { print $3, $4, $6, $7 }' \
+		"arch=$iarch" "release=$irelease" "stream=$istream" \
+		"$target") && [ -n "$out" ] ||
+		return 1
+
+	set -- ${out}
+	r_release=$irelease
+	r_stream=$istream
+	r_label=$1;
+	r_serial=$2;
+	r_arch=$iarch
+	r_url=$3
+	r_name=$4
+	return
+}
+
+query_local() {
+	local iarch=$1 irelease=$2 istream=$3 out=""
+	local label="" name="" serial="" url=""
+
+	local found=""
+	for i in "${DATA_DIR}/"$irelease/$istream/$iarch/*/info; do
+		[ -f "$i" ] && found=$i
+	done
+
+	l_release=""; l_stream=""; l_label=""; l_serial="";
+	l_arch=""; l_url=""; l_name=""
+	if [ -n "$found" ]; then
+		. "$found"
+		l_release="$release";
+		l_stream="$stream";
+		l_label="$label";
+		l_serial="$serial";
+		l_arch="$arch";
+		l_url="$url";
+		l_name="$name";
+		l_dir="${found%/*}";
+	fi
+}
+serial_gt() {
+	# is $1 a larger serial than $2 ?
+	local a=${1:-0} b=${2:-0}
+	case "$a" in
+		*.[0-9]) a="${a%.*}${a##*.}";;
+	esac
+	case "$b" in
+		*.[0-9]) b="${b%.*}${b##*.}";;
+	esac
+	[ $a -gt $b ]
+}
+
+prep_dir() {
+	local wd="$1" exdir="" tarball=""
+	shift
+	local release=$1 stream=$2 label=$3 serial=$4 arch=$5 url=$6 name=$7
+	local furl="$REMOTE_IMAGES_MIRROR/$url"
+
+	mkdir -p "$wd"
+	cat > "$wd/info" <<EOF
+release=$release
+stream=$stream
+label=$label
+serial=$serial
+arch=$arch
+url=$url
+name=$name
+EOF
+
+	# download
+	local cachepath="${TARBALL_CACHE_D}/${name}.tar.gz" rmtar=""
+	if [ -f "$cachepath" ]; then
+		tarball="${cachepath}"
+	elif [ -n "$TARBALL_CACHE_D" ]; then
+		mkdir -p "$TARBALL_CACHE_D"
+		debug 1 "downloading $name from $furl to local cache"
+		wget "$furl" --progress=dot:mega -O "${cachepath}.part$$" &&
+			mv "$cachepath.part$$" "$cachepath" || {
+			rm "$cachepath.part$$"
+			error "failed to download $furl";
+			return 1;
+		}
+		tarball="${cachepath}"
+	else
+		debug 1 "downloading $name from $furl"
+		tarball="$wd/dist.tar.gz"
+		wget "$furl" --progress=dot:mega -O "${tarball}" ||
+			{ error "failed to download $furl"; return 1; }
+		rmtar="$tarball"
+	fi
+
+	# extract
+	exdir="$wd/.xx"
+	mkdir -p "$exdir" &&
+		debug 1 "extracting tarball" &&
+		tar -Sxzf - -C  "$exdir" < "$tarball" ||
+		{ error "failed to extract tarball from $furl"; return 1; }
+
+	local x="" img="" kernel="" initrd=""
+	for x in "$exdir/"*.img; do
+		[ -f "$x" ] && img="$x" && break
+	done
+
+	for x in "$exdir/kernel" "$exdir/"*-vmlinuz*; do
+		[ -f "$x" ] && kernel="$x" && break
+	done
+
+	for x in "$exdir/initrd" "$exdir/"*-initrd*; do
+		[ -f "$x" ] && initrd="$x" && break
+	done
+
+	[ -n "$img" ] || { error "failed to find image in $furl"; return 1; }
+	mv "$img" "$wd/disk.img" ||
+		{ error "failed to move extracted image to $wd/disk.img"; return 1; }
+
+	[ -z "$kernel" ] || mv "$kernel" "$wd/kernel" ||
+		{ error "failed to move extracted kernel to $wd/kernel"; return 1; }
+
+	[ -z "$initrd" ] || mv "$initrd" "$wd/initrd" ||
+		{ error "failed to move extracted kernel to $wd/initrd"; return 1; }
+
+	rm -Rf "$exdir" || { error "failed to cleanup extract dir"; return 1; }
+	{ [ -z "$rmtar" ] || rm "$rmtar"; } ||
+		{ error "failed to remove temporary tarball $rmtar"; return 1; }
+
+	if [ -n "$EPH_UPDATE_CMD" ]; then
+		# update
+		debug 1 "invoking: ${EPH_UPDATE_CMD[*]} ./disk.img ./kernel ./initrd"
+		"${EPH_UPDATE_CMD[@]}" "$wd/disk.img" "$wd/kernel" "$wd/initrd" ||
+			{ error "failed to apply updates to $img"; return 1; }
+	else
+		[ -n "$kernel" -a -n "$initrd" ] || {
+			error "missing kernel or initrd in tarball. set \$EPH_UPDATE_CMD";
+			return 1;
+		}
+	fi
+
+	return 0
+}
+
+write_tgt_conf() {
+	local file="$1" target_name="$2" image="$3"
+	shift 2;
+	local release=$1 stream=$2 label=$3 serial=$4 arch=$5 url=$6 name=$7
+	cat > "$file" <<EOF
+<target ${target_name}>
+    readonly 1
+    backing-store "$image"
+</target>
+EOF
+}
+
+cobbler_has() {
+	local noun="$1" name="$2" out=""
+
+	out=$(cobbler "$noun" find "--name=$name" 2>/dev/null) &&
+		[ "$out" = "$name" ]
+}
+
+cobbler_add_update() {
+	# cobbler_add_update(distro_name, profile_name, 
+	#					 release, arch, kopts, kickstart,
+	#                    kernel, initrd)
+	local distro="$1" profile="$2" release="$3" arch="$4"
+	local kernel="$5" initrd="$6" kopts="$7" kickstart="$8" 
+	local op
+	
+	cobbler_has distro "$distro" && op="edit" || op="add"
+	
+	cobbler distro "$op" "--name=$distro" --breed=ubuntu \
+		"--os-version=$release" "--arch=$arch" \
+		"--kernel=$kernel" "--initrd=$initrd" ||
+		{ error "failed to $op $distro"; return 1; }
+
+	cobbler_has profile "$profile" && op="edit" || op="add"
+
+	cobbler profile "$op" "--name=$profile" "--distro=$distro" \
+		--kopts="$kopts" "--kickstart=$kickstart" ||
+		{ error "failed to $op $profile"; return 1; }
+
+	return 0
+}
+
+replace() {
+	# replace(input, key1, value1, key2, value2, ...)
+	local input="$1" key="" val=""
+	shift
+	while [ $# -ne 0 ]; do
+		input=${input//$1/$2}
+		shift 2
+	done
+	_RET=${input}
+}
+
+short_opts="hciuv"
+long_opts="help,import,update,update-check,verbose"
+getopt_out=$(getopt --name "${0##*/}" \
+	--options "${short_opts}" --long "${long_opts}" -- "$@") &&
+	eval set -- "${getopt_out}" ||
+	bad_Usage
+
+check=0
+import=0
+update=0
+
+while [ $# -ne 0 ]; do
+	cur=${1}; next=${2};
+	case "$cur" in
+		-h|--help) Usage ; exit 0;;
+		-v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
+		-i|--import) import=1;;
+		-c|--update-check) check=1;;
+		-u|--update) update=1;;
+		--) shift; break;;
+	esac
+	shift;
+done
+
+[ $import -eq 0 -a $check -eq 0 -a $update -eq 0 ] && import=1
+[ $(($import + $check + $update)) -eq 0 ] && import=1
+
+[ $(($import + $check + $update)) -eq 1 ] ||
+	bad_Usage "only one of --update-check, --update, --import may be given"
+
+[ ! -f "$CONFIG" ] || . "$CONFIG"
+[ ! -f ".${CONFIG}" ] || . ".${CONFIG}"
+
+# get default server ip
+[ -n "$SERVER_IP" ] ||
+	_ip=$(awk '$1 == "server:" { print $2 }' /etc/cobbler/settings) ||
+	fail "must set SERVER_IP to cobbler server"
+
+SERVER_IP=${SERVER_IP:-${_ip}}
+[ -n "${SERVER_IP}" ] &&
+	KOPTS="$KOPTS log_host=$SERVER_IP log_port=514"
+
+ISCSI_TARGET_IP=${ISCSI_TARGET_IP:-${SERVER_IP}}
+[ -n "$ISCSI_TARGET_IP" ] || fail "ISCSI_TARGET_IP must have a value"
+
+[ -f "$KICKSTART" ] ||
+	fail "kickstart $KICKSTART is not a file"
+
+mkdir -p "$DATA_DIR" "$DATA_DIR/.working" ||
+	fail "failed to make $DATA_DIR"
+
+TEMP_D=$(mktemp -d "$DATA_DIR/.working/${0##*/}.XXXXXX") ||
+   fail "failed to make tempdir"
+trap cleanup EXIT
+
+tgt_conf_d="$DATA_DIR/tgt.conf.d"
+tgt_conf="${DATA_DIR}/tgt.conf"
+
+mkdir -p "$tgt_conf_d" ||
+	fail "failed to make directories"
+if [ ! -f "${tgt_conf}" ]; then
+	cat > "${tgt_conf}" <<EOF
+include ${DATA_DIR}/tgt.conf.d/*.conf
+default-driver iscsi
+EOF
+fi
+
+updates=0
+for release in $RELEASES; do
+	for arch in $ARCHES; do
+		arch2cob "$arch"; arch_c=$_RET
+		arch2u "$arch"; arch_u=$_RET
+
+		query_local "$arch_u" "$release" "$STREAM" ||
+			fail "failed to query local for $release/$arch"
+		query_remote "$arch_u" "$release" "$STREAM" ||
+			fail "remote query of $REMOTE_IMAGES_MIRROR failed"
+
+		if [ $update -eq 0 -o -z "$l_dir" ]; then
+			serial_gt "$r_serial" "$l_serial" || {
+				debug 1 "$release-${arch_u} in ${l_dir} is up to date";
+				continue;
+			}
+
+			# an update is needed remote serial is newer than local
+			updates=$(($updates+1))
+
+			# check only
+			[ $check -eq 0 ] || continue
+
+			debug 1 "updating $release-$arch ($l_name => $r_name)"
+			wd="${TEMP_D}/$release/$arch"
+			prep_dir "$wd" \
+				"$r_release" "$r_stream" "$r_label" \
+				"$r_serial" "$r_arch" "$r_url" "$r_name" ||
+				fail "failed to prepare image for $release/$arch"
+
+			target_name="${TARGET_NAME_PREFIX}${r_name}"
+
+			final_d="${r_release}/${r_stream}/${r_arch}/${r_serial}"
+			fpfinal_d="${DATA_DIR}/${final_d}"
+			mkdir -p "${fpfinal_d}"
+
+			mv "$wd/"* "${fpfinal_d}/" ||
+				fail "failed to move contents to final directory ${fpfinal_d}"
+			name="${r_name}"
+		else
+			fpfinal_d="${l_dir}"
+			final_d="${l_release}/${l_stream}/${l_arch}/${l_serial}"
+
+			name="${l_name}"
+			target_name="${TARGET_NAME_PREFIX}${name}"
+			debug 1 "updating ${release}-${arch} $final_d"
+		fi
+
+		rel_tgt="../${final_d}/tgt.conf"
+
+		# iscsi_update
+		write_tgt_conf "${fpfinal_d}/tgt.conf" "$target_name" \
+			"${fpfinal_d}/disk.img" ||
+			fail "failed to write tgt.conf for $release/$arch"
+
+		ln -sf "$rel_tgt" "${tgt_conf_d}/${name}.conf" ||
+			fail "failed to symlink ${name}.conf into place"
+
+		tgt-admin --conf "$SYS_TGT_CONF" --update "${target_name}" || {
+			mv "${fpfinal_d}/info" "${fpfinal_d}/info.failed"
+			tgt-admin --conf "$SYS_TGT_CONF" --delete "$target_name"
+			rm "${tgt_conf_d}/${name}.conf"
+			fail "failed tgt-admin add for $name"
+		}
+
+		# cobbler_update
+		kopts_in="$EPH_KOPTS $EPH_KOPTS_ISCSI $EPH_KOPTS_ROOT $EPH_KOPTS_LOGGING"
+		replace "${kopts_in}" \
+			"@@server_ip@@" "$SERVER_IP" \
+			"@@iscsi_target@@" "${target_name}" \
+			"@@iscsi_target_ip@@" "${ISCSI_TARGET_IP}"
+		kopts=$_RET
+		echo kopts=${kopts}
+		continue
+
+		distro="$release-${arch_c}-maas-ephemeral"
+		profile="maas-${release}-${arch_c}-commissioning"
+		kernel="$fpfinal_d/kernel"
+		initrd="$fpfinal_d/initrd"
+		debug 1 "updating profile $profile, distro $distro kopts:${kopts}"
+		debug 2 cobbler_add_update "$distro" "$profile" "$release" "${arch_c}" \
+			"$kernel" "$initrd" "$kopts" "$KICKSTART"
+		cobbler_add_update "$distro" "$profile" "$release" "${arch_c}" \
+			"$kernel" "$initrd" "$kopts" "$KICKSTART" || {
+				mv "${fpfinal_d}/info" "${fpfinal_d}/info.failed"
+				tgt-admin --conf "$SYS_TGT_CONF" --delete "$target_name"
+				rm "${tgt_conf_d}/${name}.conf";
+				fail "failed to update cobbler for $profile/$distro"
+			}
+	done
+done
+
+if [ $check -eq 1 ]; then
+	# if --update-check, but no updates needed, exit 3
+	[ $updates_needed -eq 0 ] && exit 3
+	# if updates are needed, exit 0
+	exit 0
+fi
+
+## cleanup
+# here, go through anything non-current,
+#   * remove the tgt config
+#   * if tgt-show has entry:
+#     * remove from tgt-admin by name && remove directories
+#   * else
+#     * remove directory
+
+# vi: ts=4 noexpandtab