launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #26242
[Merge] ~cjwatson/lp-codeimport:charm into lp-codeimport:master
Colin Watson has proposed merging ~cjwatson/lp-codeimport:charm into lp-codeimport:master with ~cjwatson/lp-codeimport:jenkaas-secrets as a prerequisite.
Commit message:
Add a Juju charm
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/lp-codeimport/+git/lp-codeimport/+merge/397657
This is based on the layers provided by lp:ols-charm-deps.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lp-codeimport:charm into lp-codeimport:master.
diff --git a/Makefile b/Makefile
index ae16dc5..aa90c00 100644
--- a/Makefile
+++ b/Makefile
@@ -178,6 +178,7 @@ build-tarball: build_wheels
--exclude .git \
--exclude .gitignore \
--exclude build \
+ --exclude charm \
--exclude env \
--exclude logs \
--exclude pip-cache \
diff --git a/charm/.gitignore b/charm/.gitignore
new file mode 100644
index 0000000..04ae82c
--- /dev/null
+++ b/charm/.gitignore
@@ -0,0 +1,3 @@
+bundle.yaml
+dist
+tmp
diff --git a/charm/Makefile b/charm/Makefile
new file mode 100644
index 0000000..eccdb3c
--- /dev/null
+++ b/charm/Makefile
@@ -0,0 +1,131 @@
+# The charm tool is shipped as a snap, so make sure it's on $PATH.
+export PATH := $(PATH):/snap/bin
+
+APP_NAME := lp-codeimport
+
+BUILDDIR := $(CURDIR)/dist
+TMPDIR := $(CURDIR)/tmp
+export CHARM_LAYERS_DIR := $(TMPDIR)/deps/ols-layers/layer
+export CHARM_INTERFACES_DIR := $(TMPDIR)/deps/ols-layers/interface
+CHARM_WHEELS_DIR := $(TMPDIR)/deps/charm-wheels
+
+BUILD_LABEL = $(shell git rev-parse HEAD)
+TARBALL = $(APP_NAME).tar.gz
+ASSET = ../build/$(BUILD_LABEL)/$(TARBALL)
+
+CHARMS := lp-codeimport
+
+PUBLISH_REPO_PREFIX := lp:~launchpad/lp-codeimport/+git/charm-build-
+PUBLISHDIR := $(BUILDDIR)/publish
+# We may need to force username and email when publishing, because git may
+# not be able to autodetect this in automatic build environments.
+DOMAIN ?= $(shell hostname -f)
+GIT_USERNAME = $(shell git config --get user.name || echo $(USER))
+GIT_EMAIL = $(shell git config --get user.email || echo $(USER)@$(DOMAIN))
+
+all: build lint
+
+$(BUILDDIR) $(TMPDIR) $(PUBLISHDIR):
+ @mkdir -p $@
+
+CHARM_DEPS := $(CHARM_LAYERS_DIR)/.done $(CHARM_INTERFACES_DIR)/.done
+$(CHARM_DEPS): $(CURDIR)/dependencies.txt | $(TMPDIR)
+ @echo "Fetching dependencies..."
+ @mkdir -p $(TMPDIR)/deps
+ @cd $(TMPDIR)/deps && codetree $<
+ @touch $(CHARM_DEPS)
+
+build: $(foreach charm,$(CHARMS),build-$(charm))
+
+build-lp-codeimport: dist/.built-lp-codeimport
+
+dist/.built-%: $(CHARM_DEPS) | $(BUILDDIR)
+ @echo "Building $*..."
+ @cd $* && \
+ PIP_NO_INDEX=true PIP_FIND_LINKS=$(CHARM_WHEELS_DIR) \
+ charm build -o $(BUILDDIR)
+ @touch $@
+
+clean-%:
+ @echo "Cleaning $*..."
+ @rm -rf dist/.built-$* dist/$* $(PUBLISHDIR)/$*
+
+tmp/ssh-key: | $(TMPDIR)
+ ssh-keygen -t rsa -b 2048 -f $@ -N ''
+
+tmp/gpg-key.sec: | $(TMPDIR)
+ mkdir -p -m0700 $(TMPDIR)/gpg
+ cp test-gpg-parameters $(TMPDIR)/gpg/parameters
+ gpg --homedir $(TMPDIR)/gpg \
+ --batch --generate-key $(TMPDIR)/gpg/parameters
+ gpg --homedir $(TMPDIR)/gpg --export-secret-key --armor \
+ >tmp/gpg-key.sec
+ gpg --homedir $(TMPDIR)/gpg --export --armor >tmp/gpg-key.pub
+
+bundle.yaml: bundle.yaml.in tmp/ssh-key tmp/gpg-key.sec
+ sed \
+ -e 's/%BUILD_LABEL%/$(BUILD_LABEL)/g' \
+ -e "s/%PRIVATE_SSH_KEY%/$$(base64 -w 0 <tmp/ssh-key)/g" \
+ -e "s/%PUBLIC_SSH_KEY%/$$(base64 -w 0 <tmp/ssh-key.pub)/g" \
+ -e "s/%PRIVATE_GPG_KEY%/$$(base64 -w 0 <tmp/gpg-key.sec)/g" \
+ -e "s/%PUBLIC_GPG_KEY%/$$(base64 -w 0 <tmp/gpg-key.pub)/g" \
+ bundle.yaml.in >bundle.yaml
+
+deploy: build payload bundle.yaml
+ @echo "Deploying $(APP_NAME)..."
+ @juju deploy ./bundle.yaml
+
+payload: $(ASSET)
+$(ASSET):
+ @echo "Building asset for $(BUILD_LABEL)..."
+ @$(MAKE) -C .. build-tarball
+
+clean:
+ @find . -name \*.pyc -delete
+ @find . -depth -name '__pycache__' -exec rm -rf '{}' \;
+ @rm -f bundle.yaml
+ @rm -f layer/*/codetree-collect-info.yaml
+ @rm -rf $(BUILDDIR) $(TMPDIR)
+
+lint: build
+ @echo "Linting charms..."
+ @set -e; for charm in $(CHARMS); do \
+ charm proof dist/$$charm; \
+ done
+ @echo "Linting python sources..."
+ @flake8 layer $(CHARMS)
+
+publish: build lint | $(PUBLISHDIR)
+ @set -e; for charm in $(CHARMS); do \
+ if [ -d $(PUBLISHDIR)/$$charm ]; then \
+ git -C $(PUBLISHDIR)/$$charm pull; \
+ else \
+ git clone $(PUBLISH_REPO_PREFIX)$$charm \
+ $(PUBLISHDIR)/$$charm; \
+ fi; \
+ rsync -a -m --ignore-times --exclude .git --delete \
+ dist/$$charm/ $(PUBLISHDIR)/$$charm/; \
+ git -C $(PUBLISHDIR)/$$charm add .; \
+ if [ "$$(git -C $(PUBLISHDIR)/$$charm status --porcelain || \
+ echo status failed)" ]; then \
+ git -C $(PUBLISHDIR)/$$charm \
+ -c user.name="$(GIT_USERNAME)" \
+ -c user.email="$(GIT_EMAIL)" \
+ commit -a \
+ -m "Build of $$charm from $(BUILD_LABEL)"; \
+ git -C $(PUBLISHDIR)/$$charm tag build/$(BUILD_LABEL); \
+ fi; \
+ git -C $(PUBLISHDIR)/$$charm push --tags origin master; \
+ done
+
+# Prepare a Jenkins-as-a-service container for charm building.
+setup-jenkaas:
+ sudo systemctl stop snapd.socket
+ sudo systemctl stop snapd
+ echo SNAPPY_STORE_NO_CDN=1 | sudo tee -a /etc/environment >/dev/null
+ echo SNAPPY_TESTING=1 | sudo tee -a /etc/environment >/dev/null
+ sudo systemctl start snapd.socket
+ sudo snap install --classic charm
+
+.PHONY: $(foreach charm,$(CHARMS),build-$(charm))
+.PHONY: all build clean deploy lint payload publish setup-jenkaas
diff --git a/charm/README.md b/charm/README.md
new file mode 100644
index 0000000..56a23d0
--- /dev/null
+++ b/charm/README.md
@@ -0,0 +1,20 @@
+# Overview
+
+This charm provides the Launchpad code import worker.
+
+# Usage
+
+ $ juju add-model lp-codeimport
+ $ make deploy
+
+You'll also need corresponding test deployments of Launchpad and turnip. To
+set up the code import worker to be able to push to your local turnip
+deployment, you'll need to do something like this:
+
+ $ juju config -m lp-codeimport lp-codeimport \
+ git_certificate="$(juju config -m turnip haproxy ssl_cert)"
+
+# Contact Information
+
+Colin Watson <cjwatson@xxxxxxxxxxxxx>
+https://launchpad.net/~cjwatson
diff --git a/charm/bundle.yaml.in b/charm/bundle.yaml.in
new file mode 100644
index 0000000..6ebcbdd
--- /dev/null
+++ b/charm/bundle.yaml.in
@@ -0,0 +1,15 @@
+series: xenial
+description: "lp-codeimport development bundle"
+applications:
+ lp-codeimport:
+ charm: ./dist/lp-codeimport
+ num_units: 1
+ options:
+ build_label: "%BUILD_LABEL%"
+ private_ssh_key: "%PRIVATE_SSH_KEY%"
+ public_ssh_key: "%PUBLIC_SSH_KEY%"
+ bzr_identity: "VCS Imports <vcs-imports@xxxxxxxxxxxxxx>"
+ private_gpg_key: "%PRIVATE_GPG_KEY%"
+ public_gpg_key: "%PUBLIC_GPG_KEY%"
+ resources:
+ lp-codeimport: "../build/%BUILD_LABEL%/lp-codeimport.tar.gz"
diff --git a/charm/dependencies.txt b/charm/dependencies.txt
new file mode 100644
index 0000000..983566b
--- /dev/null
+++ b/charm/dependencies.txt
@@ -0,0 +1,2 @@
+ols-layers git+ssh://git.launchpad.net/~ubuntuone-pqm-team/ols-charm-deps/+git/ols-layers;revno=4b403705
+charm-wheels git+ssh://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels;revno=c38224af
diff --git a/charm/lp-codeimport/config.yaml b/charm/lp-codeimport/config.yaml
new file mode 100644
index 0000000..4a1489a
--- /dev/null
+++ b/charm/lp-codeimport/config.yaml
@@ -0,0 +1,80 @@
+options:
+ git_hostname:
+ type: string
+ default: git.launchpad.test
+ description: The hostname of the Launchpad Git server.
+ git_certificate:
+ type: string
+ default: ""
+ description: >
+ Base64-encoded TLS certificate for the Launchpad Git server. The
+ default is to use the global CA infrastructure.
+ bazaar_branch_store:
+ type: string
+ default: sftp://hoover@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/srv/importd/www/
+ description: Where the Bazaar imports are stored.
+ foreign_tree_store:
+ type: string
+ default: sftp://hoover@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/srv/importd/sources/
+ description: >
+ Where the tarballs of foreign branches are uploaded for storage.
+ private_ssh_key:
+ type: string
+ default: ""
+ description: >
+ Base64-encoded private SSH key, used to communicate with the Launchpad
+ Bazaar server.
+ public_ssh_key:
+ type: string
+ default: ""
+ description: >
+ Base64-encoded public SSH key, used to communicate with the Launchpad
+ Bazaar server.
+ bzr_identity:
+ type: string
+ default: ""
+ description: Email identity used when making Bazaar commits.
+ private_gpg_key:
+ type: string
+ default: ""
+ description: >
+ Base64-encoded private GPG key, used when signing Bazaar commits.
+ public_gpg_key:
+ type: string
+ default: ""
+ description: >
+ Base64-encoded public GPG key, used when signing Bazaar commits.
+ scheduler_endpoint:
+ type: string
+ default: http://xmlrpc-private.launchpad.test:8087/codeimportscheduler
+ description: Where to find the code import scheduler service.
+ oops_prefix:
+ type: string
+ default: DEVEL
+ description: A prefix for OOPS codes for this instance.
+ rabbitmq_host:
+ type: string
+ default: ""
+ description: The host:port at which RabbitMQ is listening.
+ rabbitmq_user:
+ type: string
+ default: ""
+ description: The RabbitMQ user name.
+ rabbitmq_password:
+ type: string
+ default: ""
+ description: The RabbitMQ password.
+ rabbitmq_virtual_host:
+ type: string
+ default: ""
+ description: The RabbitMQ virtual host name.
+ error_email:
+ type: string
+ default: ""
+ description: An email address where errors are sent.
+ log_hosts_allow:
+ type: string
+ default: ""
+ description: >
+ Hosts that should be allowed to rsync logs. Note that this relies on
+ basenode.
diff --git a/charm/lp-codeimport/icon.svg b/charm/lp-codeimport/icon.svg
new file mode 100644
index 0000000..b7920cc
--- /dev/null
+++ b/charm/lp-codeimport/icon.svg
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+ xmlns:dc="http://purl.org/dc/elements/1.1/";
+ xmlns:cc="http://creativecommons.org/ns#";
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+ xmlns:svg="http://www.w3.org/2000/svg";
+ xmlns="http://www.w3.org/2000/svg";
+ xmlns:xlink="http://www.w3.org/1999/xlink";
+ xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd";
+ xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape";
+ width="100px"
+ height="100px"
+ viewBox="0 0 100 100"
+ version="1.1"
+ id="svg18"
+ sodipodi:docname="lp-charm-icon.svg"
+ inkscape:version="0.92.3 (2405546, 2018-03-11)">
+ <metadata
+ id="metadata22">
+ <rdf:RDF>
+ <cc:Work
+ rdf:about="">
+ <dc:format>image/svg+xml</dc:format>
+ <dc:type
+ rdf:resource="http://purl.org/dc/dcmitype/StillImage"; />
+ <dc:title>eclispe-che</dc:title>
+ </cc:Work>
+ </rdf:RDF>
+ </metadata>
+ <sodipodi:namedview
+ pagecolor="#ffffff"
+ bordercolor="#666666"
+ borderopacity="1"
+ objecttolerance="10"
+ gridtolerance="10"
+ guidetolerance="10"
+ inkscape:pageopacity="0"
+ inkscape:pageshadow="2"
+ inkscape:window-width="1920"
+ inkscape:window-height="1016"
+ id="namedview20"
+ showgrid="false"
+ inkscape:pagecheckerboard="true"
+ inkscape:zoom="1.668772"
+ inkscape:cx="29.156684"
+ inkscape:cy="41.621867"
+ inkscape:window-x="0"
+ inkscape:window-y="27"
+ inkscape:window-maximized="1"
+ inkscape:current-layer="layer2"
+ inkscape:snap-bbox="false"
+ inkscape:snap-bbox-midpoints="true"
+ inkscape:snap-object-midpoints="true" />
+ <!-- Generator: Sketch 45.2 (43514) - http://www.bohemiancoding.com/sketch -->
+ <title
+ id="title2">eclispe-che</title>
+ <desc
+ id="desc4">Created with Sketch.</desc>
+ <defs
+ id="defs7">
+ <path
+ d="M50.0004412,4.04252804e-14 C22.3871247,4.04252804e-14 0,22.3848726 0,49.9995588 C0,77.6133626 22.3871247,100 50.0004412,100 C77.6137577,100 100,77.6133626 100,49.9995588 C100,22.3848726 77.6128753,3.55271368e-14 50.0004412,4.04252804e-14 Z"
+ id="path-1" />
+ <style
+ id="style6"
+ type="text/css">
+
+ .fil0 {fill:#F8C300}
+ .fil6 {fill:#3895BD}
+ .fil3 {fill:#3941BF}
+ .fil1 {fill:#8FB635}
+ .fil4 {fill:#A02C35}
+ .fil7 {fill:#BB3A84}
+ .fil5 {fill:#D18C3B}
+ .fil8 {fill:#72706F;fill-rule:nonzero}
+ .fil9 {fill:#F8C300;fill-rule:nonzero}
+ .fil2 {fill:white;fill-rule:nonzero}
+
+ </style>
+ </defs>
+ <g
+ inkscape:groupmode="layer"
+ id="layer1"
+ inkscape:label="BACKGROUND">
+ <g
+ style="fill:#ffffff;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1;stroke-opacity:1"
+ id="Page-1">
+ <g
+ id="eclispe-che"
+ style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-opacity:1">
+ <g
+ id="path3023-path"
+ style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-opacity:1">
+ <use
+ xlink:href="#path-1"
+ id="use9"
+ style="fill:#ffffff;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-opacity:1"
+ x="0"
+ y="0"
+ width="100%"
+ height="100%" />
+ <path
+ d="M 50.000441,0.5 C 22.662621,0.5 0.5,22.661661 0.5,49.999559 0.5,77.337051 22.663098,99.5 50.000441,99.5 77.337613,99.5 99.5,77.337222 99.5,49.999559 99.5,22.661796 77.337514,0.5 50.000441,0.5 Z"
+ id="path11"
+ style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1;stroke-opacity:1"
+ inkscape:connector-curvature="0" />
+ </g>
+ </g>
+ </g>
+ </g>
+ <g
+ inkscape:groupmode="layer"
+ id="layer2"
+ inkscape:label="PLACE LOGO HERE">
+ <g
+ style="fill-rule:evenodd"
+ id="g3473"
+ transform="matrix(3.2292607,0,0,3.2292607,-486.24769,-600.53027)">
+ <g
+ id="g3489">
+ <path
+ inkscape:connector-curvature="0"
+ style="fill:#3895bd;fill-opacity:1;fill-rule:evenodd"
+ d="m 165.22326,188.53265 -9.92868,5.72521 4.76477,2.75658 5.15144,-2.96862 z"
+ id="path3475" />
+ <path
+ inkscape:connector-curvature="0"
+ style="fill:#8fb635;fill-opacity:1;fill-rule:evenodd"
+ d="m 166.90714,188.53265 v 5.51317 l 5.15144,2.96862 4.77724,-2.75658 z"
+ id="path3477" />
+ <path
+ inkscape:connector-curvature="0"
+ style="fill:#f8c300;fill-rule:evenodd"
+ d="m 166.05897,195.50518 -5.15144,2.96863 v 5.96219 l 5.15144,2.95616 5.15143,-2.95616 v -5.96219 z"
+ id="path3479" />
+ <path
+ inkscape:connector-curvature="0"
+ style="fill:#d18c3b;fill-opacity:1;fill-rule:evenodd"
+ d="m 177.67153,195.71723 -4.77724,2.76905 v 5.93725 l 4.77724,2.75658 z"
+ id="path3481" />
+ <path
+ inkscape:connector-curvature="0"
+ style="fill:#3941bf;fill-opacity:1;fill-rule:evenodd"
+ d="m 154.4464,195.7297 v 11.45041 l 4.77724,-2.75658 v -5.94972 z"
+ id="path3483" />
+ <path
+ inkscape:connector-curvature="0"
+ style="fill:#bb3a84;fill-opacity:1;fill-rule:evenodd"
+ d="m 160.05935,205.8829 -4.76477,2.75658 9.91621,5.73768 V 208.864 l -5.13897,-2.9811 z"
+ id="path3485" />
+ <path
+ inkscape:connector-curvature="0"
+ style="fill:#a02c35;fill-opacity:1;fill-rule:evenodd"
+ d="m 172.05858,205.89537 -5.15144,2.96863 v 5.51316 l 9.91621,-5.73768 z"
+ id="path3487" />
+ </g>
+ </g>
+ </g>
+</svg>
diff --git a/charm/lp-codeimport/layer.yaml b/charm/lp-codeimport/layer.yaml
new file mode 100644
index 0000000..c5cd846
--- /dev/null
+++ b/charm/lp-codeimport/layer.yaml
@@ -0,0 +1,25 @@
+includes:
+ - layer:ols
+options:
+ apt:
+ packages:
+ - bzr
+ - cvs
+ - git
+ - libffi-dev
+ - libssl-dev
+ - libsvn-dev
+ - python
+ - python-pkg-resources
+ - python-sqlite
+ - python-tdb
+ - subversion
+ - virtualenv
+ ols:
+ service_name: lp-codeimport
+ config_filename: service.conf
+ user: importd
+ tarball_payload: true
+ symlink_switch_payload: true
+ python_bin: python2.7
+repo: https://git.launchpad.net/lp-codeimport
diff --git a/charm/lp-codeimport/metadata.yaml b/charm/lp-codeimport/metadata.yaml
new file mode 100644
index 0000000..f509db3
--- /dev/null
+++ b/charm/lp-codeimport/metadata.yaml
@@ -0,0 +1,16 @@
+name: lp-codeimport
+display-name: lp-codeimport
+summary: Launchpad code import worker
+maintainer: Colin Watson <cjwatson@xxxxxxxxxxxxx>
+description: A worker for importing from other version control repositories.
+tags:
+ # https://juju.is/docs/charm-metadata#heading--charm-store-fields
+ - network
+series:
+ - xenial
+subordinate: false
+resources:
+ lp-codeimport:
+ type: file
+ filename: lp-codeimport.tar.gz
+ description: lp-codeimport code
diff --git a/charm/lp-codeimport/reactive/lp-codeimport.py b/charm/lp-codeimport/reactive/lp-codeimport.py
new file mode 100644
index 0000000..4e5d00f
--- /dev/null
+++ b/charm/lp-codeimport/reactive/lp-codeimport.py
@@ -0,0 +1,240 @@
+# Copyright 2018-2021 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import base64
+import os.path
+import shutil
+import subprocess
+
+from charmhelpers.core import (
+ hookenv,
+ host,
+ templating,
+ )
+from charms.reactive import (
+ remove_state,
+ set_state,
+ when,
+ when_not,
+ )
+from ols import base
+
+
+# Monkey-patch layer:ols.
+def create_virtualenv(wheels_dir, codedir, python_exe):
+ subprocess.run(
+ ['make', 'compile', 'PYTHON={}'.format(python_exe)],
+ cwd=codedir, check=True)
+
+
+base.create_virtualenv = create_virtualenv
+
+
+def data_dir():
+ return os.path.join(base.base_dir(), 'data')
+
+
+def scripts_dir():
+ return os.path.join(base.base_dir(), 'scripts')
+
+
+def var_dir():
+ return os.path.join(base.base_dir(), 'var')
+
+
+def oopses_dir():
+ return os.path.join(base.base_dir(), 'oopses')
+
+
+def home_dir():
+ return os.path.join('/home', base.user())
+
+
+def ensure_lp_directories():
+ for dirpath in (data_dir(), var_dir(), oopses_dir()):
+ host.mkdir(dirpath, group=base.user(), perms=0o775)
+ host.mkdir(home_dir(), owner=base.user(), group=base.user(), perms=0o755)
+
+
+def base64_decode(value):
+ return base64.b64decode(value.encode('ASCII'))
+
+
+def install_ca_certificates(config):
+ hookenv.log('Writing CA certificates.')
+ git_certificate_path = os.path.join(
+ '/usr/local/share/ca-certificates',
+ '{}.crt'.format(config['git_hostname']))
+ if config['git_certificate']:
+ host.write_file(
+ git_certificate_path, base64_decode(config['git_certificate']),
+ perms=0o644)
+ else:
+ if os.path.exists(git_certificate_path):
+ os.unlink(git_certificate_path)
+ subprocess.run(['update-ca-certificates'], check=True)
+
+
+def install_keys(config):
+ hookenv.log('Writing keys.')
+
+ ssh_dir = os.path.join(home_dir(), '.ssh')
+ ssh_private_path = os.path.join(ssh_dir, 'lp-codeimport')
+ ssh_public_path = os.path.join(ssh_dir, 'lp-codeimport.pub')
+ ssh_config_path = os.path.join(ssh_dir, 'config')
+ if config['private_ssh_key'] and config['public_ssh_key']:
+ if not os.path.exists(ssh_dir):
+ host.mkdir(
+ ssh_dir, owner=base.user(), group=base.user(), perms=0o700)
+ host.write_file(
+ ssh_private_path, base64_decode(config['private_ssh_key']),
+ owner=base.user(), group=base.user(), perms=0o600)
+ host.write_file(
+ ssh_public_path, base64_decode(config['public_ssh_key']),
+ owner=base.user(), group=base.user(), perms=0o644)
+ templating.render(
+ 'ssh_config.j2', ssh_config_path,
+ config, owner=base.user(), group=base.user(), perms=0o644)
+ else:
+ for path in (ssh_private_path, ssh_public_path, ssh_config_path):
+ if os.path.exists(path):
+ os.unlink(path)
+
+ if config['private_gpg_key'] and config['public_gpg_key']:
+ # Unfortunately we can't use check=True, since these will fail if
+ # the key has already been added.
+ subprocess.run(
+ ['sudo', '-H', '-u', base.user(), 'gpg', '--import'],
+ input=base64_decode(config['private_gpg_key']))
+ subprocess.run(
+ ['sudo', '-H', '-u', base.user(), 'gpg', '--import'],
+ input=base64_decode(config['public_gpg_key']))
+
+ bazaar_dir = os.path.join(home_dir(), '.bazaar')
+ if not os.path.exists(bazaar_dir):
+ host.mkdir(
+ bazaar_dir, owner=base.user(), group=base.user(), perms=0o755)
+ templating.render(
+ 'bazaar.conf.j2', os.path.join(bazaar_dir, 'bazaar.conf'),
+ config, owner=base.user(), group=base.user(), perms=0o644)
+
+
+def install_scripts(config):
+ hookenv.log('Writing scripts.')
+ src = os.path.join(hookenv.charm_dir(), 'scripts')
+ dst = scripts_dir()
+ if not os.path.exists(dst):
+ host.mkdir(dst, perms=0o755)
+ for name in ('ps_dump.sh', 'ps_dump_clean.sh'):
+ shutil.copy2(os.path.join(src, name), os.path.join(dst, name))
+ templating.render(
+ 'clean_importd_logs.sh.j2',
+ os.path.join(dst, 'clean_importd_logs.sh'),
+ config, perms=0o755)
+
+
+def configure_logrotate(config):
+ hookenv.log('Writing logrotate configuration.')
+ templating.render(
+ 'logrotate.conf.j2',
+ os.path.join('/etc/logrotate.d', hookenv.application_name()),
+ config, perms=0o644)
+
+
+def configure_rsync(config):
+ hookenv.log('Writing rsync configuration.')
+ rsync_path = '/etc/rsync-juju.d/010-lp-codeimport.conf'
+ if config['log_hosts_allow']:
+ templating.render(
+ 'lp-codeimport-rsync.j2', rsync_path, config, perms=0o644)
+ elif os.path.exists(rsync_path):
+ os.unlink(rsync_path)
+ if not host.service_restart('rsync'):
+ raise RuntimeError('Failed to restart rsync')
+
+
+@when('ols.configured')
+@when_not('service.configured')
+def configure():
+ ensure_lp_directories()
+
+ system_packages = os.path.join(base.code_dir(), 'system-packages.txt')
+ if os.path.exists(system_packages):
+ site_packages = subprocess.run(
+ [os.path.join(base.code_dir(), 'env', 'bin', 'python'), '-c',
+ 'from distutils.sysconfig import get_python_lib; '
+ 'print(get_python_lib())'],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True,
+ universal_newlines=True).stdout.rstrip('\n')
+ link_system_packages = os.path.join(
+ base.code_dir(), 'utilities', 'link-system-packages.py')
+ subprocess.run(
+ [link_system_packages, site_packages, system_packages], check=True)
+
+ config = hookenv.config()
+ config_path = os.path.join(
+ base.code_dir(), 'production-configs', 'charm', 'codeimport-lazr.conf')
+ svc_config = dict(config)
+ svc_config.update({
+ 'base_dir': base.base_dir(),
+ 'code_dir': base.code_dir(),
+ 'logs_dir': base.logs_dir(),
+ 'data_dir': data_dir(),
+ 'scripts_dir': scripts_dir(),
+ 'var_dir': var_dir(),
+ 'oopses_dir': oopses_dir(),
+ 'home_dir': home_dir(),
+ 'user': base.user(),
+ # Chosen to allow distributing dispatch start time over a 30-second
+ # interval.
+ 'dispatch_offset': host.modulo_distribution(modulo=6, wait=5),
+ })
+ if config['private_gpg_key']:
+ gpg_key_colons = subprocess.run(
+ ['gpg', '--quiet', '--with-colons'],
+ input=base64_decode(config['private_gpg_key']),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ check=True).stdout.decode('UTF-8')
+ svc_config['gpg_keyid'] = [
+ line for line in gpg_key_colons.splitlines()
+ if line.startswith('sec:')][0].split(':')[4]
+
+ hookenv.log('Writing service configuration.')
+ templating.render(
+ 'codeimport-lazr.conf.j2', config_path, svc_config,
+ owner='root', group=base.user(), perms=0o440)
+
+ install_ca_certificates(svc_config)
+ install_keys(svc_config)
+ install_scripts(svc_config)
+ configure_logrotate(svc_config)
+ configure_rsync(svc_config)
+
+ hookenv.log('Writing crontab.')
+ crontab_path = os.path.join(home_dir(), 'crontab')
+ templating.render(
+ 'crontab.j2', crontab_path,
+ svc_config, owner=base.user(), group=base.user(), perms=0o644)
+ subprocess.run(
+ ['sudo', '-H', '-u', 'importd', 'crontab', crontab_path], check=True)
+
+ set_state('service.configured')
+
+
+@when('service.configured')
+def check_is_running():
+ hookenv.status_set('active', 'Ready')
+
+
+@when('config.changed.build_label')
+def build_label_changed():
+ remove_state('ols.service.installed')
+ remove_state('ols.configured')
+ remove_state('service.configured')
+
+
+@when('config.changed')
+def config_changed():
+ remove_state('service.configured')
diff --git a/charm/lp-codeimport/scripts/ps_dump.sh b/charm/lp-codeimport/scripts/ps_dump.sh
new file mode 100755
index 0000000..895f34e
--- /dev/null
+++ b/charm/lp-codeimport/scripts/ps_dump.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+ echo "Usage: $0 <path to store results>"
+ exit 1
+fi
+DIR=$1
+if [ ! -d "$DIR" ]; then
+ if ! mkdir -p "$DIR"; then
+ echo "$0: Unable to create Stats directory: $DIR"
+ exit 1
+ fi
+fi
+
+# delay 0-30 seconds, random. avoid crontab sameness/issues
+DELAY=$(( $(dd if=/dev/urandom count=1 2> /dev/null | cksum | cut -f1 -d" ") % 31 ))
+sleep $DELAY
+
+OUT="$DIR/$(date +%F:%T).ps"
+ps axfww -o user,pid,ppid,ni,pri,cputime,pmem,rss,size,vsize,stat,blocked,nlwp,lstart,etime,cmd > "$OUT"
diff --git a/charm/lp-codeimport/scripts/ps_dump_clean.sh b/charm/lp-codeimport/scripts/ps_dump_clean.sh
new file mode 100755
index 0000000..38d64ab
--- /dev/null
+++ b/charm/lp-codeimport/scripts/ps_dump_clean.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+if [ -z "$1" ]; then
+ echo "Usage: $0 <path to store results>"
+ exit 1
+fi
+DIR=$1
+
+YESTERDAY=$(date +%F --date="yesterday")
+cd "$DIR"
+tar -cj -f "stats.$YESTERDAY.tar.bz2" "$YESTERDAY"*.ps
+rm "$YESTERDAY"*.ps
+find . -maxdepth 1 -type f -mtime +7 -name 'stats.*.tar.bz2' -o -name '*.ps' | xargs --no-run-if-empty rm
diff --git a/charm/lp-codeimport/templates/bazaar.conf.j2 b/charm/lp-codeimport/templates/bazaar.conf.j2
new file mode 100644
index 0000000..d7662fb
--- /dev/null
+++ b/charm/lp-codeimport/templates/bazaar.conf.j2
@@ -0,0 +1,7 @@
+email={{ bzr_identity }}
+check_signatures=require
+{%- if private_gpg_key %}
+create_signatures=always
+gpg_signing_key={{ gpg_keyid }}
+{%- endif %}
+
diff --git a/charm/lp-codeimport/templates/clean_importd_logs.sh.j2 b/charm/lp-codeimport/templates/clean_importd_logs.sh.j2
new file mode 100644
index 0000000..59a4708
--- /dev/null
+++ b/charm/lp-codeimport/templates/clean_importd_logs.sh.j2
@@ -0,0 +1,39 @@
+#! /bin/bash
+#
+# Clean up old importd scripts
+# Archive 1 day modified old+ to .../archive/YYYY-MM-DD/
+# and compress
+
+# Base initialisation
+IFS=$' \t\n'
+unset -f unalias
+\unalias -a
+unset -f command
+PATH="/bin:/usr/bin"
+
+##################
+
+DEST_ROOT='{{ logs_dir }}'
+DEST_ARCH="$DEST_ROOT/archives"
+
+
+# Check for Archive directory, barf loudly if not found!
+if [ ! -d "$DEST_ARCH" ]; then
+ echo "ERROR! Archive path does NOT exist. $DEST_ARCH"
+ exit 1
+fi
+
+DEST="$DEST_ARCH"/$(date +%F)
+
+if ! mkdir -p "$DEST"; then
+ echo "ERROR! cannot create archive directory! $DEST"
+ exit 1
+fi
+
+FINDARGS="-maxdepth 1 -mtime +1 -type f -name code-import-worker-*.log"
+
+find "$DEST_ROOT" $FINDARGS -print0 | xargs -r0 mv --target-directory="$DEST"
+find "$DEST" $FINDARGS -print0 | xargs -r0 bzip2
+
+find "$DEST_ARCH" -maxdepth 1 -type d -mtime +90 -print0 | xargs -r0 rm -r
+
diff --git a/charm/lp-codeimport/templates/codeimport-lazr.conf.j2 b/charm/lp-codeimport/templates/codeimport-lazr.conf.j2
new file mode 100644
index 0000000..47e9a3f
--- /dev/null
+++ b/charm/lp-codeimport/templates/codeimport-lazr.conf.j2
@@ -0,0 +1,38 @@
+[meta]
+extends: ../../lib/lp/services/config/schema-lazr.conf
+
+[codehosting]
+git_browse_root: https://{{ git_hostname }}/
+
+[codeimport]
+bazaar_branch_store: {{ bazaar_branch_store }}
+foreign_tree_store: {{ foreign_tree_store }}
+
+[codeimportdispatcher]
+codeimportscheduler_url: {{ scheduler_endpoint }}
+worker_log_dir: {{ logs_dir }}
+
+[codeimportworker]
+working_directory_root: {{ data_dir }}
+
+[error_reports]
+oops_prefix: {{ oops_prefix }}
+error_dir: {{ oopses_dir }}
+{%- if not rabbitmq_host %}
+error_exchange: none
+{%- endif %}
+
+[rabbitmq]
+{%- if rabbitmq_host %}
+host: {{ rabbitmq_host }}
+{%- endif %}
+{%- if rabbitmq_user %}
+userid: {{ rabbitmq_user }}
+{%- endif %}
+{%- if rabbitmq_password %}
+password: {{ rabbitmq_password }}
+{%- endif %}
+{%- if rabbitmq_virtual_host %}
+virtual_host: {{ rabbitmq_virtual_host }}
+{%- endif %}
+
diff --git a/charm/lp-codeimport/templates/crontab.j2 b/charm/lp-codeimport/templates/crontab.j2
new file mode 100644
index 0000000..c2e10ce
--- /dev/null
+++ b/charm/lp-codeimport/templates/crontab.j2
@@ -0,0 +1,23 @@
+{%- if error_email %}
+MAILTO={{ error_email }}
+{%- endif %}
+LPCONFIG=charm
+LP_PY={{ code_dir }}/bin/py
+
+0 0 * * 1,4 /usr/sbin/logrotate -s ~/.logrotate.state {{ etc_dir }}/logrotate.conf
+
+* * * * * sleep {{ dispatch_offset }} && [ -f {{ base_dir }}/maintenance.txt ] || {{ code_dir }}/cronscripts/code-import-dispatcher.py -v --max-jobs=10 --log-file {{ logs_dir }}/code-import-dispatcher.log >> {{ logs_dir }}/code-import-dispatcher-out.log 2>&1
+* * * * * sleep {{ dispatch_offset + 30 }} && [ -f {{ base_dir }}/maintenance.txt ] || {{ code_dir }}/cronscripts/code-import-dispatcher.py -v --max-jobs=10 --log-file {{ logs_dir }}/code-import-dispatcher.log >> {{ logs_dir }}/code-import-dispatcher-out.log 2>&1
+
+2 1 * * * {{ scripts_dir }}/clean_importd_logs.sh
+
+# Processes watcher and cleaner
+* * * * * {{ scripts_dir }}/ps_dump.sh {{ var_dir }}/ps_stats
+15 0 * * * {{ scripts_dir }}/ps_dump_clean.sh {{ var_dir }}/ps_stats
+
+# OOPS amqp
+*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_user }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_virtual_host }} --repo {{ oopses_dir }} --key ""
+
+# Work around https://bugs.launchpad.net/lp-codeimport/+bug/810288
+5 * * * * find /tmp -ignore_readdir_race -maxdepth 1 -type f \( -name 'tmp*.pack' -o -name 'tmp*.idx' \) -mtime +3 -delete
+
diff --git a/charm/lp-codeimport/templates/logrotate.conf.j2 b/charm/lp-codeimport/templates/logrotate.conf.j2
new file mode 100644
index 0000000..95e1819
--- /dev/null
+++ b/charm/lp-codeimport/templates/logrotate.conf.j2
@@ -0,0 +1,12 @@
+{{ logs_dir }}/code-import-dispatcher*.log {
+ rotate 5
+ daily
+ dateext
+ compress
+ delaycompress
+ compresscmd /bin/bzip2
+ uncompresscmd /bin/bunzip2
+ compressext .bz2
+ compressoptions -9
+}
+
diff --git a/charm/lp-codeimport/templates/lp-codeimport-rsync.j2 b/charm/lp-codeimport/templates/lp-codeimport-rsync.j2
new file mode 100644
index 0000000..0f1ee48
--- /dev/null
+++ b/charm/lp-codeimport/templates/lp-codeimport-rsync.j2
@@ -0,0 +1,8 @@
+
+[lp-codeimport-logs]
+ path = {{ logs_dir }}
+ comment = LP Code Import Logs
+ list = false
+ read only = true
+ hosts allow = {{ log_hosts_allow }}
+
diff --git a/charm/lp-codeimport/templates/ssh_config.j2 b/charm/lp-codeimport/templates/ssh_config.j2
new file mode 100644
index 0000000..926b3f5
--- /dev/null
+++ b/charm/lp-codeimport/templates/ssh_config.j2
@@ -0,0 +1,3 @@
+Host *
+ IdentityFile ~/.ssh/lp-codeimport
+
diff --git a/charm/packages.txt b/charm/packages.txt
new file mode 100644
index 0000000..c33fce5
--- /dev/null
+++ b/charm/packages.txt
@@ -0,0 +1,3 @@
+flake8
+python-codetree
+squashfuse
diff --git a/charm/test-gpg-parameters b/charm/test-gpg-parameters
new file mode 100644
index 0000000..93edead
--- /dev/null
+++ b/charm/test-gpg-parameters
@@ -0,0 +1,8 @@
+%no-protection
+Key-Type: RSA
+Key-Length: 2048
+Key-Usage: sign
+Name-Real: VCS Imports
+Name-Email: vcs-imports@xxxxxxxxxxxxxx
+Expire-Date: 0
+%commit
Follow ups
