launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #08058
[Merge] lp:~allenap/maas/database-run into lp:maas
Gavin Panella has proposed merging lp:~allenap/maas/database-run into lp:maas with lp:~allenap/maas/convert-enums-redux as a prerequisite.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~allenap/maas/database-run/+merge/104759
This moves the management of the database cluster into a tested Python module. I attempted to use van.pg, but it wasn't quite suitable; the exposed API was too restrictive.
The database (including cluster) are also now managed as a service, but to support testing and other uses, it also allows direct use of the new bin/database command to launch a shell, get status, stop, or destroy the cluster. Each user of the cluster also takes out a lock on the cluster, so that requests to stop it or shut it down while in use are not followed.
--
https://code.launchpad.net/~allenap/maas/database-run/+merge/104759
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~allenap/maas/database-run into lp:maas.
=== modified file 'Makefile'
--- Makefile 2012-05-18 13:11:58 +0000
+++ Makefile 2012-05-21 14:04:25 +0000
@@ -7,6 +7,7 @@
build: \
bin/buildout \
+ bin/database \
bin/maas bin/test.maas \
bin/twistd.pserv bin/test.pserv \
bin/twistd.txlongpoll \
@@ -19,6 +20,10 @@
$(python) bootstrap.py --distribute --setup-source distribute_setup.py
@touch --no-create $@ # Ensure it's newer than its dependencies.
+bin/database: bin/buildout buildout.cfg versions.cfg setup.py
+ bin/buildout install database
+ @touch --no-create $@
+
bin/maas: bin/buildout buildout.cfg versions.cfg setup.py $(js_enums)
bin/buildout install maas
@touch --no-create $@
@@ -51,9 +56,6 @@
bin/buildout install repl
@touch --no-create bin/py bin/ipy
-dev-db:
- utilities/maasdb start ./db/ disposable
-
test: bin/test.maas bin/test.pserv $(js_enums)
bin/test.maas
bin/test.pserv
@@ -99,7 +101,6 @@
$(RM) celerybeat-schedule
distclean: clean stop
- utilities/maasdb delete-cluster ./db/
$(RM) -r eggs develop-eggs
$(RM) -r bin build dist logs/* parts
$(RM) tags TAGS .installed.cfg
@@ -109,10 +110,13 @@
$(RM) -r run/* services/*/supervise
$(RM) twisted/plugins/dropin.cache
-harness: bin/maas dev-db
+harness: bin/maas services/database/@start
bin/maas shell --settings=maas.demo
-syncdb: bin/maas dev-db
+dbharness: bin/database
+ bin/database shell --preserve
+
+syncdb: bin/maas services/database/@start
bin/maas syncdb --noinput
bin/maas migrate maasserver --noinput
bin/maas migrate metadataserver --noinput
@@ -121,7 +125,7 @@
build
check
clean
- dev-db
+ dbharness
distclean
doc
enums
@@ -138,7 +142,7 @@
# Development services.
#
-service_names := pserv celeryd reloader txlongpoll web webapp
+service_names := celeryd database pserv reloader txlongpoll web webapp
services := $(patsubst %,services/%/,$(service_names))
run:
@@ -211,6 +215,10 @@
# Dependencies for individual services.
+services/celeryd/@deps:
+
+services/database/@deps: bin/database
+
services/pserv/@deps: bin/twistd.pserv
services/reloader/@deps:
@@ -219,9 +227,7 @@
services/web/@deps:
-services/webapp/@deps: bin/maas dev-db
-
-services/celeryd/@deps:
+services/webapp/@deps: bin/maas
#
# Phony stuff.
=== modified file 'buildout.cfg'
--- buildout.cfg 2012-05-15 13:32:50 +0000
+++ buildout.cfg 2012-05-21 14:04:25 +0000
@@ -47,7 +47,6 @@
# Convenient developer dependencies
Jinja2
Pygments
- pyinotify
Sphinx
docutils
lxml
@@ -59,35 +58,44 @@
extra-paths =
${buildout:directory}/src
${buildout:directory}
-dev-eggs =
- pyinotify
test-eggs =
coverage
fixtures
lxml
nose
nose-subunit
+ postgresfixture
python-subunit
sst
testresources
testscenarios
testtools
+[database]
+recipe = z3c.recipe.scripts
+eggs = postgresfixture
+extra-paths = ${common:extra-paths}
+interpreter =
+entry-points = database=postgresfixture.main:main
+
[maas]
recipe = zc.recipe.egg
# avahi and dbus should be listed as eggs
# but they don't have links on PyPI and that makes buildout really
# unhappy. It refuses to see them, even if they are in site-packages :-(
# We rely on them being installed through system packages instead.
-eggs =
- ${common:dev-eggs}
- ${common:test-eggs}
- celery
- convoy
- django
- django-celery
+dev-eggs =
django-debug-toolbar
+test-eggs =
+ ${common:test-eggs}
django-nose
+eggs =
+ ${maas:dev-eggs}
+ ${maas:test-eggs}
+ celery
+ convoy
+ django
+ django-celery
django-piston
docutils
oauth
@@ -124,7 +132,6 @@
[pserv]
recipe = zc.recipe.egg
eggs =
- ${common:dev-eggs}
formencode
oops-datedir-repo
oops-twisted
=== modified file 'qa/scripts/prepare-for-juju-with-vdenv'
--- qa/scripts/prepare-for-juju-with-vdenv 2012-03-22 12:31:16 +0000
+++ qa/scripts/prepare-for-juju-with-vdenv 2012-05-21 14:04:25 +0000
@@ -37,7 +37,7 @@
--username "${LOGNAME}" --password test \
--email "${LOGNAME}@example.com"
bin/maas reconcile
-utilities/maasdb shell db <<'EOF'
+make dbharness <<'EOF'
UPDATE maasserver_node
SET owner_id = NULL, status = 4
WHERE hostname LIKE 'odev-node%';
=== added directory 'services/database'
=== added file 'services/database/down'
=== added file 'services/database/run'
--- services/database/run 1970-01-01 00:00:00 +0000
+++ services/database/run 2012-05-21 14:04:25 +0000
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+# Exit immediately if a command exits with a non-zero status.
+set -o errexit
+# Treat unset variables as an error when substituting.
+set -o nounset
+
+# Move to the project root.
+cd "$(dirname "$0")/../.."
+
+# Start logging, if requested. Not using multilog here right now
+# because there are race issues when restarting.
+[ -z "${logdir:-}" ] || exec &>> "${logdir}/current"
+
+# Start the database.
+script="$(readlink -f bin/database)"
+exec "${script}" run --preserve
=== modified file 'src/maas/demo.py'
--- src/maas/demo.py 2012-04-18 10:51:03 +0000
+++ src/maas/demo.py 2012-05-21 14:04:25 +0000
@@ -11,7 +11,7 @@
__metaclass__ = type
-import os
+from os.path import abspath
from maas import (
development,
@@ -27,7 +27,7 @@
import_settings(settings)
import_settings(development)
-MEDIA_ROOT = os.path.join(os.getcwd(), "media/demo")
+MEDIA_ROOT = abspath("media/demo")
MIDDLEWARE_CLASSES += (
'debug_toolbar.middleware.DebugToolbarMiddleware',
@@ -39,7 +39,7 @@
# For demo purposes, use a real provisioning server.
USE_REAL_PSERV = True
-MAAS_CLI = os.path.join(os.getcwd(), 'bin', 'maas')
+MAAS_CLI = abspath("bin/maas")
RABBITMQ_PUBLISH = True
=== modified file 'src/maas/development.py'
--- src/maas/development.py 2012-04-16 10:00:51 +0000
+++ src/maas/development.py 2012-05-21 14:04:25 +0000
@@ -12,7 +12,7 @@
__metaclass__ = type
import logging
-import os
+from os.path import abspath
from maas import (
import_local_settings,
@@ -58,13 +58,13 @@
'NAME': 'maas',
# For PostgreSQL, a "hostname" starting with a slash indicates a
# Unix socket directory.
- 'HOST': '%s/db' % os.getcwd(),
+ 'HOST': abspath('db'),
}
}
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
-MEDIA_ROOT = os.path.join(os.getcwd(), "media/development")
+MEDIA_ROOT = abspath("media/development")
INSTALLED_APPS += (
'django.contrib.admin',
=== added file 'src/maasserver/management/commands/dbshell.py'
--- src/maasserver/management/commands/dbshell.py 1970-01-01 00:00:00 +0000
+++ src/maasserver/management/commands/dbshell.py 2012-05-21 14:04:25 +0000
@@ -0,0 +1,37 @@
+# Copyright 2012 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Django command: start a database shell.
+
+Overrides the default implementation.
+"""
+
+from __future__ import (
+ absolute_import,
+ print_function,
+ unicode_literals,
+ )
+
+__metaclass__ = type
+__all__ = ['Command']
+
+from django.core.management.commands import dbshell
+from django.db import (
+ connections,
+ DEFAULT_DB_ALIAS,
+ )
+from postgresfixture import ClusterFixture
+
+
+class Command(dbshell.Command):
+ """Customized "dbshell" command."""
+
+ def handle(self, **options):
+ # Don't call up to Django's dbshell, because that ends up exec'ing the
+ # shell, preventing this from clearing down the fixture.
+ connection = connections[options.get('database', DEFAULT_DB_ALIAS)]
+ datadir = connection.settings_dict["HOST"]
+ with ClusterFixture(datadir, preserve=True) as cluster:
+ dbname = connection.settings_dict["NAME"]
+ cluster.createdb(dbname)
+ cluster.shell(dbname)
=== removed file 'src/maasserver/management/commands/deletedb.py'
--- src/maasserver/management/commands/deletedb.py 2012-04-16 10:00:51 +0000
+++ src/maasserver/management/commands/deletedb.py 1970-01-01 00:00:00 +0000
@@ -1,31 +0,0 @@
-# Copyright 2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Django command: stop and delete the local database cluster."""
-
-from __future__ import (
- absolute_import,
- print_function,
- unicode_literals,
- )
-
-__metaclass__ = type
-__all__ = ['Command']
-
-from subprocess import check_call
-
-from django.core.management.base import (
- BaseCommand,
- CommandError,
- )
-
-
-class Command(BaseCommand):
- """Stop and delete the local development database cluster."""
-
- help = "Delete the development database cluster."
-
- def handle(self, *args, **kwargs):
- if len(args) != 0:
- raise CommandError("Too many arguments.")
- check_call(['utilities/maasdb', 'delete-cluster', 'db'])
=== removed file 'src/maasserver/management/commands/query.py'
--- src/maasserver/management/commands/query.py 2012-04-16 10:00:51 +0000
+++ src/maasserver/management/commands/query.py 1970-01-01 00:00:00 +0000
@@ -1,41 +0,0 @@
-# Copyright 2012 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Django command: access the development database directly in SQL."""
-
-from __future__ import (
- absolute_import,
- print_function,
- unicode_literals,
- )
-
-__metaclass__ = type
-__all__ = ['Command']
-
-from subprocess import check_call
-
-from django.core.management.base import (
- BaseCommand,
- CommandError,
- )
-
-
-class Command(BaseCommand):
- """Custom django command: access the local development database directly.
-
- Executes an SQL statement given on the command line, or opens an SQL
- shell if no statement was given.
- """
-
- args = "[SQL statement]"
- help = "Access the database directly in SQL."
-
- def handle(self, *args, **kwargs):
- if len(args) > 1:
- raise CommandError("Too many arguments.")
- elif len(args) == 1:
- subcommand = 'query'
- else:
- subcommand = 'shell'
- check_call(
- ['utilities/maasdb', subcommand, 'db'] + list(args))
=== modified file 'src/maastesting/runner.py'
--- src/maastesting/runner.py 2012-04-16 10:00:51 +0000
+++ src/maastesting/runner.py 2012-05-21 14:04:25 +0000
@@ -14,9 +14,9 @@
"TestRunner",
]
-from subprocess import check_call
-
+from django.conf import settings
from django_nose import NoseTestSuiteRunner
+from postgresfixture import ClusterFixture
class TestRunner(NoseTestSuiteRunner):
@@ -24,5 +24,13 @@
def setup_databases(self, *args, **kwargs):
"""Fire up the db cluster, then punt to original implementation."""
- check_call(['utilities/maasdb', 'start', './db/', 'disposable'])
+ self.cluster = ClusterFixture("db", preserve=True)
+ self.cluster.setUp()
+ for database in settings.DATABASES.values():
+ if database["HOST"] == self.cluster.datadir:
+ self.cluster.createdb(database["NAME"])
return super(TestRunner, self).setup_databases(*args, **kwargs)
+
+ def teardown_databases(self, *args, **kwargs):
+ super(TestRunner, self).teardown_databases(*args, **kwargs)
+ self.cluster.cleanUp()
=== removed file 'utilities/maasdb'
--- utilities/maasdb 2012-03-15 13:58:32 +0000
+++ utilities/maasdb 1970-01-01 00:00:00 +0000
@@ -1,207 +0,0 @@
-#! /bin/bash -e
-#
-# MAAS database control script. See main() at the bottom for usage.
-#
-# Most functions take as their first argument a database cluster's data
-# directory. This is where the database's socket, pidfile, log, and data will
-# reside.
-#
-# Some design choices for this module:
-#
-# * Everything is PostgreSQL on Ubuntu.
-# * Each branch gets its own cluster. Kill & delete when done.
-# * Databases run under the system user that creates them. No root required.
-# * No global configuration apart from a basic PostgreSQL install.
-# * Connections use Unix sockets. No TCP port hogging.
-
-POSTGRES_VERSION=9.1
-PGCTL="/usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_ctl"
-
-
-# Figure out the full absolute data directory path for a given cluster, even
-# if a relative path is given.
-maasdb_locate() {
- local DATADIR
- DATADIR="$1"
-
- if test -z "$1"
- then
- echo "Specify a data directory for the MAAS database cluster." >&2
- return 1
- fi
- if ! echo "$DATADIR" | grep '^/'
- then
- echo "`pwd`/$DATADIR"
- fi
-}
-
-
-# Create a database cluster.
-maasdb_create_cluster() {
- local DATADIR
- DATADIR="`maasdb_locate "$1"`"
-
- if ! test -d "$DATADIR/base"
- then
- mkdir -p -- "$DATADIR"
- $PGCTL init -s -D "$DATADIR" -o '-E utf8 -A trust'
- fi
-}
-
-
-# Start a database cluster.
-maasdb_start_cluster() {
- local DATADIR DISPOSABLE EXTRA_POSTGRES_OPTS
- DATADIR="`maasdb_locate "$1"`"
- # Pass "disposable" as the second argument if the data in this database
- # is not important at all and you're willing to cut corners for speed.
- DISPOSABLE="$2"
-
- if test "$DISPOSABLE" = "disposable"
- then
- # -F -- don't bother fsync'ing.
- EXTRA_POSTGRES_OPTS="-F"
- else
- EXTRA_POSTGRES_OPTS=""
- fi
-
- maasdb_create_cluster "$DATADIR"
-
- if ! test -f "$DATADIR/postmaster.pid"
- then
- # pg_ctl options:
- # -D <dir> -- data directory.
- # -l <file> -- log file.
- # -s -- no informational messages.
- # -w -- wait until startup is complete.
- # postgres options:
- # -h <arg> -- host name; empty arg means Unix socket only.
- # -k -- socket directory.
- $PGCTL start \
- -D "$DATADIR" -l "$DATADIR/backend.log" -s -w \
- -o "-h '' -k '$DATADIR' $EXTRA_POSTGRES_OPTS"
- fi
-}
-
-
-# Stop a database cluster.
-maasdb_stop_cluster() {
- local DATADIR
- DATADIR="`maasdb_locate "$1"`"
-
- if test -f "$DATADIR/postmaster.pid"
- then
- $PGCTL stop -W -m fast -D "$DATADIR"
- fi
-}
-
-
-# Initialize a MAAS database.
-maasdb_init_db() {
- local DATADIR DISPOSABLE MARKER
- DATADIR="`maasdb_locate "$1"`"
- # Pass "disposable" as the second argument if the data in this database
- # is not important at all and you're willing to cut corners for speed.
- DISPOSABLE="$2"
-
- maasdb_start_cluster "$DATADIR" "$DISPOSABLE"
-
- MARKER="$DATADIR/maas-created"
- if ! test -f "$MARKER"
- then
- createdb -h "$DATADIR" maas && touch "$MARKER"
- fi
-}
-
-
-# Open a psql shell on a MAAS database.
-maasdb_shell() {
- local DATADIR
- DATADIR="`maasdb_locate "$1"`"
-
- maasdb_init_db "$DATADIR"
- psql -h "$DATADIR" maas
-}
-
-
-# Execute a query on a MAAS database.
-maasdb_query() {
- local DATADIR QUERY
- DATADIR="`maasdb_locate "$1"`"
- QUERY="$2"
-
- maasdb_init_db "$DATADIR"
- psql -h "$DATADIR" maas -c "$QUERY"
-}
-
-
-# Delete an entire MAAS database and cluster. Use only with extreme care!
-maasdb_delete_cluster() {
- local DATADIR
- DATADIR="`maasdb_locate "$1"`"
-
- # Before deleting anything, does this at least look like a MAAS database
- # cluster?
- if test -d "$DATADIR/base"
- then
- maasdb_stop_cluster "$DATADIR"
- # Removing the data directory may fail because of a race condition
- # where db/global is nonempty because of a statistics write. Rather
- # than block on shutdown, be optimistic and spin on retries.
- while ! rm -rf -- "$DATADIR"
- do
- # If this fails for a more persistent reason, too bad. Ctrl-C.
- echo "Retrying deletion of $DATADIR." >&2
- sleep 0.01
- done
- fi
-}
-
-
-usage() {
- cat <<EOF >&2
-Usage: maasdb <command> <cluster-path>
-
-Where <command> is one of:
- start
- stop
- query
- shell
- delete-cluster
-
-And <cluster-path> is the path to the MAAS development database cluster,
-typically "./db"
-EOF
-}
-
-
-unknown_command() {
- echo >&2 "** Unknown command: $1 **"
- echo
- usage
- exit 1
-}
-
-
-main() {
- local COMMAND DATADIR
- COMMAND="$1"
- DATADIR="$2"
- if ! shift 2
- then
- usage
- exit 1
- fi
-
- case "$COMMAND" in
- start) maasdb_init_db "$DATADIR" "$@" ;;
- stop) maasdb_stop_cluster "$DATADIR" "$@" ;;
- query) maasdb_query "$DATADIR" "$@" ;;
- shell) maasdb_shell "$DATADIR" "$@" ;;
- delete-cluster) maasdb_delete_cluster "$DATADIR" "$@" ;;
- *) unknown_command "$COMMAND" ;;
- esac
-}
-
-
-main "$@"
=== modified file 'versions.cfg'
--- versions.cfg 2012-05-15 13:32:50 +0000
+++ versions.cfg 2012-05-21 14:04:25 +0000
@@ -31,8 +31,8 @@
oops-datedir-repo = 0.0.17
oops-twisted = 0.0.6
oops-wsgi = 0.0.9
+postgresfixture = 0.1.1
pyasn1 = 0.0.11a
-pyinotify = 0.9.2
pymongo = 2.1.1
python-dateutil = 1.5
PyYAML = 3.10
Follow ups
