launchpad-reviewers team mailing list archive

[Colin Watson <mp+451925@xxxxxxxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/launchpad:stagingsetup-remote into launchpad:master.

Commit message:
Allow stagingsetup to work remotely

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/451925

I haven't actually been able to test this yet, so this is a little theoretical, but it should be a no-op for the existing setup (because `LPCONFIG=staging-db ../../utilities/pgoptions.py` returns empty).

William and I think that `walblock` will prove to be unnecessary since replication should be able to keep up on PS5, so for now let's try without it rather than trying to figure out how to port it.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:stagingsetup-remote into launchpad:master.

diff --git a/database/replication/Makefile b/database/replication/Makefile
index 7cbfa77..c8de967 100644
--- a/database/replication/Makefile
+++ b/database/replication/Makefile
@@ -34,6 +34,19 @@ STAGING_LOGDIR=/srv/staging.launchpad.net/staging-logs
 STAGING_POSTGRESQL_VERSION=10
 DOGFOOD_DUMP=launchpad.dump
 
+# Database options to pass to PostgreSQL tools.
+STAGING_DBOPTS:=$(shell LPCONFIG=${STAGING_CONFIG} ../../utilities/pgoptions.py)
+
+# If we're restoring locally, then block the restore process while there are
+# too many unshipped WAL files.  If we're restoring remotely, we can't
+# easily do this, so we have to just hope that the remote systems can keep
+# up.
+ifeq (,${STAGING_DBOPTS})
+STAGING_WALBLOCK=./walblock.py -n 5000 -d /var/lib/postgresql/${STAGING_POSTGRESQL_VERSION}/staging/pg_wal
+else
+STAGING_WALBLOCK=cat
+endif
+
 # Names of underlying PostgreSQL databases.
 STAGING_DBNAME_MAIN=lpmain_staging
 STAGING_DBNAME_SESSION=session_staging
@@ -74,12 +87,12 @@ stagingsetup:
 			${STAGING_PGBOUNCER} -c "$$verb $$db"; \
 		done; \
 	done
-	-${PGMASSACRE} ${STAGING_DBNAME_MAIN}
+	-LP_DESTROY_REMOTE_DATABASE=yes ${PGMASSACRE} ${STAGING_DBNAME_MAIN}
 
 	# Quickly clear out the session DB. No need to DISABLE here, as
 	# we bring the DB back quickly.
 	${STAGING_PGBOUNCER} -c 'KILL ${STAGING_PGBOUNCER_SESSION}'
-	psql -d ${STAGING_DBNAME_SESSION} -c 'TRUNCATE sessiondata CASCADE;'
+	psql ${STAGING_DBOPTS} -d ${STAGING_DBNAME_SESSION} -c 'TRUNCATE sessiondata CASCADE;'
 	${STAGING_PGBOUNCER} -c 'RESUME ${STAGING_PGBOUNCER_SESSION}'
 
 	# Create the DB with the desired default tablespace.
@@ -87,8 +100,7 @@ stagingsetup:
 	# Restore the database. We need to restore permissions, despite
 	# later running security.py, to pull in permissions granted on
 	# production to users not maintained by security.py.
-	cat ${STAGING_DUMP} \
-	    | ./walblock.py -n 5000 -d /var/lib/postgresql/${STAGING_POSTGRESQL_VERSION}/staging/pg_wal \
+	< ${STAGING_DUMP} ${STAGING_WALBLOCK}
 	    | pg_restore --dbname=${STAGING_DBNAME_MAIN} --no-owner ${EXIT_ON_ERROR} \
 		--use-list=${DUMPLIST} -v
 	rm ${DUMPLIST}
@@ -100,8 +112,8 @@ stagingsetup:
 	LPCONFIG=${STAGING_CONFIG} ${SHHH} ../schema/security.py \
 	    --log-file=INFO:${STAGING_LOGDIR}/dbupgrade.log
 	@echo Setting feature flags
-	psql -d ${STAGING_DBNAME_MAIN} -c "INSERT INTO featureflag (flag, scope, priority, value) VALUES ('profiling.enabled', 'team:launchpad', 0, 'on') ON CONFLICT DO NOTHING"
-	psql -d ${STAGING_DBNAME_MAIN} -c "INSERT INTO featureflag (flag, scope, priority, value) VALUES ('librarian.swift.enabled', 'default', 0, 'on') ON CONFLICT DO NOTHING"
+	psql ${STAGING_DBOPTS} -d ${STAGING_DBNAME_MAIN} -c "INSERT INTO featureflag (flag, scope, priority, value) VALUES ('profiling.enabled', 'team:launchpad', 0, 'on') ON CONFLICT DO NOTHING"
+	psql ${STAGING_DBOPTS} -d ${STAGING_DBNAME_MAIN} -c "INSERT INTO featureflag (flag, scope, priority, value) VALUES ('librarian.swift.enabled', 'default', 0, 'on') ON CONFLICT DO NOTHING"
 	set -e; for db in ${STAGING_PGBOUNCER_MAIN}; do \
 		${STAGING_PGBOUNCER} -c "ENABLE $$db"; \
 	done