launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27817
[Merge] ~jugmac00/launchpad:build-tarball into launchpad:master
Jürgen Gmach has proposed merging ~jugmac00/launchpad:build-tarball into launchpad:master.
Commit message:
Add link to all merge proposals
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1951128 in Launchpad itself: "no button / navigation to list of all merge proposals if there are no active open proposals"
https://bugs.launchpad.net/launchpad/+bug/1951128
For more details, see:
https://code.launchpad.net/~jugmac00/launchpad/+git/launchpad/+merge/412858
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/launchpad:build-tarball into launchpad:master.
diff --git a/.gitignore b/.gitignore
index 3b2e7d2..fe700ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
*.pyc
.tags
version-info.py
+version-info.txt
logs/*
+*
/botmaster
diff --git a/Makefile b/Makefile
index 90a640e..be3eb85 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,8 @@ PY=$(WD)/bin/py
PYTHONPATH:=$(WD)/lib:${PYTHONPATH}
VERBOSITY=-vv
+DEPENDENCY_REPO ?= https://git.launchpad.net/lp-source-dependencies
+
# virtualenv and pip fail if setlocale fails, so force a valid locale.
PIP_ENV := LC_ALL=C.UTF-8
# Run with "make PIP_NO_INDEX=" if you want pip to find software
@@ -95,6 +97,21 @@ PIP_BIN = \
bin/watch_jsbuild \
bin/with-xvfb
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+TARBALL_FILE_NAME = launchpad.tar.gz
+TARBALL_BUILDS_DIR ?= dist
+TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
+SWIFT_CONTAINER_NAME ?= launchpad-builds
+# This must match the object path used by fetch_payload in the ols charm
+# layer.
+SWIFT_OBJECT_PATH = \
+ launchpad-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
+
# DO NOT ALTER : this should just build by default
.PHONY: default
default: inplace
@@ -173,6 +190,17 @@ inplace: build logs clean_logs codehosting-dir
.PHONY: build
build: compile apidoc jsbuild css_combine
+# Bootstrap download-cache and sourcecode. Useful for CI jobs that want to
+# set these up from scratch.
+.PHONY: bootstrap
+bootstrap:
+ if [ -d download-cache/.git ]; then \
+ git -C download-cache pull; \
+ else \
+ git clone --depth=1 $(DEPENDENCY_REPO) download-cache; \
+ fi
+ utilities/update-sourcecode
+
# LP_SOURCEDEPS_PATH should point to the sourcecode directory, but we
# want the parent directory where the download-cache and env directories
# are. We re-use the variable that is using for the rocketfuel-get script.
@@ -258,29 +286,48 @@ requirements/combined.txt: \
--include requirements/launchpad.txt \
>"$@"
-# This target is used by LOSAs to prepare a build to be pushed out to
-# destination machines. We only want wheels: they are the expensive bits,
-# and the other bits might run into problems like bug 575037. This target
-# runs pip, builds a wheelhouse with predictable paths that can be used even
-# if the build is pushed to a different path on the destination machines,
-# and then removes everything created except for the wheels.
-#
# It doesn't seem to be straightforward to build a wheelhouse of all our
# dependencies without also building a useless wheel of Launchpad itself;
# fortunately that doesn't take too long, and we just remove it afterwards.
-.PHONY: build_wheels
-build_wheels: $(PIP_BIN) requirements/combined.txt
+.PHONY: build_wheels_only
+build_wheels_only: $(PIP_BIN) requirements/combined.txt
$(RM) -r wheelhouse wheels
+ $(SHHH) $(PIP) wheel -w wheels -r requirements/setup.txt
$(SHHH) $(PIP) wheel \
-c requirements/setup.txt -c requirements/combined.txt \
-w wheels .
$(RM) wheels/lp-[0-9]*.whl
- $(MAKE) clean_pip
+
+# This target is used by deployment machinery to prepare a build to be
+# pushed out to destination machines. We only want wheels: they are the
+# expensive bits, and the other bits might run into problems like bug
+# 575037. This target runs pip, builds a wheelhouse with predictable paths
+# that can be used even if the build is pushed to a different path on the
+# destination machines, and then removes everything created except for the
+# wheels.
+.PHONY: build_wheels
+build_wheels: build_wheels_only
+ $(MAKE) clean_js clean_pip
# Compatibility
.PHONY: build_eggs
build_eggs: build_wheels
+# Build a tarball that can be unpacked and built on another machine,
+# including all the wheels we need. This will eventually supersede
+# build_wheels.
+.PHONY: build-tarball
+build-tarball:
+ utilities/build-tarball $(TARBALL_BUILD_DIR)
+
+# Publish a buildable tarball to Swift.
+.PHONY: publish-tarball
+publish-tarball: build-tarball
+ [ ! -e ~/.config/swift/launchpad ] || . ~/.config/swift/launchpad; \
+ utilities/publish-to-swift --debug \
+ $(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+ $(TARBALL_BUILD_PATH)
+
# setuptools won't touch files that would have the same contents, but for
# Make's sake we need them to get fresh timestamps, so we touch them after
# building.
@@ -314,7 +361,7 @@ compile: $(VENV_PYTHON)
$(PYTHON) utilities/link-system-packages.py \
"$(SITE_PACKAGES)" system-packages.txt
${SHHH} bin/build-twisted-plugin-cache
- scripts/update-version-info.sh
+ [ ! -d .git ] || scripts/update-version-info.sh
.PHONY: test_build
test_build: build
diff --git a/lib/lp/code/browser/tests/test_gitlisting.py b/lib/lp/code/browser/tests/test_gitlisting.py
index 4bed9f8..e89417d 100644
--- a/lib/lp/code/browser/tests/test_gitlisting.py
+++ b/lib/lp/code/browser/tests/test_gitlisting.py
@@ -346,6 +346,22 @@ class TestProductGitListingView(TestTargetGitListingView,
view = create_initialized_view(self.target, '+git')
self.assertIsNotNone(find_tag_by_id(view(), 'active-review-count'))
+ def test_all_reviews_link(self):
+ main_repo = self.factory.makeGitRepository(
+ owner=self.owner, target=self.target, name="foo")
+ git_refs = self.factory.makeGitRefs(
+ main_repo,
+ paths=["refs/heads/master", "refs/heads/1.0", "refs/tags/1.1"])
+
+ with admin_logged_in():
+ self.setDefaultRepository(target=self.target, repository=main_repo)
+
+ self.factory.makeBranchMergeProposalForGit(
+ target_ref=git_refs[0],
+ set_state=BranchMergeProposalStatus.NEEDS_REVIEW)
+ view = create_initialized_view(self.target, '+git')
+ self.assertIsNotNone(find_tag_by_id(view(), 'all-reviews'))
+
def test_personal_git_instructions_not_present(self):
with person_logged_in(self.owner):
view = create_initialized_view(
diff --git a/lib/lp/code/templates/gitlisting.pt b/lib/lp/code/templates/gitlisting.pt
index 73109d8..77c7d40 100644
--- a/lib/lp/code/templates/gitlisting.pt
+++ b/lib/lp/code/templates/gitlisting.pt
@@ -74,6 +74,8 @@ git push --set-upstream origin master
<tal:repository-management
replace="structure repository/@@++repository-management" />
</div>
+ <p>
+ </p>
<p id="active-review-count"
tal:define="count context/menu:branches/active_review_count|nothing;
link context/menu:branches/active_reviews|nothing"
@@ -82,6 +84,9 @@ git push --set-upstream origin master
<tal:active-count replace="count"/>
<tal:link replace="structure python: link.render().lower()"/>.
</p>
+ <p id="all-reviews">
+ <a href="+merges">See all merge proposals</a>.
+ </p>
</div>
diff --git a/ols-vms.conf b/ols-vms.conf
new file mode 100644
index 0000000..cd26bce
--- /dev/null
+++ b/ols-vms.conf
@@ -0,0 +1,11 @@
+# Options defined here provide defaults for all sections
+vm.architecture = amd64
+vm.release = xenial
+
+apt.sources = ppa:launchpad/ppa
+vm.packages = launchpad-dependencies
+
+[launchpad]
+vm.class = lxd
+vm.update = True
+jenkaas.secrets = swift/launchpad:.config/swift/launchpad
diff --git a/scripts/update-version-info.sh b/scripts/update-version-info.sh
index f76b927..46c5d7c 100755
--- a/scripts/update-version-info.sh
+++ b/scripts/update-version-info.sh
@@ -47,3 +47,8 @@ else
mv ${newfile} version-info.py
fi
fi
+
+# talisker.config uses version-info.txt instead, so update that too. We
+# don't need to be particularly careful about file modification times here,
+# since there are no Makefile dependencies on this.
+echo "$revision_id" >version-info.txt
diff --git a/utilities/build-tarball b/utilities/build-tarball
new file mode 100755
index 0000000..70acfdb
--- /dev/null
+++ b/utilities/build-tarball
@@ -0,0 +1,39 @@
+#! /bin/sh
+#
+# Copyright 2021 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+set -e
+
+if [ -z "$1" ]; then
+ echo "Usage: $0 OUTPUT-DIRECTORY" >&2
+ exit 2
+fi
+output_dir="$1"
+
+# Build wheels to include in the distributed tarball.
+make build_wheels_only
+
+# Ensure that we have an updated idea of this tree's version.
+scripts/update-version-info.sh
+
+echo "Creating deployment tarball in $output_dir"
+
+# Prepare a file list.
+mkdir -p "$output_dir"
+(
+ git ls-files | sed 's,^,./,'
+ # Most of download-cache is unnecessary since we include a wheelhouse
+ # instead, but JavaScript-enabled builds need yarn.
+ find ./download-cache/dist/ -name yarn-\*.tar.gz
+ # The subdirectories of sourcecode may be symlinks. Force tar to
+ # include the actual files instead.
+ find ./sourcecode/*/ \
+ \( -name .bzr -o -name __pycache__ \) -prune -o -type f -print
+ echo ./version-info.py
+ find ./wheels/ -name \*.whl -print
+) | sort >"$output_dir/.files"
+
+# Create the tarball.
+tar -czf "$output_dir/launchpad.tar.gz" --no-recursion \
+ --files-from "$output_dir/.files"
diff --git a/utilities/publish-to-swift b/utilities/publish-to-swift
new file mode 100755
index 0000000..84de4c1
--- /dev/null
+++ b/utilities/publish-to-swift
@@ -0,0 +1,160 @@
+#! /usr/bin/python3 -S
+#
+# Copyright 2021 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Publish a built tarball to Swift for deployment."""
+
+import _pythonpath # noqa: F401
+
+from argparse import ArgumentParser
+import os
+
+import iso8601
+import requests
+from swiftclient.service import (
+ get_conn,
+ process_options,
+ SwiftService,
+ SwiftUploadObject,
+ )
+from swiftclient.shell import add_default_args
+
+
+def ensure_container_privs(options, container_name):
+ """Ensure that the container exists and is world-readable.
+
+ This allows us to give services suitable credentials for getting the
+ built code from a container.
+ """
+ options = dict(options)
+ options["read_acl"] = ".r:*"
+ with SwiftService(options=options) as swift:
+ swift.post(container=container_name)
+
+
+def get_swift_storage_url(options):
+ """Return the storage URL under which Swift objects are published."""
+ return get_conn(options).get_auth()[0]
+
+
+def publish_file_to_swift(options, container_name, object_path, local_path,
+ overwrite=True):
+ """Publish a file to a Swift container."""
+ storage_url = get_swift_storage_url(options)
+
+ with SwiftService(options=options) as swift:
+ stat_results = swift.stat(
+ container=container_name, objects=[object_path])
+ if stat_results and next(stat_results)["success"]:
+ print("Object {} already published to {}.".format(
+ object_path, container_name))
+ if not overwrite:
+ return
+
+ print("Publishing {} to {} as {}.".format(
+ local_path, container_name, object_path))
+ for r in swift.upload(
+ container_name,
+ [SwiftUploadObject(local_path, object_name=object_path)]):
+ if not r["success"]:
+ raise r["error"]
+
+ print("Published file: {}/{}/{}".format(
+ storage_url, container_name, object_path))
+
+
+def prune_old_files_from_swift(options, container_name, object_dir):
+ """Prune files from Swift that we no longer need."""
+ response = requests.head("https://launchpad.net/";)
+ response.raise_for_status()
+ production_revision = response.headers["X-VCS-Revision"]
+
+ with SwiftService(options=options) as swift:
+ objs = {}
+ production_mtime = None
+ for stats in swift.list(
+ container=container_name,
+ options={"prefix": "{}/".format(object_dir)}):
+ if not stats["success"]:
+ raise stats["error"]
+ for item in stats["listing"]:
+ if item.get("subdir") is None:
+ mtime = iso8601.parse_date(item["last_modified"])
+ objs[item["name"]] = mtime
+ if item["name"].startswith(
+ "{}/{}/".format(object_dir, production_revision)):
+ production_mtime = mtime
+
+ if production_mtime is None:
+ print(
+ "No file in {} corresponding to production revision {}; "
+ "not pruning.".format(container_name, production_revision))
+ return
+
+ for object_name, mtime in sorted(objs.items()):
+ if mtime < production_mtime:
+ print("Pruning {} (older than production)".format(object_name))
+ for r in swift.delete(
+ container=container_name, objects=[object_name]):
+ if not r["success"]:
+ raise r["error"]
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument("container_name")
+ parser.add_argument("swift_object_path")
+ parser.add_argument("local_path")
+ add_default_args(parser)
+ args = parser.parse_args()
+
+ if args.debug:
+ # Print OpenStack-related environment variables for ease of
+ # debugging. Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
+ # be secret, but for safety we only show unredacted contents of
+ # variables specifically known to be safe. See "swift --os-help"
+ # for most of these.
+ safe_keys = {
+ "OS_AUTH_URL",
+ "OS_AUTH_VERSION",
+ "OS_CACERT",
+ "OS_CERT",
+ "OS_ENDPOINT_TYPE",
+ "OS_IDENTITY_API_VERSION",
+ "OS_INTERFACE",
+ "OS_KEY",
+ "OS_PROJECT_DOMAIN_ID",
+ "OS_PROJECT_DOMAIN_NAME",
+ "OS_PROJECT_ID",
+ "OS_PROJECT_NAME",
+ "OS_REGION_NAME",
+ "OS_SERVICE_TYPE",
+ "OS_STORAGE_URL",
+ "OS_TENANT_ID",
+ "OS_TENANT_NAME",
+ "OS_USERNAME",
+ "OS_USER_DOMAIN_ID",
+ "OS_USER_DOMAIN_NAME",
+ "OS_USER_ID",
+ }
+ for key, value in sorted(os.environ.items()):
+ if key.startswith("OS_"):
+ if key not in safe_keys:
+ value = "<redacted>"
+ print("{}: {}".format(key, value))
+
+ options = vars(args)
+ process_options(options)
+
+ overwrite = "FORCE_REBUILD" in os.environ
+ ensure_container_privs(options, args.container_name)
+ publish_file_to_swift(
+ options, args.container_name, args.swift_object_path, args.local_path,
+ overwrite=overwrite)
+ prune_old_files_from_swift(
+ options, args.container_name, args.swift_object_path.split("/")[0])
+
+
+if __name__ == "__main__":
+ main()
