launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging ~cjwatson/turnip:use-pip-cache into turnip:master.

Commit message:
Actually use pip cache at build time

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/turnip/+git/turnip/+merge/399717

turnip's Makefile was building a pip cache, but inexplicably not actually using it at build time.  On my laptop, using the pip cache takes the build time on my laptop (removing ~/.cache/pip/ before each test) from 8m3s to 31s.

In the process, I switched to configuring pip using environment variables; this is more reliable, as environment variables are passed through to pip processes run by setuptools when handling setup_requires.

I suspect this is a large part of why turnip code asset deployments have been so slow.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/turnip:use-pip-cache into turnip:master.

diff --git a/Makefile b/Makefile
index e0362ee..26bf506 100644
--- a/Makefile
+++ b/Makefile
@@ -16,10 +16,17 @@ VENV_ARGS := -p python3
 DEPENDENCIES_URL := https://git.launchpad.net/~canonical-launchpad-branches/turnip/+git/dependencies
 PIP_SOURCE_DIR := dependencies
 
-PIP_ARGS ?= --quiet
+# virtualenv and pip fail if setlocale fails, so force a valid locale.
+PIP_ENV := LC_ALL=C.UTF-8
+# "make PIP_QUIET=0" causes pip to be verbose.
+PIP_QUIET := 1
+PIP_ENV += PIP_QUIET=$(PIP_QUIET)
+PIP_FIND_LINKS := file://$(PIP_CACHE)/
 ifneq ($(PIP_SOURCE_DIR),)
-override PIP_ARGS += --no-index --find-links=file://$(shell readlink -f $(PIP_SOURCE_DIR))/
+PIP_ENV += PIP_NO_INDEX=1
+PIP_FIND_LINKS += file://$(shell readlink -f $(PIP_SOURCE_DIR))/
 endif
+PIP_ENV += PIP_FIND_LINKS="$(PIP_FIND_LINKS)"
 
 # Create archives in labelled directories (e.g.
 # <rev-id>/$(PROJECT_NAME).tar.gz)
@@ -60,9 +67,9 @@ endif
 	(echo '[easy_install]'; \
 	 echo 'find_links = file://$(realpath $(PIP_SOURCE_DIR))/') \
 		>$(ENV)/.pydistutils.cfg
-	$(VIRTUALENV) $(VENV_ARGS) --never-download $(ENV)
-	$(PIP) install $(PIP_ARGS) -r bootstrap-requirements.txt
-	$(PIP) install $(PIP_ARGS) -c requirements.txt \
+	$(PIP_ENV) $(VIRTUALENV) $(VENV_ARGS) --never-download $(ENV)
+	$(PIP_ENV) $(PIP) install -r bootstrap-requirements.txt
+	$(PIP_ENV) $(PIP) install -c requirements.txt \
 		-e '.[test,deploy]'
 
 bootstrap-test: PATH := /usr/sbin:/sbin:$(PATH)
@@ -131,7 +138,7 @@ stop:
 
 $(PIP_CACHE): $(ENV)
 	mkdir -p $(PIP_CACHE)
-	$(PIP) install $(PIP_ARGS) -d $(PIP_CACHE) \
+	$(PIP_ENV) $(PIP) install -d $(PIP_CACHE) \
 		-r bootstrap-requirements.txt \
 		-r requirements.txt