launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #23819
[Merge] lp:~cjwatson/launchpad/mirror-prober-redirects into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/mirror-prober-redirects into lp:launchpad.
Commit message:
Allow archive mirrors to return redirect responses.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
Related bugs:
Bug #1836712 in Launchpad itself: "Launchpad mirror prober considers http redirects broken"
https://bugs.launchpad.net/launchpad/+bug/1836712
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/mirror-prober-redirects/+merge/370493
APT used to not support these, but this was changed over ten years ago.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/mirror-prober-redirects into lp:launchpad.
=== modified file 'database/sampledata/current-dev.sql'
--- database/sampledata/current-dev.sql 2019-05-22 14:57:45 +0000
+++ database/sampledata/current-dev.sql 2019-07-23 14:58:40 +0000
@@ -3443,6 +3443,7 @@
INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (8, 1, 'canonical-archive', 'http://archive.ubuntu.com/ubuntu/', NULL, NULL, NULL, NULL, 1, 70, 225, 1, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (9, 1, 'canonical-releases', 'http://releases.ubuntu.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (10, 1, 'random-releases-mirror', 'http://releases.random.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 10, NULL, NULL, false);
+INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (11, 1, 'archive-redirect-mirror', 'http://localhost:11375/redirect-to-valid-mirror/', NULL, NULL, NULL, NULL, 1, 10, 9, 1, true, true, '2019-07-23 13:16:51.953354', NULL, 30, NULL, NULL, false);
ALTER TABLE distributionmirror ENABLE TRIGGER ALL;
=== modified file 'database/sampledata/current.sql'
--- database/sampledata/current.sql 2019-05-22 14:57:45 +0000
+++ database/sampledata/current.sql 2019-07-23 14:58:40 +0000
@@ -1,4 +1,4 @@
--- Copyright 2010-2017 Canonical Ltd. This software is licensed under the
+-- Copyright 2010-2019 Canonical Ltd. This software is licensed under the
-- GNU Affero General Public License version 3 (see the file LICENSE).
-- Created using pg_dump (PostgreSQL) 9.3.5
@@ -3380,6 +3380,7 @@
INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (8, 1, 'canonical-archive', 'http://archive.ubuntu.com/ubuntu/', NULL, NULL, NULL, NULL, 1, 70, 225, 1, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (9, 1, 'canonical-releases', 'http://releases.ubuntu.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (10, 1, 'random-releases-mirror', 'http://releases.random.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 10, NULL, NULL, false);
+INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (11, 1, 'archive-redirect-mirror', 'http://localhost:11375/redirect-to-valid-mirror/', NULL, NULL, NULL, NULL, 1, 10, 9, 1, true, true, '2019-07-23 13:16:51.953354', NULL, 30, NULL, NULL, false);
ALTER TABLE distributionmirror ENABLE TRIGGER ALL;
=== modified file 'lib/lp/registry/doc/distribution-mirror.txt'
--- lib/lp/registry/doc/distribution-mirror.txt 2019-05-22 14:57:45 +0000
+++ lib/lp/registry/doc/distribution-mirror.txt 2019-07-23 14:58:40 +0000
@@ -122,7 +122,8 @@
[(u'canonical-archive', '100 Mbps', 'OFFICIAL'),
(u'archive-404-mirror', '512 Kbps', 'OFFICIAL'),
(u'archive-mirror', '128 Kbps', 'OFFICIAL'),
- (u'archive-mirror2', '128 Kbps', 'OFFICIAL')]
+ (u'archive-mirror2', '128 Kbps', 'OFFICIAL'),
+ (u'archive-redirect-mirror', '128 Kbps', 'OFFICIAL')]
>>> [(mirror.name, mirror.speed.title, mirror.status.name)
... for mirror in ubuntu.cdimage_mirrors]
@@ -242,7 +243,7 @@
>>> [mirror.name
... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.ARCHIVE)]
[u'canonical-archive', u'archive-404-mirror', u'archive-mirror',
- u'archive-mirror2']
+ u'archive-mirror2', u'archive-redirect-mirror']
>>> [mirror.name
... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.RELEASE)]
@@ -263,7 +264,8 @@
>>> [mirror.name
... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.ARCHIVE)]
- [u'canonical-archive', u'archive-404-mirror', u'archive-mirror2']
+ [u'canonical-archive', u'archive-404-mirror', u'archive-mirror2',
+ u'archive-redirect-mirror']
>>> [mirror.name
... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.RELEASE)]
@@ -278,7 +280,7 @@
... MirrorContent.ARCHIVE, ignore_last_probe=True)
>>> [mirror.name for mirror in mirrors]
[u'canonical-archive', u'archive-404-mirror', u'archive-mirror',
- u'archive-mirror2']
+ u'archive-mirror2', u'archive-redirect-mirror']
It's also possible to limit the number of mirrors we want to probe by
passing an extra limit= argument to mirrorset.getMirrorsToProbe(). If
@@ -306,9 +308,10 @@
archive-404-mirror NEVER
archive-mirror TODAY
archive-mirror2 2006-05-24...
+ archive-redirect-mirror NEVER
>>> mirrors = mirrorset.getMirrorsToProbe(
- ... MirrorContent.ARCHIVE, ignore_last_probe=True, limit=3)
+ ... MirrorContent.ARCHIVE, ignore_last_probe=True, limit=4)
>>> for mirror in mirrors:
... last_probe = mirror.last_probe_record
... last_probe_date = 'NEVER'
@@ -318,6 +321,7 @@
canonical-archive NEVER
archive-404-mirror NEVER
archive-mirror2 2006-05-24...
+ archive-redirect-mirror NEVER
If when we finish probing a mirror, that mirror doesn't have any
@@ -400,6 +404,7 @@
getExpectedPackagesPaths() and getExpectedSourcesPaths() methods of
IDistributionMirror.
+ >>> mirror = mirrorset.getByName('archive-mirror2')
>>> paths = mirror.getExpectedPackagesPaths()
>>> [path for (series, pocket, component, path) in paths]
[u'dists/breezy-autotest/main/binary-i386/Packages.gz',
@@ -646,7 +651,7 @@
>>> print stderr
INFO Creating lockfile: /var/lock/launchpad-distributionmirror-prober.lock
INFO Probing Archive Mirrors
- INFO Probed 4 mirrors.
+ INFO Probed 5 mirrors.
INFO Disabling 1 mirror(s):
http://launchpad.test/ubuntu/+mirror/archive-404-mirror
INFO Re-enabling 1 mirror(s):
=== modified file 'lib/lp/registry/scripts/distributionmirror_prober.py'
--- lib/lp/registry/scripts/distributionmirror_prober.py 2018-07-13 12:48:19 +0000
+++ lib/lp/registry/scripts/distributionmirror_prober.py 2019-07-23 14:58:40 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -688,7 +688,9 @@
callbacks = ArchiveMirrorProberCallbacks(
mirror, series, pocket, component, url, logfile)
unchecked_keys.append(url)
- prober = ProberFactory(url)
+ # APT has supported redirects since 0.7.21 (2009-04-14), so allow
+ # them here too.
+ prober = RedirectAwareProberFactory(url)
deferred = request_manager.run(prober.request_host, prober.probe)
deferred.addCallbacks(
=== modified file 'lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt'
--- lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt 2019-05-22 14:57:45 +0000
+++ lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt 2019-07-23 14:58:40 +0000
@@ -42,8 +42,10 @@
>>> print browser.title
Mirrors :...
>>> print_mirrors_by_countries(browser.contents)
- Antarctica: [(u'Archive-mirror2', u'http', u'128 Kbps',
- u'Six hours behind')]
+ Antarctica:
+ [(u'Archive-mirror2', u'http', u'128 Kbps', u'Six hours behind'),
+ (u'Archive-redirect-mirror', u'http', u'128 Kbps',
+ u'Last update unknown')]
France:
[(u'Archive-404-mirror', u'http', u'512 Kbps', u'Last update unknown'),
(u'Archive-mirror', u'http', u'128 Kbps', u'Last update unknown')]
Follow ups
