curtin-dev team mailing list archive
-
curtin-dev team
-
Mailing list archive
-
Message #03620
[Merge] ~ogayot/curtin:apt-transport-mirror into curtin:master
Olivier Gayot has proposed merging ~ogayot/curtin:apt-transport-mirror into curtin:master.
Commit message:
do not squash
Requested reviews:
Server Team CI bot (server-team-bot): continuous-integration
curtin developers (curtin-dev)
For more details, see:
https://code.launchpad.net/~ogayot/curtin/+git/curtin/+merge/472307
Add support for mirrorlist-s in curtin.
A mirrorlist allows one to specify multiple mirrors, with different selection criteria, for a single APT source.
During installation, we want to make Subiquity lean on this new feature of curtin to specify http://archive.ubuntu.com/ubuntu (or http://ports.ubuntu.com/ubuntu-ports) as a low priority mirror so that APT can automatically use it as the fallback if the requested mirror (country mirror most likely) does not work.
The mirrorlist is expected to look like this:
```
http://fr.archive.ubuntu.com/ubuntu priority=1
http://archive.ubuntu.com/ubuntu
```
And the associated apt-config, should look like this:
```
apt:
mirrorlists:
- path: /etc/apt/mirrorlists/subiquity.txt
mirrors:
- uri: http://fr.archive.ubuntu.com/ubuntu
meta-data: {priority:1}
- uri: http://archive.ubuntu.com/ubuntu
primary:
- arches:
- default
uri: mirror+file:/etc/apt/mirrorlists/subiquity.txt
- security:
...
```
--
Your team curtin developers is requested to review the proposed merge of ~ogayot/curtin:apt-transport-mirror into curtin:master.
diff --git a/curtin/commands/apt_config.py b/curtin/commands/apt_config.py
index d38cc4c..62d6714 100644
--- a/curtin/commands/apt_config.py
+++ b/curtin/commands/apt_config.py
@@ -10,6 +10,8 @@ import glob
import os
import re
import sys
+from pathlib import Path
+from typing import List
from aptsources.sourceslist import SourceEntry
@@ -262,6 +264,32 @@ def convert_sources_to_deb822(sources):
return '\n'.join([deb822_entry_to_str(e) for e in index.values()])
+def add_apt_mirrorlists(mirrorlist_list, target):
+ for mirrorlist in mirrorlist_list:
+ requested_path = Path(mirrorlist["path"])
+ if requested_path.is_absolute():
+ path = Path(target) / requested_path.relative_to("/")
+ else:
+ path = Path(target) / "etc/apt/mirrorlists" / requested_path
+
+ if not path.resolve().is_relative_to(target):
+ # This is to avoid path traversal.
+ raise ValueError(f"Path {path} is outside of {target}")
+
+ directives: List[str] = []
+ for mirror in mirrorlist["mirrors"]:
+ uri = mirror["uri"]
+ metadata = mirror.get("meta-data")
+ if metadata:
+ md_str = " ".join([f"{k}:{v}" for k, v in metadata.items()])
+ directives.append(f"{uri}\t{md_str}")
+ else:
+ directives.append(uri)
+
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text("\n".join(directives) + "\n")
+
+
def handle_apt(cfg, target=None):
""" handle_apt
process the config for apt_config. This can be called from
@@ -306,6 +334,9 @@ def handle_apt(cfg, target=None):
add_apt_sources(cfg['sources'], target,
template_params=params, aa_repo_match=matcher)
+ if 'mirrorlists' in cfg:
+ add_apt_mirrorlists(cfg['mirrorlists'], target)
+
def debconf_set_selections(selections, target=None):
util.subp(['debconf-set-selections'], data=selections, target=target,
diff --git a/examples/apt-source.yaml b/examples/apt-source.yaml
index 30e30a2..9de34ad 100644
--- a/examples/apt-source.yaml
+++ b/examples/apt-source.yaml
@@ -4,8 +4,9 @@ apt:
#
# On one hand there is the global configuration for the apt feature.
#
- # On one hand (down in this file) there is the source dictionary which allows
- # to define various entries to be considered by apt.
+ # On one hand (down in this file) there is the sources dictionary and the
+ # mirrorlists dictionary, which allow to define various entries to be
+ # considered by apt.
##############################################################################
# Section 1: global apt configuration
@@ -166,7 +167,9 @@ apt:
##############################################################################
- # Section 2: source list entries
+ # Section 2: source list entries and mirrorlist entries
+ #
+ # 2.1 sources
#
# This is a dictionary (unlike most block/net which are lists)
#
@@ -179,7 +182,7 @@ apt:
# not used as filename - yet it can still be used as index for merging
# configuration.
#
- # The values inside the entries consost of the following optional entries:
+ # The values inside the entries consist of the following optional entries:
# 'source': a sources.list entry (some variable replacements apply)
# 'keyid': providing a key to import via shortid or fingerprint
# 'key': providing a raw PGP key
@@ -206,23 +209,22 @@ apt:
# The following examples number the subfeatures per sources entry to ease
# identification in discussions.
-
sources:
curtin-dev-ppa.list:
- # 2.1 source
+ # 2.1.1 source
#
# Creates a file in /etc/apt/sources.list.d/ for the sources list entry
# based on the key: "/etc/apt/sources.list.d/curtin-dev-ppa.list"
source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main"
- # 2.2 keyid
+ # 2.1.2 keyid
#
# Importing a gpg key for a given key id. Used keyserver defaults to
# keyserver.ubuntu.com
keyid: F430BBA5 # GPG key ID published on a key server
ignored1:
- # 2.3 PPA shortcut
+ # 2.1.3 PPA shortcut
#
# Setup correct apt sources.list line and Auto-Import the signing key
# from LP
@@ -234,7 +236,7 @@ apt:
source: "ppa:curtin-dev/test-archive" # Quote the string
my-repo2.list:
- # 2.4 replacement variables
+ # 2.1.4 replacement variables
#
# sources can use $MIRROR, $PRIMARY, $SECURITY and $RELEASE replacement
# variables.
@@ -251,20 +253,20 @@ apt:
filename: curtin-dev-ppa.list
ignored2:
- # 2.5 key only
+ # 2.1.5 key only
#
# this would only import the key without adding a ppa or other source spec
# since this doesn't generate a source.list file the filename key is ignored
keyid: F430BBA5 # GPG key ID published on a key server
ignored3:
- # 2.6 key id alternatives
+ # 2.1.6 key id alternatives
#
# Keyid's can also be specified via their long fingerprints
keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77
ignored4:
- # 2.7 alternative keyservers
+ # 2.1.7 alternative keyservers
#
# One can also specify alternative keyservers to fetch keys from.
keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77
@@ -272,7 +274,7 @@ apt:
my-repo4.list:
- # 2.8 raw key
+ # 2.1.8 raw key
#
# The apt signing key can also be specified by providing a pgp public key
# block. Providing the PGP key this way is the most robust method for
@@ -293,3 +295,57 @@ apt:
uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey
=Y2oI
-----END PGP PUBLIC KEY BLOCK-----
+
+
+ # 2.2 mirrorlists
+ #
+ # This is a list of individual mirrorlist entries
+ #
+ # The values inside the entries consist of the following properties:
+ # 'path': the location where to store the mirrorlist. It will be prepended
+ # by /etc/apt/mirrorlists/ if it does not start with a '/'.
+ # 'mirrors': specifying the list of mirrors to use (see below)
+
+ # Each mirror entry consists of the following properties:
+ # 'uri': (required) specifying the URI to use to reach the mirror
+ # 'meta-data': (optional) providing a dictionary of meta-data as documented
+ # in apt-transport-mirror(1).
+
+ # The following examples showcase some of the possible use-cases:
+
+ mirrorlists:
+ # 2.2.1 Random use of two country mirrors
+ #
+ # Specify two country mirrors. APT will contact them in an unspecified
+ # order.
+ #
+ # To use the mirrorlist, one must "activate" it by using the following in a
+ # deb822 source list:
+ #
+ # Types: deb
+ # URIs: mirror+file:/etc/apt/mirrorlist.txt
+ # [...]
+ - path: /etc/apt/mirrorlist.txt
+ mirrors:
+ - uri: "http://fr.archive.ubuntu.com/ubuntu";
+ - uri: "http://de.archive.ubuntu.com/ubuntu";
+
+ # 2.2.2 Conditional fallback
+ #
+ # Using this mirrorlist, APT will;
+ #
+ # * first, contact internal mirrors in a random order
+ # * if that fails, contact the country mirrors (french and german) also in
+ # a random order
+ # * if everything else fails, use the main archive
+ - path: custom.txt
+ mirrors:
+ - uri: "http://mirror1.internal/ubuntu";
+ meta-data: {priority: 1}
+ - uri: "http://mirror2.internal/ubuntu";
+ meta-data: {priority: 1}
+ - uri: "http://fr.archive.ubuntu.com/ubuntu/";
+ meta-data: {priority: 2}
+ - uri: "http://de.archive.ubuntu.com/ubuntu/";
+ meta-data: {priority: 2}
+ - uri: "http://archive.ubuntu.com/ubuntu/";
diff --git a/tests/unittests/test_apt_source.py b/tests/unittests/test_apt_source.py
index 2b07045..83758dd 100644
--- a/tests/unittests/test_apt_source.py
+++ b/tests/unittests/test_apt_source.py
@@ -7,6 +7,7 @@ import glob
import os
import re
import socket
+from pathlib import Path
from unittest import mock
@@ -1731,6 +1732,54 @@ deb-src http://ubuntu.com//ubuntu xenial universe multiverse
.format(filename, entry, contents)
)
+ def test_add_apt_mirrorlists(self):
+ mirror_list1 = {
+ 'path': '/etc/apt/test1.txt',
+ 'mirrors': [
+ # Try fr.archive.ubuntu.com/ubuntu first, then fallback to
+ # archive.ubuntu.com/ubuntu
+ {'uri': 'http://fr.archive.ubuntu.com/ubuntu', 'meta-data':
+ {'priority': 1}},
+ {'uri': 'http://archive.ubuntu.com/ubuntu'},
+ ]
+ }
+ # This one is inspired from apt-transport-mirror(1) manual page.
+ mirror_list2 = {
+ 'path': 'test2.txt',
+ 'mirrors': [
+ {'uri': 'file:/srv/local/ubuntu/mirror/',
+ 'meta-data': {'priority': 1, 'type': 'index'}},
+ {'uri': 'http://partial.example.org/mirror/',
+ 'meta-data': {'priority': 2, 'type': 'deb'}},
+ {'uri': 'http://us.archive.ubuntu.com/ubuntu/',
+ 'meta-data': {'type': 'deb'}},
+ {'uri': 'http://de.archive.ubuntu.com/ubuntu/',
+ 'meta-data': {'type': 'deb'}},
+ {'uri': 'http://archive.ubuntu.com/ubuntu/'},
+ ]
+ }
+
+ apt_config.add_apt_mirrorlists([mirror_list1, mirror_list2], target=self.target)
+
+ expected_content1 = '''\
+http://fr.archive.ubuntu.com/ubuntu\tpriority:1
+http://archive.ubuntu.com/ubuntu
+'''
+ expected_content2 = '''\
+file:/srv/local/ubuntu/mirror/\tpriority:1 type:index
+http://partial.example.org/mirror/\tpriority:2 type:deb
+http://us.archive.ubuntu.com/ubuntu/\ttype:deb
+http://de.archive.ubuntu.com/ubuntu/\ttype:deb
+http://archive.ubuntu.com/ubuntu/
+'''
+
+ self.assertEqual(
+ expected_content1,
+ (Path(self.target) / "etc/apt/test1.txt").read_text())
+ self.assertEqual(
+ expected_content2,
+ (Path(self.target) / "etc/apt/mirrorlists/test2.txt").read_text())
+
class TestDebconfSelections(CiTestCase):
