apport-hackers team mailing list archive
-
apport-hackers team
-
Mailing list archive
-
Message #00206
[Merge] lp:~brian-murray/apport/support-ppa-packages into lp:apport
Brian Murray has proposed merging lp:~brian-murray/apport/support-ppa-packages into lp:apport.
Requested reviews:
Apport upstream developers (apport-hackers)
For more details, see:
https://code.launchpad.net/~brian-murray/apport/support-ppa-packages/+merge/263437
This branch builds upon Tim Lunn's initial work (https://code.launchpad.net/~darkxst/apport/per-ppa-config2/+merge/180972) to add PPA support to apport-retrace.
A list of "origins" is created for packages with a foreign origin. This is then passed to install packages so that the sandbox will be created with proper apt data sources for the PPA. Its also possible to setup an apt data source that will be used when creating the sandbox. For consistency with apt these sources are located in /etc/apt/sources.list.d/, they can either be named the same as the PPA (brian-murray-ppa.list) or with LP-PPA- preceeding the PPA name. This was done since the origin information is listed as LP-PPA-brian-murray-ppa and people may use that when setting up a configuration for apport-retrace. I've also added apt keyring support for any PPAs added.
The following new tests were added to test_backend_apt_dpkg.py:
def test_create_sources_for_a_named_ppa(self):
def test_create_sources_for_an_unnamed_ppa(self):
def test_use_sources_for_a_ppa(self):
def test_install_package_from_a_ppa(self):
They test the creation of a sources.list entry for two different types of PPAs, using a configured sources.list entry for a PPA, and installing a package from a PPA. Three of the tests are dependent upon two PPAs being available which might be a point of concern.
--
Your team Apport upstream developers is requested to review the proposed merge of lp:~brian-murray/apport/support-ppa-packages into lp:apport.
=== modified file 'apport/sandboxutils.py'
--- apport/sandboxutils.py 2015-06-11 05:49:39 +0000
+++ apport/sandboxutils.py 2015-06-30 22:39:00 +0000
@@ -10,7 +10,7 @@
# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.
-import atexit, os, os.path, shutil, tempfile
+import atexit, os, os.path, re, shutil, tempfile
import apport
@@ -179,12 +179,19 @@
if config_dir == 'system':
config_dir = None
+ pkg_list = report.get('Package', '') + '\n' + report.get('Dependencies', '')
+ m = re.compile('\[origin: ([a-zA-Z0-9][a-zA-Z0-9\+\.\-]+)\]$')
+ origins = set(m.findall(pkg_list))
+ origins -= set([u'unknown'])
+ if origins:
+ apport.log("Origins: %s" % origins)
+
# unpack packages, if any, using cache and sandbox
try:
outdated_msg = apport.packaging.install_packages(
sandbox_dir, config_dir, report['DistroRelease'], pkgs,
verbose, cache_dir, permanent_rootdir,
- architecture=report.get('Architecture'))
+ architecture=report.get('Architecture'), origins=origins)
except SystemError as e:
apport.fatal(str(e))
=== modified file 'backends/packaging-apt-dpkg.py'
--- backends/packaging-apt-dpkg.py 2015-06-29 13:31:02 +0000
+++ backends/packaging-apt-dpkg.py 2015-06-30 22:39:00 +0000
@@ -24,9 +24,10 @@
from urllib import urlopen, quote, unquote
(pickle, urlopen, quote, unquote) # pyflakes
URLError = IOError
+ HTTPError = IOError
except ImportError:
# python 3
- from urllib.error import URLError
+ from urllib.error import URLError, HTTPError
from urllib.request import urlopen
from urllib.parse import quote, unquote
import pickle
@@ -45,8 +46,10 @@
self._contents_dir = None
self._mirror = None
self._virtual_mapping_obj = None
+ self._origins = None
self._launchpad_base = 'https://api.launchpad.net/devel'
self._archive_url = self._launchpad_base + '/%s/main_archive'
+ self._ppa_archive_url = self._launchpad_base + '/~%s/+archive/ubuntu/%s'
def __del__(self):
try:
@@ -95,7 +98,7 @@
'''
self._apt_cache = None
if not self._sandbox_apt_cache:
- self._build_apt_sandbox(aptroot, apt_sources)
+ self._build_apt_sandbox(aptroot, apt_sources, self._origins)
rootdir = os.path.abspath(aptroot)
self._sandbox_apt_cache = apt.Cache(rootdir=rootdir)
try:
@@ -641,7 +644,8 @@
def install_packages(self, rootdir, configdir, release, packages,
verbose=False, cache_dir=None,
- permanent_rootdir=False, architecture=None):
+ permanent_rootdir=False, architecture=None,
+ origins=None):
'''Install packages into a sandbox (for apport-retrace).
In order to work without any special permissions and without touching
@@ -670,6 +674,9 @@
the given architecture (as specified in a report's "Architecture"
field). If not given it defaults to the host system's architecture.
+ If origins is given, the sandbox will be created with apt data sources
+ for any origins that are Launchpad PPAs.
+
Return a string with outdated packages, or None if all packages were
installed.
@@ -691,6 +698,8 @@
if os.path.exists(arch_apt_sources):
apt_sources = arch_apt_sources
+ self._origins = origins
+
# set mirror for get_file_package()
try:
self.set_mirror(self._get_primary_mirror_from_apt_sources(apt_sources))
@@ -730,7 +739,7 @@
if not tmp_aptroot:
cache = self._sandbox_cache(aptroot, apt_sources, fetchProgress)
else:
- self._build_apt_sandbox(aptroot, apt_sources)
+ self._build_apt_sandbox(aptroot, apt_sources, self._origins)
cache = apt.Cache(rootdir=os.path.abspath(aptroot))
try:
cache.update(fetchProgress)
@@ -1184,7 +1193,46 @@
return None
@classmethod
- def _build_apt_sandbox(klass, apt_root, apt_sources):
+ def create_ppa_source_from_origin(klass, origin, codename):
+ '''For an origin from a Launchpad PPA create sources.list content.'''
+
+ if origin.startswith("LP-PPA-"):
+ components = origin[7:].split("-")
+ try_ppa = True
+ if len(components) == 1:
+ components.append('ppa')
+ try_ppa = False
+
+ index = 1
+ while (index < len(components)):
+ user = str.join('-', components[0:index])
+ ppa_name = str.join('-', components[index:len(components)])
+ try:
+ urlopen(apport.packaging._ppa_archive_url % (user, ppa_name))
+ except (URLError, HTTPError):
+ index += 1
+ if index == len(components):
+ if try_ppa:
+ components.append('ppa')
+ try_ppa = False
+ index = 2
+ else:
+ user = None
+ continue
+ break
+ if user and ppa_name:
+ ppa_line = "deb http://ppa.launchpad.net/%s/%s/ubuntu %s main" % (user, ppa_name, codename)
+ debug_url = "http://ppa.launchpad.net/%s/%s/ubuntu/dists/%s/main/debug"; % (user, ppa_name, codename)
+ try:
+ urlopen(debug_url)
+ add_debug = " main/debug"
+ except (URLError, HTTPError):
+ add_debug = ""
+ return ppa_line + add_debug + "\ndeb-src" + ppa_line[3:] + "\n"
+ return None
+
+ @classmethod
+ def _build_apt_sandbox(klass, apt_root, apt_sources, origins=None):
# pre-create directories, to avoid apt.Cache() printing "creating..."
# messages on stdout
if not os.path.exists(os.path.join(apt_root, 'var', 'lib', 'apt')):
@@ -1204,7 +1252,48 @@
os.makedirs(list_d)
with open(apt_sources) as src:
with open(os.path.join(apt_root, 'etc', 'apt', 'sources.list'), 'w') as dest:
- dest.write(src.read())
+ sources = src.read()
+ release = None
+ release = apport.packaging.get_distro_codename()
+ for line in sources.split('\n'):
+ if not line.startswith('#') and len(line.split()) >= 3:
+ release = line.split()[2]
+ if '-' in release:
+ continue
+ else:
+ break
+ dest.write(sources)
+
+ if origins:
+ source_list = ''
+ origin_data = {}
+ for origin in origins:
+ if os.path.isdir(apt_sources + '.d'):
+ origin_path = os.path.join(apt_sources + '.d', origin + '.list')
+ if 'LP-PPA' in origin and not os.path.exists(origin_path):
+ origin_path = os.path.join(apt_sources + '.d',
+ origin.strip('LP-PPA-') + '.list')
+ else:
+ origin_path = ''
+ if os.path.exists(origin_path):
+ with open(origin_path) as src_ext:
+ source_list = src_ext.read()
+ else:
+ source_list = klass.create_ppa_source_from_origin(origin, release)
+ if source_list:
+ with open(os.path.join(apt_root, 'etc', 'apt',
+ 'sources.list.d', origin + '.list'), 'a') as dest:
+ dest.write(source_list)
+ for line in source_list.splitlines():
+ if line.startswith('#'):
+ continue
+ if 'ppa.launchpad.net' not in line:
+ continue
+ user = line.split()[1].split('/')[3]
+ ppa = line.split()[1].split('/')[4]
+ origin_data[origin] = (user, ppa)
+ else:
+ apport.warning("Could not find source config for %s" % origin)
# install apt keyrings; prefer the ones from the config dir, fall back
# to system
@@ -1225,6 +1314,41 @@
else:
os.makedirs(trusted_d)
+ # install apt keyrings for PPAs
+ if origins and source_list:
+ for origin in origin_data:
+ ppa_user = origin_data[origin][0]
+ ppa_name = origin_data[origin][1]
+ ppa_archive_url = apport.packaging._ppa_archive_url % \
+ (quote(ppa_user), quote(ppa_name))
+ ppa_info = apport.packaging.json_request(ppa_archive_url)
+ if not ppa_info:
+ continue
+ try:
+ signing_key_fingerprint = ppa_info["signing_key_fingerprint"]
+ except IndexError:
+ apport.warning("Error: can't find signing_key_fingerprint at %s"
+ % ppa_archive_url)
+ continue
+ tmpdir = tempfile.mkdtemp()
+ gpg = ['/usr/bin/gpg']
+ argv = gpg + ['--no-options',
+ '--no-default-keyring',
+ '--no-auto-check-trustdb',
+ '--keyring',
+ os.path.join(trusted_d,
+ '%s.gpg' % origin),
+ ]
+ argv += ['--secret-keyring',
+ os.path.join(tmpdir, 'secring.gpg'),
+ '--quiet', '--batch',
+ '--keyserver', 'hkp://keyserver.ubuntu.com:80/',
+ '--recv', signing_key_fingerprint]
+ if subprocess.call(argv) != 0:
+ apport.warning('Unable to import key for %s' %
+ ppa_archive_url)
+ pass
+
@classmethod
def _deb_version(klass, pkg):
'''Return the version of a .deb file'''
=== modified file 'test/test_backend_apt_dpkg.py'
--- test/test_backend_apt_dpkg.py 2015-06-29 13:31:02 +0000
+++ test/test_backend_apt_dpkg.py 2015-06-30 22:39:00 +0000
@@ -976,7 +976,62 @@
self.assertTrue(res.endswith('/debian-installer-20101020ubuntu318.16'),
'unexpected version: ' + res.split('/')[-1])
- def _setup_foonux_config(self, updates=False, release='trusty'):
+ @unittest.skipUnless(_has_internet(), 'online test')
+ def test_create_sources_for_a_named_ppa(self):
+ '''Add sources.list entries for a named PPA.'''
+ ppa = 'LP-PPA-ci-train-ppa-service-stable-phone-overlay'
+ self._setup_foonux_config(release='vivid')
+ impl._build_apt_sandbox(self.rootdir, os.path.join(self.configdir, 'Foonux 1.2', 'sources.list'),
+ origins=[ppa])
+ with open(os.path.join(self.rootdir, 'etc', 'apt', 'sources.list.d', ppa + '.list')) as f:
+ sources = f.read().splitlines()
+ self.assertIn('deb http://ppa.launchpad.net/ci-train-ppa-service/stable-phone-overlay/ubuntu vivid main main/debug', sources)
+ self.assertIn('deb-src http://ppa.launchpad.net/ci-train-ppa-service/stable-phone-overlay/ubuntu vivid main', sources)
+
+ @unittest.skipUnless(_has_internet(), 'online test')
+ def test_create_sources_for_an_unnamed_ppa(self):
+ '''Add sources.list entries for an unnamed PPA.'''
+ ppa = 'LP-PPA-brian-murray'
+ self._setup_foonux_config()
+ impl._build_apt_sandbox(self.rootdir, os.path.join(self.configdir, 'Foonux 1.2', 'sources.list'),
+ origins=[ppa])
+ with open(os.path.join(self.rootdir, 'etc', 'apt', 'sources.list.d', ppa + '.list')) as f:
+ sources = f.read().splitlines()
+ self.assertIn('deb http://ppa.launchpad.net/brian-murray/ppa/ubuntu trusty main', sources)
+ self.assertIn('deb-src http://ppa.launchpad.net/brian-murray/ppa/ubuntu trusty main', sources)
+
+ def test_use_sources_for_a_ppa(self):
+ '''Use a sources.list.d file for a PPA.'''
+ ppa = 'fooser-bar-ppa'
+ self._setup_foonux_config(ppa=True)
+ impl._build_apt_sandbox(self.rootdir, os.path.join(self.configdir, 'Foonux 1.2', 'sources.list'),
+ origins=['LP-PPA-%s' % ppa])
+ with open(os.path.join(self.rootdir, 'etc', 'apt', 'sources.list.d', ppa + '.list')) as f:
+ sources = f.read().splitlines()
+ self.assertIn('deb http://ppa.launchpad.net/fooser/bar-ppa/ubuntu trusty main main/debug', sources)
+ self.assertIn('deb-src http://ppa.launchpad.net/fooser/bar-ppa/ubuntu trusty main', sources)
+
+ @unittest.skipUnless(_has_internet(), 'online test')
+ def test_install_package_from_a_ppa(self):
+ '''Install a package from a PPA.'''
+ ppa = 'LP-PPA-brian-murray'
+ self._setup_foonux_config(release='trusty')
+ obsolete = impl.install_packages(self.rootdir, self.configdir, 'Foonux 1.2',
+ [('apport',
+ '2.14.1-0ubuntu3.7~ppa4')
+ ], False, self.cachedir, origins=[ppa])
+
+ self.assertEqual(obsolete, '')
+
+ def sandbox_ver(pkg):
+ with gzip.open(os.path.join(self.rootdir, 'usr/share/doc', pkg,
+ 'changelog.Debian.gz')) as f:
+ return f.readline().decode().split()[1][1:-1]
+
+ self.assertEqual(sandbox_ver('apport'),
+ '2.14.1-0ubuntu3.7~ppa4')
+
+ def _setup_foonux_config(self, updates=False, release='trusty', ppa=False):
'''Set up directories and configuration for install_packages()'''
self.cachedir = os.path.join(self.workdir, 'cache')
@@ -994,6 +1049,11 @@
f.write('deb http://archive.ubuntu.com/ubuntu/ %s-updates main\n' % release)
f.write('deb-src http://archive.ubuntu.com/ubuntu/ %s-updates main\n' % release)
f.write('deb http://ddebs.ubuntu.com/ %s-updates main\n' % release)
+ if ppa:
+ os.mkdir(os.path.join(self.configdir, 'Foonux 1.2', 'sources.list.d'))
+ with open(os.path.join(self.configdir, 'Foonux 1.2', 'sources.list.d', 'fooser-bar-ppa.list'), 'w') as f:
+ f.write('deb http://ppa.launchpad.net/fooser/bar-ppa/ubuntu %s main main/debug\n' % release)
+ f.write('deb-src http://ppa.launchpad.net/fooser/bar-ppa/ubuntu %s main\n' % release)
os.mkdir(os.path.join(self.configdir, 'Foonux 1.2', 'armhf'))
with open(os.path.join(self.configdir, 'Foonux 1.2', 'armhf', 'sources.list'), 'w') as f:
f.write('deb http://ports.ubuntu.com/ %s main\n' % release)
