launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29070
[Merge] ~andrey-fedoseev/launchpad:uct-upstream into launchpad:master
Andrey Fedoseev has proposed merging ~andrey-fedoseev/launchpad:uct-upstream into launchpad:master with ~andrey-fedoseev/launchpad:uct-export as a prerequisite.
Commit message:
UCT import/export: handle upstream package status and ESM packages
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~andrey-fedoseev/launchpad/+git/launchpad/+merge/428893
This also includes the changes related to UCT export that were added in https://code.launchpad.net/~andrey-fedoseev/launchpad/+git/launchpad/+merge/428152
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~andrey-fedoseev/launchpad:uct-upstream into launchpad:master.
diff --git a/lib/lp/bugs/interfaces/bugtask.py b/lib/lp/bugs/interfaces/bugtask.py
index a8c4e45..8fb2ce2 100644
--- a/lib/lp/bugs/interfaces/bugtask.py
+++ b/lib/lp/bugs/interfaces/bugtask.py
@@ -846,7 +846,7 @@ class IBugTask(IHasBug, IBugTaskDelete):
@call_with(user=REQUEST_USER)
@export_write_operation()
@operation_for_version("beta")
- def transitionToMilestone(new_milestone, user=None):
+ def transitionToMilestone(new_milestone, user):
"""Set the BugTask milestone.
Set the bugtask milestone, making sure that the user is
@@ -859,7 +859,7 @@ class IBugTask(IHasBug, IBugTaskDelete):
@call_with(user=REQUEST_USER)
@export_write_operation()
@operation_for_version("beta")
- def transitionToImportance(new_importance, user=None):
+ def transitionToImportance(new_importance, user):
"""Set the BugTask importance.
Set the bugtask importance, making sure that the user is
@@ -883,7 +883,7 @@ class IBugTask(IHasBug, IBugTaskDelete):
@call_with(user=REQUEST_USER)
@export_write_operation()
@operation_for_version("beta")
- def transitionToStatus(new_status, user=None):
+ def transitionToStatus(new_status, user):
"""Perform a workflow transition to the new_status.
:new_status: new status from `BugTaskStatus`
diff --git a/lib/lp/bugs/model/bugtask.py b/lib/lp/bugs/model/bugtask.py
index 33d3647..90f4e36 100644
--- a/lib/lp/bugs/model/bugtask.py
+++ b/lib/lp/bugs/model/bugtask.py
@@ -933,9 +933,9 @@ class BugTask(StormBase):
# setter methods directly.
setattr(self, synched_attr, PassthroughValue(replica_attr_value))
- def transitionToMilestone(self, new_milestone, user=None):
+ def transitionToMilestone(self, new_milestone, user):
"""See `IBugTask`."""
- if user and not self.userHasBugSupervisorPrivileges(user):
+ if not self.userHasBugSupervisorPrivileges(user):
raise UserCannotEditBugTaskMilestone(
"User does not have sufficient permissions "
"to edit the bug task milestone."
@@ -945,9 +945,9 @@ class BugTask(StormBase):
# notified.
self.bug.clearBugNotificationRecipientsCache()
- def transitionToImportance(self, new_importance, user=None):
+ def transitionToImportance(self, new_importance, user):
"""See `IBugTask`."""
- if user and not self.userHasBugSupervisorPrivileges(user):
+ if not self.userHasBugSupervisorPrivileges(user):
raise UserCannotEditBugTaskImportance(
"User does not have sufficient permissions "
"to edit the bug task importance."
@@ -1030,9 +1030,9 @@ class BugTask(StormBase):
# Non-supervisors can transition to non-supervisor statuses.
return new_status not in BUG_SUPERVISOR_BUGTASK_STATUSES
- def transitionToStatus(self, new_status, user=None, when=None):
+ def transitionToStatus(self, new_status, user, when=None):
"""See `IBugTask`."""
- if not new_status:
+ if not new_status or user is None:
# This is mainly to facilitate tests which, unlike the
# normal status form, don't always submit a status when
# testing the edit form.
@@ -1040,7 +1040,7 @@ class BugTask(StormBase):
new_status = normalize_bugtask_status(new_status)
- if user and not self.canTransitionToStatus(new_status, user):
+ if not self.canTransitionToStatus(new_status, user):
raise UserCannotEditBugTaskStatus(
"Only Bug Supervisors may change status to %s."
% (new_status.title,)
diff --git a/lib/lp/bugs/scripts/bugimport.py b/lib/lp/bugs/scripts/bugimport.py
index 323c9b1..6694d86 100644
--- a/lib/lp/bugs/scripts/bugimport.py
+++ b/lib/lp/bugs/scripts/bugimport.py
@@ -356,10 +356,14 @@ class BugImporter:
# set up bug task
bugtask.datecreated = datecreated
bugtask.transitionToImportance(
- get_enum_value(BugTaskImportance, get_value(bugnode, "importance"))
+ get_enum_value(
+ BugTaskImportance, get_value(bugnode, "importance")
+ ),
+ self.bug_importer,
)
bugtask.transitionToStatus(
- get_enum_value(BugTaskStatus, get_value(bugnode, "status"))
+ get_enum_value(BugTaskStatus, get_value(bugnode, "status")),
+ self.bug_importer,
)
bugtask.transitionToAssignee(
self.getPerson(get_element(bugnode, "assignee"))
diff --git a/lib/lp/bugs/scripts/tests/test_uct.py b/lib/lp/bugs/scripts/tests/test_uct.py
index 47ff535..00f995d 100644
--- a/lib/lp/bugs/scripts/tests/test_uct.py
+++ b/lib/lp/bugs/scripts/tests/test_uct.py
@@ -91,20 +91,20 @@ class TestUCTRecord(TestCase):
UCTRecord.Package(
name="linux",
statuses=[
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="upstream",
+ UCTRecord.SeriesPackageStatus(
+ series="upstream",
status=UCTRecord.PackageStatus.RELEASED,
reason="5.17~rc1",
priority=None,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="impish",
+ UCTRecord.SeriesPackageStatus(
+ series="impish",
status=UCTRecord.PackageStatus.RELEASED,
reason="5.13.0-37.42",
priority=UCTRecord.Priority.MEDIUM,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="devel",
+ UCTRecord.SeriesPackageStatus(
+ series="devel",
status=UCTRecord.PackageStatus.NOT_AFFECTED,
reason="5.15.0-25.25",
priority=UCTRecord.Priority.MEDIUM,
@@ -126,20 +126,20 @@ class TestUCTRecord(TestCase):
UCTRecord.Package(
name="linux-hwe",
statuses=[
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="upstream",
+ UCTRecord.SeriesPackageStatus(
+ series="upstream",
status=UCTRecord.PackageStatus.RELEASED,
reason="5.17~rc1",
priority=None,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="impish",
+ UCTRecord.SeriesPackageStatus(
+ series="impish",
status=UCTRecord.PackageStatus.DOES_NOT_EXIST,
reason="",
priority=None,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="devel",
+ UCTRecord.SeriesPackageStatus(
+ series="devel",
status=UCTRecord.PackageStatus.DOES_NOT_EXIST,
reason="",
priority=None,
@@ -186,8 +186,14 @@ class TextCVE(TestCaseWithFactory):
status=SeriesStatus.DEVELOPMENT,
name="kinetic",
)
- dsp1 = self.factory.makeDistributionSourcePackage(distribution=ubuntu)
- dsp2 = self.factory.makeDistributionSourcePackage(distribution=ubuntu)
+ product_1 = self.factory.makeProduct()
+ product_2 = self.factory.makeProduct()
+ dsp1 = self.factory.makeDistributionSourcePackage(
+ sourcepackagename=product_1.name, distribution=ubuntu
+ )
+ dsp2 = self.factory.makeDistributionSourcePackage(
+ sourcepackagename=product_2.name, distribution=ubuntu
+ )
assignee = self.factory.makePerson()
self.uct_record = UCTRecord(
@@ -224,24 +230,30 @@ class TextCVE(TestCaseWithFactory):
UCTRecord.Package(
name=dsp1.sourcepackagename.name,
statuses=[
- UCTRecord.DistroSeriesPackageStatus(
- distroseries=supported_series.name,
+ UCTRecord.SeriesPackageStatus(
+ series=supported_series.name,
status=UCTRecord.PackageStatus.NOT_AFFECTED,
reason="reason 1",
priority=UCTRecord.Priority.MEDIUM,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries=current_series.name,
+ UCTRecord.SeriesPackageStatus(
+ series=current_series.name,
status=UCTRecord.PackageStatus.RELEASED,
reason="reason 2",
priority=UCTRecord.Priority.MEDIUM,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="devel",
+ UCTRecord.SeriesPackageStatus(
+ series="devel",
status=UCTRecord.PackageStatus.RELEASED,
reason="reason 3",
priority=None,
),
+ UCTRecord.SeriesPackageStatus(
+ series="upstream",
+ status=UCTRecord.PackageStatus.RELEASED,
+ reason="reason 4",
+ priority=None,
+ ),
],
priority=None,
tags=set(),
@@ -250,20 +262,26 @@ class TextCVE(TestCaseWithFactory):
UCTRecord.Package(
name=dsp2.sourcepackagename.name,
statuses=[
- UCTRecord.DistroSeriesPackageStatus(
- distroseries=supported_series.name,
+ UCTRecord.SeriesPackageStatus(
+ series=supported_series.name,
status=UCTRecord.PackageStatus.DOES_NOT_EXIST,
reason="",
priority=None,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries=current_series.name,
+ UCTRecord.SeriesPackageStatus(
+ series=current_series.name,
status=UCTRecord.PackageStatus.DOES_NOT_EXIST,
reason="",
priority=None,
),
- UCTRecord.DistroSeriesPackageStatus(
- distroseries="devel",
+ UCTRecord.SeriesPackageStatus(
+ series="devel",
+ status=UCTRecord.PackageStatus.RELEASED,
+ reason="",
+ priority=None,
+ ),
+ UCTRecord.SeriesPackageStatus(
+ series="upstream",
status=UCTRecord.PackageStatus.RELEASED,
reason="",
priority=None,
@@ -353,6 +371,20 @@ class TextCVE(TestCaseWithFactory):
status_explanation="",
),
],
+ upstream_packages=[
+ CVE.UpstreamPackage(
+ package=product_1,
+ importance=None,
+ status=BugTaskStatus.FIXRELEASED,
+ status_explanation="reason 4",
+ ),
+ CVE.SeriesPackage(
+ package=product_2,
+ importance=None,
+ status=BugTaskStatus.FIXRELEASED,
+ status_explanation="",
+ ),
+ ],
importance=BugTaskImportance.CRITICAL,
status=VulnerabilityStatus.ACTIVE,
assignee=assignee,
@@ -393,7 +425,7 @@ class TestUCTImporterExporter(TestCaseWithFactory):
super().setUp(*args, **kwargs)
celebrities = getUtility(ILaunchpadCelebrities)
self.ubuntu = celebrities.ubuntu
- self.esm = self.factory.makeDistribution("esm")
+ self.esm = self.factory.makeDistribution("ubuntu-esm")
self.bug_importer = celebrities.bug_importer
self.ubuntu_supported_series = self.factory.makeDistroSeries(
distribution=self.ubuntu,
@@ -418,11 +450,13 @@ class TestUCTImporterExporter(TestCaseWithFactory):
status=SeriesStatus.CURRENT,
name="trusty",
)
+ self.product_1 = self.factory.makeProduct()
+ self.product_2 = self.factory.makeProduct()
self.ubuntu_package = self.factory.makeDistributionSourcePackage(
- distribution=self.ubuntu
+ sourcepackagename=self.product_1.name, distribution=self.ubuntu
)
self.esm_package = self.factory.makeDistributionSourcePackage(
- distribution=self.esm
+ sourcepackagename=self.product_2.name, distribution=self.esm
)
for series in (
self.ubuntu_supported_series,
@@ -446,7 +480,9 @@ class TestUCTImporterExporter(TestCaseWithFactory):
)
self.lp_cve = self.factory.makeCVE("2022-23222")
- self.now = datetime.datetime.now(datetime.timezone.utc)
+ self.now = datetime.datetime.now(datetime.timezone.utc).replace(
+ microsecond=0
+ )
self.cve = CVE(
sequence="CVE-2022-23222",
crd=None,
@@ -509,6 +545,20 @@ class TestUCTImporterExporter(TestCaseWithFactory):
status_explanation="needs triage",
),
],
+ upstream_packages=[
+ CVE.UpstreamPackage(
+ package=self.product_1,
+ importance=BugTaskImportance.HIGH,
+ status=BugTaskStatus.FIXRELEASED,
+ status_explanation="fix released",
+ ),
+ CVE.UpstreamPackage(
+ package=self.product_2,
+ importance=BugTaskImportance.LOW,
+ status=BugTaskStatus.WONTFIX,
+ status_explanation="ignored",
+ ),
+ ],
importance=BugTaskImportance.MEDIUM,
status=VulnerabilityStatus.ACTIVE,
assignee=self.factory.makePerson(),
@@ -552,7 +602,10 @@ class TestUCTImporterExporter(TestCaseWithFactory):
bug_tasks = bug.bugtasks # type: List[BugTask]
self.assertEqual(
- len(cve.distro_packages) + len(cve.series_packages), len(bug_tasks)
+ len(cve.distro_packages)
+ + len(cve.series_packages)
+ + len(cve.upstream_packages),
+ len(bug_tasks),
)
bug_tasks_by_target = {t.target: t for t in bug_tasks}
@@ -563,7 +616,7 @@ class TestUCTImporterExporter(TestCaseWithFactory):
t = bug_tasks_by_target[distro_package.package]
package_importance = distro_package.importance or cve.importance
package_importances[
- distro_package.package.sourcepackagename
+ distro_package.package.sourcepackagename.name
] = package_importance
conjoined_primary = t.conjoined_primary
if conjoined_primary:
@@ -580,7 +633,7 @@ class TestUCTImporterExporter(TestCaseWithFactory):
self.assertIn(series_package.package, bug_tasks_by_target)
t = bug_tasks_by_target[series_package.package]
package_importance = package_importances[
- series_package.package.sourcepackagename
+ series_package.package.sourcepackagename.name
]
sp_importance = series_package.importance or package_importance
self.assertEqual(sp_importance, t.importance)
@@ -589,6 +642,19 @@ class TestUCTImporterExporter(TestCaseWithFactory):
series_package.status_explanation, t.status_explanation
)
+ for upstream_package in cve.upstream_packages:
+ self.assertIn(upstream_package.package, bug_tasks_by_target)
+ t = bug_tasks_by_target[upstream_package.package]
+ package_importance = package_importances[
+ upstream_package.package.name
+ ]
+ sp_importance = upstream_package.importance or package_importance
+ self.assertEqual(sp_importance, t.importance)
+ self.assertEqual(upstream_package.status, t.status)
+ self.assertEqual(
+ upstream_package.status_explanation, t.status_explanation
+ )
+
for t in bug_tasks:
self.assertEqual(cve.assignee, t.assignee)
@@ -625,6 +691,32 @@ class TestUCTImporterExporter(TestCaseWithFactory):
lp_cve.cvss,
)
+ def checkCVE(self, expected: CVE, actual: CVE):
+ self.assertEqual(expected.sequence, actual.sequence)
+ self.assertEqual(expected.crd, actual.crd)
+ self.assertEqual(expected.public_date, actual.public_date)
+ self.assertEqual(
+ expected.public_date_at_USN, actual.public_date_at_USN
+ )
+ self.assertListEqual(expected.distro_packages, actual.distro_packages)
+ self.assertListEqual(expected.series_packages, actual.series_packages)
+ self.assertListEqual(
+ expected.upstream_packages, actual.upstream_packages
+ )
+ self.assertEqual(expected.importance, actual.importance)
+ self.assertEqual(expected.status, actual.status)
+ self.assertEqual(expected.assignee, actual.assignee)
+ self.assertEqual(expected.discovered_by, actual.discovered_by)
+ self.assertEqual(expected.description, actual.description)
+ self.assertEqual(
+ expected.ubuntu_description, actual.ubuntu_description
+ )
+ self.assertListEqual(expected.bug_urls, actual.bug_urls)
+ self.assertListEqual(expected.references, actual.references)
+ self.assertEqual(expected.notes, actual.notes)
+ self.assertEqual(expected.mitigation, actual.mitigation)
+ self.assertListEqual(expected.cvss, actual.cvss)
+
def test_create_bug(self):
bug = self.importer.create_bug(self.cve, self.lp_cve)
@@ -891,20 +983,13 @@ class TestUCTImporterExporter(TestCaseWithFactory):
self.importer.import_cve(self.cve)
bug = self.importer._find_existing_bug(self.cve, self.lp_cve)
cve = self.exporter._make_cve_from_bug(bug)
- self.assertEqual(self.cve.sequence, cve.sequence)
- self.assertEqual(self.cve.crd, cve.crd)
- self.assertEqual(self.cve.public_date, cve.public_date)
- self.assertEqual(self.cve.public_date_at_USN, cve.public_date_at_USN)
- self.assertListEqual(self.cve.distro_packages, cve.distro_packages)
- self.assertListEqual(self.cve.series_packages, cve.series_packages)
- self.assertEqual(self.cve.importance, cve.importance)
- self.assertEqual(self.cve.status, cve.status)
- self.assertEqual(self.cve.assignee, cve.assignee)
- self.assertEqual(self.cve.discovered_by, cve.discovered_by)
- self.assertEqual(self.cve.description, cve.description)
- self.assertEqual(self.cve.ubuntu_description, cve.ubuntu_description)
- self.assertListEqual(self.cve.bug_urls, cve.bug_urls)
- self.assertListEqual(self.cve.references, cve.references)
- self.assertEqual(self.cve.notes, cve.notes)
- self.assertEqual(self.cve.mitigation, cve.mitigation)
- self.assertListEqual(self.cve.cvss, cve.cvss)
+ self.checkCVE(self.cve, cve)
+
+ def test_export_bug_to_uct_file(self):
+ self.importer.import_cve(self.cve)
+ bug = self.importer._find_existing_bug(self.cve, self.lp_cve)
+ output_dir = Path(self.makeTemporaryDirectory())
+ cve_path = self.exporter.export_bug_to_uct_file(bug.id, output_dir)
+ uct_record = UCTRecord.load(cve_path)
+ cve = CVE.make_from_uct_record(uct_record)
+ self.checkCVE(self.cve, cve)
diff --git a/lib/lp/bugs/scripts/uct/__init__.py b/lib/lp/bugs/scripts/uct/__init__.py
index 970e1fd..da0635b 100644
--- a/lib/lp/bugs/scripts/uct/__init__.py
+++ b/lib/lp/bugs/scripts/uct/__init__.py
@@ -1,9 +1,6 @@
# Copyright 2022 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
-from lp.bugs.scripts.uct.models import CVE, CVSS, UCTRecord # noqa: F401
-from lp.bugs.scripts.uct.uctexport import UCTExporter # noqa: F401
-from lp.bugs.scripts.uct.uctimport import ( # noqa: F401
- UCTImporter,
- UCTImportError,
-)
+from .models import CVE, CVSS, UCTRecord
+from .uctexport import UCTExporter
+from .uctimport import UCTImporter, UCTImportError
diff --git a/lib/lp/bugs/scripts/uct/models.py b/lib/lp/bugs/scripts/uct/models.py
index 6c87f86..440f79d 100644
--- a/lib/lp/bugs/scripts/uct/models.py
+++ b/lib/lp/bugs/scripts/uct/models.py
@@ -2,7 +2,7 @@
# GNU Affero General Public License version 3 (see the file LICENSE).
import logging
-from collections import defaultdict
+from collections import OrderedDict, defaultdict
from datetime import datetime
from enum import Enum
from pathlib import Path
@@ -13,11 +13,12 @@ import dateutil.parser
from contrib.cve_lib import load_cve
from zope.component import getUtility
-from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.bugs.enums import VulnerabilityStatus
from lp.bugs.interfaces.bugtask import BugTaskImportance, BugTaskStatus
+from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.distroseries import IDistroSeriesSet
from lp.registry.interfaces.person import IPersonSet
+from lp.registry.interfaces.product import IProductSet
from lp.registry.interfaces.series import SeriesStatus
from lp.registry.interfaces.sourcepackagename import ISourcePackageNameSet
from lp.registry.model.distribution import Distribution
@@ -26,6 +27,7 @@ from lp.registry.model.distributionsourcepackage import (
)
from lp.registry.model.distroseries import DistroSeries
from lp.registry.model.person import Person
+from lp.registry.model.product import Product
from lp.registry.model.sourcepackage import SourcePackage
from lp.registry.model.sourcepackagename import SourcePackageName
from lp.services.propertycache import cachedproperty
@@ -73,10 +75,10 @@ class UCTRecord:
NEEDED = "needed"
PENDING = "pending"
- DistroSeriesPackageStatus = NamedTuple(
- "DistroSeriesPackageStatus",
+ SeriesPackageStatus = NamedTuple(
+ "SeriesPackageStatus",
(
- ("distroseries", str),
+ ("series", str),
("status", PackageStatus),
("reason", str),
("priority", Optional[Priority]),
@@ -95,7 +97,7 @@ class UCTRecord:
"Package",
(
("name", str),
- ("statuses", List[DistroSeriesPackageStatus]),
+ ("statuses", List[SeriesPackageStatus]),
("priority", Optional[Priority]),
("tags", Set[str]),
("patches", List[Patch]),
@@ -167,23 +169,23 @@ class UCTRecord:
cve_data, "pkgs"
).items():
statuses = []
- for distroseries, (status, reason) in statuses_dict.items():
- distroseries_priority = cls._pop_cve_property(
+ for series, (status, reason) in statuses_dict.items():
+ series_priority = cls._pop_cve_property(
cve_data,
- "Priority_{package}_{distroseries}".format(
+ "Priority_{package}_{series}".format(
package=package,
- distroseries=distroseries,
+ series=series,
),
required=False,
)
statuses.append(
- cls.DistroSeriesPackageStatus(
- distroseries=distroseries,
+ cls.SeriesPackageStatus(
+ series=series,
status=cls.PackageStatus(status),
reason=reason,
priority=(
- cls.Priority(distroseries_priority)
- if distroseries_priority
+ cls.Priority(series_priority)
+ if series_priority
else None
),
)
@@ -334,7 +336,7 @@ class UCTRecord:
)
for status in package.statuses:
self._write_field(
- "{}_{}".format(status.distroseries, package.name),
+ "{}_{}".format(status.series, package.name),
(
"{} ({})".format(status.status.value, status.reason)
if status.reason
@@ -351,9 +353,7 @@ class UCTRecord:
for status in package.statuses:
if status.priority:
self._write_field(
- "Priority_{}_{}".format(
- package.name, status.distroseries
- ),
+ "Priority_{}_{}".format(package.name, status.series),
status.priority.value,
output,
)
@@ -436,6 +436,16 @@ class CVE:
),
)
+ UpstreamPackage = NamedTuple(
+ "UpstreamPackage",
+ (
+ ("package", Product),
+ ("importance", Optional[BugTaskImportance]),
+ ("status", BugTaskStatus),
+ ("status_explanation", str),
+ ),
+ )
+
PRIORITY_MAP = {
UCTRecord.Priority.CRITICAL: BugTaskImportance.CRITICAL,
UCTRecord.Priority.HIGH: BugTaskImportance.HIGH,
@@ -478,6 +488,7 @@ class CVE:
public_date_at_USN: Optional[datetime],
distro_packages: List[DistroPackage],
series_packages: List[SeriesPackage],
+ upstream_packages: List[UpstreamPackage],
importance: BugTaskImportance,
status: VulnerabilityStatus,
assignee: Optional[Person],
@@ -496,6 +507,7 @@ class CVE:
self.public_date_at_USN = public_date_at_USN
self.distro_packages = distro_packages
self.series_packages = series_packages
+ self.upstream_packages = upstream_packages
self.importance = importance
self.status = status
self.assignee = assignee
@@ -518,6 +530,7 @@ class CVE:
distro_packages = []
series_packages = []
+ upstream_packages = []
spn_set = getUtility(ISourcePackageNameSet)
@@ -530,11 +543,6 @@ class CVE:
)
for uct_package_status in uct_package.statuses:
- distro_series = cls.get_distro_series(
- uct_package_status.distroseries
- )
- if distro_series is None:
- continue
if uct_package_status.status not in cls.BUG_TASK_STATUS_MAP:
logger.warning(
@@ -543,6 +551,34 @@ class CVE:
)
continue
+ series_package_importance = (
+ cls.PRIORITY_MAP[uct_package_status.priority]
+ if uct_package_status.priority
+ else None
+ )
+
+ if uct_package_status.series == "upstream":
+ product = cls.get_product(uct_package.name)
+ if product is None:
+ continue
+ upstream_packages.append(
+ cls.UpstreamPackage(
+ package=product,
+ importance=series_package_importance,
+ status=cls.BUG_TASK_STATUS_MAP[
+ uct_package_status.status
+ ],
+ status_explanation=uct_package_status.reason,
+ )
+ )
+ continue
+
+ distro_series = cls.get_distro_series(
+ uct_package_status.series
+ )
+ if distro_series is None:
+ continue
+
distro_package = cls.DistroPackage(
package=DistributionSourcePackage(
distribution=distro_series.distribution,
@@ -553,12 +589,6 @@ class CVE:
if distro_package not in distro_packages:
distro_packages.append(distro_package)
- series_package_importance = (
- cls.PRIORITY_MAP[uct_package_status.priority]
- if uct_package_status.priority
- else None
- )
-
series_packages.append(
cls.SeriesPackage(
package=SourcePackage(
@@ -589,6 +619,7 @@ class CVE:
public_date_at_USN=uct_record.public_date_at_USN,
distro_packages=distro_packages,
series_packages=series_packages,
+ upstream_packages=upstream_packages,
importance=cls.PRIORITY_MAP[uct_record.priority],
status=cls.infer_vulnerability_status(uct_record),
assignee=assignee,
@@ -616,23 +647,28 @@ class CVE:
series_package.package.sourcepackagename
].append(series_package)
- packages = [] # type: List[UCTRecord.Package]
+ packages_by_name = OrderedDict() # type: Dict[str, UCTRecord.Package]
processed_packages = set() # type: Set[SourcePackageName]
for distro_package in self.distro_packages:
spn = distro_package.package.sourcepackagename
if spn in processed_packages:
continue
processed_packages.add(spn)
- statuses = [] # type: List[UCTRecord.DistroSeriesPackageStatus]
+ statuses = [] # type: List[UCTRecord.SeriesPackageStatus]
for series_package in series_packages_by_name[spn]:
series = series_package.package.distroseries
if series.status == SeriesStatus.DEVELOPMENT:
series_name = "devel"
else:
series_name = series.name
+ distro_name = distro_package.package.distribution.name
+ if distro_name != "ubuntu":
+ if distro_name == "ubuntu-esm":
+ distro_name = "esm"
+ series_name = "{}/{}".format(series_name, distro_name)
statuses.append(
- UCTRecord.DistroSeriesPackageStatus(
- distroseries=series_name,
+ UCTRecord.SeriesPackageStatus(
+ series=series_name,
status=self.BUG_TASK_STATUS_MAP_REVERSE[
series_package.status
],
@@ -647,19 +683,43 @@ class CVE:
)
)
- packages.append(
- UCTRecord.Package(
- name=spn.name,
- statuses=statuses,
- priority=(
- self.PRIORITY_MAP_REVERSE[distro_package.importance]
- if distro_package.importance
- else None
- ),
+ packages_by_name[spn.name] = UCTRecord.Package(
+ name=spn.name,
+ statuses=statuses,
+ priority=(
+ self.PRIORITY_MAP_REVERSE[distro_package.importance]
+ if distro_package.importance
+ else None
+ ),
+ tags=set(),
+ patches=[],
+ )
+
+ for upstream_package in self.upstream_packages:
+ status = UCTRecord.SeriesPackageStatus(
+ series="upstream",
+ status=self.BUG_TASK_STATUS_MAP_REVERSE[
+ upstream_package.status
+ ],
+ reason=upstream_package.status_explanation,
+ priority=(
+ self.PRIORITY_MAP_REVERSE[upstream_package.importance]
+ if upstream_package.importance
+ else None
+ ),
+ )
+ package_name = upstream_package.package.name
+ if package_name in packages_by_name:
+ packages_by_name[package_name].statuses.append(status)
+ else:
+ packages_by_name[package_name] = UCTRecord.Package(
+ name=package_name,
+ statuses=[status],
+ priority=None,
tags=set(),
patches=[],
)
- )
+
return UCTRecord(
parent_dir=self.VULNERABILITY_STATUS_MAP_REVERSE.get(
self.status, ""
@@ -678,7 +738,7 @@ class CVE:
priority=self.PRIORITY_MAP_REVERSE[self.importance],
references=self.references,
ubuntu_description=self.ubuntu_description,
- packages=packages,
+ packages=list(packages_by_name.values()),
)
@property
@@ -722,20 +782,29 @@ class CVE:
if "/" in distro_series_name:
series_name, distro_name = distro_series_name.split("/", 1)
if distro_name == "esm":
- # TODO: ESM needs special handling
- pass
- return
+ distro_name = "ubuntu-esm"
else:
+ distro_name = "ubuntu"
series_name = distro_series_name
- distribution = getUtility(ILaunchpadCelebrities).ubuntu
- if series_name == "devel":
- distro_series = cls.get_devel_series(distribution)
- else:
- distro_series = getUtility(IDistroSeriesSet).queryByName(
- distribution, series_name
- )
+ distribution = getUtility(IDistributionSet).getByName(distro_name)
+ if distribution is None:
+ logger.warning("Could not find the distribution: %s", distro_name)
+ return
+ if series_name == "devel":
+ distro_series = cls.get_devel_series(distribution)
+ else:
+ distro_series = getUtility(IDistroSeriesSet).queryByName(
+ distribution, series_name
+ )
if not distro_series:
logger.warning(
"Could not find the distro series: %s", distro_series_name
)
return distro_series
+
+ @classmethod
+ def get_product(cls, product_name: str) -> Optional[Product]:
+ product = getUtility(IProductSet).getByName(product_name)
+ if not product:
+ logger.warning("Could not find the product: %s", product_name)
+ return product
diff --git a/lib/lp/bugs/scripts/uct/uctexport.py b/lib/lp/bugs/scripts/uct/uctexport.py
index 3e69302..45366a9 100644
--- a/lib/lp/bugs/scripts/uct/uctexport.py
+++ b/lib/lp/bugs/scripts/uct/uctexport.py
@@ -14,12 +14,14 @@ from lp.bugs.model.bug import Bug as BugModel
from lp.bugs.model.bugtask import BugTask
from lp.bugs.model.cve import Cve as CveModel
from lp.bugs.model.vulnerability import Vulnerability
-from lp.bugs.scripts.uct.models import CVE, CVSS
from lp.registry.model.distributionsourcepackage import (
DistributionSourcePackage,
)
+from lp.registry.model.product import Product
from lp.registry.model.sourcepackage import SourcePackage
+from .models import CVE, CVSS
+
__all__ = [
"UCTExporter",
]
@@ -70,7 +72,7 @@ class UCTExporter:
Create a `CVE` instances from a `Bug` model and the related
Vulnerabilities and `Cve`.
- `BugTasks` are converted to `CVE.DistroPackage` and `CVE.SeriesPackage`
+ `BugTasks` are converted to `CVE.DistroPackage` and `CVE.SEriesPackage`
objects.
Other `CVE` fields are populated from the information contained in the
@@ -141,6 +143,26 @@ class UCTExporter:
)
)
+ upstream_packages = []
+ for bug_task in bug_tasks:
+ target = removeSecurityProxy(bug_task.target)
+ if not isinstance(target, Product):
+ continue
+ up_importance = bug_task.importance
+ package_importance = package_importances.get(target.name)
+ upstream_packages.append(
+ CVE.UpstreamPackage(
+ package=target,
+ importance=(
+ up_importance
+ if up_importance != package_importance
+ else None
+ ),
+ status=bug_task.status,
+ status_explanation=bug_task.status_explanation,
+ )
+ )
+
return CVE(
sequence="CVE-{}".format(lp_cve.sequence),
crd=None, # TODO: fix this
@@ -148,6 +170,7 @@ class UCTExporter:
public_date_at_USN=None, # TODO: fix this
distro_packages=distro_packages,
series_packages=series_packages,
+ upstream_packages=upstream_packages,
importance=cve_importance,
status=vulnerability.status,
assignee=bug_tasks[0].assignee,
@@ -174,7 +197,7 @@ class UCTExporter:
Some `CVE` fields can't be mapped to Launchpad models.
They are saved to bug description.
- This method extracts those fields from the bug description.
+ This method extract those fields from the bug description.
:param bug_description: bug description
:return: parsed description
diff --git a/lib/lp/bugs/scripts/uct/uctimport.py b/lib/lp/bugs/scripts/uct/uctimport.py
index 3bc3512..13639b5 100644
--- a/lib/lp/bugs/scripts/uct/uctimport.py
+++ b/lib/lp/bugs/scripts/uct/uctimport.py
@@ -14,12 +14,21 @@ For each entry in UCT we:
3. Create a Bug Task for each distribution/series package in the CVE entry
4. Update the statuses of Bug Tasks based on the information in the CVE entry
5. Update the information the related Launchpad's `Cve` model, if necessary
+
+Three types of bug tags are created:
+
+1. Bug tasks with a distribution package as a target - they represent
+ importance of the package
+2. Bug tasks with distribution series packages as a target - they represent
+ importance and status of the package in a particular series
+3. Bug tasks with a product as a target - they represent importance and
+ status of the package in upstream.
"""
import logging
from datetime import timezone
from itertools import chain
from pathlib import Path
-from typing import List, Optional
+from typing import Dict, List, Optional
import transaction
from zope.component import getUtility
@@ -36,11 +45,12 @@ from lp.bugs.model.bug import Bug as BugModel
from lp.bugs.model.bugtask import BugTask
from lp.bugs.model.cve import Cve as CveModel
from lp.bugs.model.vulnerability import Vulnerability
-from lp.bugs.scripts.uct.models import CVE, UCTRecord
from lp.registry.model.distribution import Distribution
from lp.registry.model.person import Person
from lp.services.database.constants import UTC_NOW
+from .models import CVE, UCTRecord
+
__all__ = [
"UCTImporter",
"UCTImportError",
@@ -156,10 +166,17 @@ class UCTImporter:
self._update_external_bug_urls(bug, cve.bug_urls)
self._create_bug_tasks(
- bug, cve.distro_packages[1:], cve.series_packages
+ bug,
+ cve.distro_packages[1:],
+ cve.series_packages,
+ cve.upstream_packages,
)
self._update_statuses_and_importances(
- bug, cve.importance, cve.distro_packages, cve.series_packages
+ bug,
+ cve.importance,
+ cve.distro_packages,
+ cve.series_packages,
+ cve.upstream_packages,
)
self._assign_bug_tasks(bug, cve.assignee)
@@ -188,9 +205,18 @@ class UCTImporter:
"""
bug.description = self._make_bug_description(cve)
- self._create_bug_tasks(bug, cve.distro_packages, cve.series_packages)
+ self._create_bug_tasks(
+ bug,
+ cve.distro_packages,
+ cve.series_packages,
+ cve.upstream_packages,
+ )
self._update_statuses_and_importances(
- bug, cve.importance, cve.distro_packages, cve.series_packages
+ bug,
+ cve.importance,
+ cve.distro_packages,
+ cve.series_packages,
+ cve.upstream_packages,
)
self._assign_bug_tasks(bug, cve.assignee)
self._update_external_bug_urls(bug, cve.bug_urls)
@@ -228,6 +254,7 @@ class UCTImporter:
bug: BugModel,
distro_packages: List[CVE.DistroPackage],
series_packages: List[CVE.SeriesPackage],
+ upstream_packages: List[CVE.UpstreamPackage],
) -> None:
"""
Add bug tasks to the given `Bug` model based on the information
@@ -246,7 +273,8 @@ class UCTImporter:
bug_task_by_target = {t.target: t for t in bug_tasks}
bug_task_set = getUtility(IBugTaskSet)
for target in (
- p.package for p in chain(distro_packages, series_packages)
+ p.package
+ for p in chain(distro_packages, series_packages, upstream_packages)
):
if target not in bug_task_by_target:
bug_task_set.createTask(bug, self.bug_importer, target)
@@ -331,6 +359,7 @@ class UCTImporter:
cve_importance: BugTaskImportance,
distro_packages: List[CVE.DistroPackage],
series_packages: List[CVE.SeriesPackage],
+ upstream_packages: List[CVE.UpstreamPackage],
) -> None:
"""
Update statuses and importances of bug tasks according to the
@@ -350,23 +379,33 @@ class UCTImporter:
bug_tasks = bug.bugtasks # type: List[BugTask]
bug_task_by_target = {t.target: t for t in bug_tasks}
- package_importances = {}
+ package_importances = {} # type: Dict[str, BugTaskImportance]
for dp in distro_packages:
task = bug_task_by_target[dp.package]
dp_importance = dp.importance or cve_importance
- package_importances[dp.package.sourcepackagename] = dp_importance
- task.transitionToImportance(dp_importance)
+ package_importances[
+ dp.package.sourcepackagename.name
+ ] = dp_importance
+ if task.importance != dp_importance:
+ task.transitionToImportance(dp_importance, self.bug_importer)
- for sp in series_packages:
+ for sp in chain(series_packages, upstream_packages):
task = bug_task_by_target[sp.package]
- package_importance = package_importances[
- sp.package.sourcepackagename
- ]
+ if isinstance(sp, CVE.SeriesPackage):
+ package_name = sp.package.sourcepackagename.name
+ elif isinstance(sp, CVE.UpstreamPackage):
+ package_name = sp.package.name
+ else:
+ raise AssertionError()
+ package_importance = package_importances[package_name]
sp_importance = sp.importance or package_importance
- task.transitionToImportance(sp_importance)
- task.transitionToStatus(sp.status)
- task.status_explanation = sp.status_explanation
+ if task.importance != sp_importance:
+ task.transitionToImportance(sp_importance, self.bug_importer)
+ if task.status != sp.status:
+ task.transitionToStatus(sp.status, self.bug_importer)
+ if task.status_explanation != sp.status_explanation:
+ task.status_explanation = sp.status_explanation
def _update_external_bug_urls(
self, bug: BugModel, bug_urls: List[str]
diff --git a/lib/lp/registry/model/distributionsourcepackage.py b/lib/lp/registry/model/distributionsourcepackage.py
index 6d31eab..fd9da20 100644
--- a/lib/lp/registry/model/distributionsourcepackage.py
+++ b/lib/lp/registry/model/distributionsourcepackage.py
@@ -141,6 +141,9 @@ class DistributionSourcePackage(
self.distribution = distribution
self.sourcepackagename = sourcepackagename
+ def __repr__(self):
+ return "<{} '{}'>".format(self.__class__.__name__, self.display_name)
+
@property
def name(self):
"""See `IDistributionSourcePackage`."""
diff --git a/setup.cfg b/setup.cfg
index 367bb34..761ea15 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -209,6 +209,9 @@ ignore =
# operators, at least for now.
W503,
W504
+per-file-ignores =
+ # Ignore unused imports in `__init__` files
+ */__init__.py: F401
[isort]
# database/* have some implicit relative imports.
