launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #25131
[Merge] ~cjwatson/launchpad:stormify-cve into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:stormify-cve into launchpad:master.
Commit message:
Convert Cve and CveReference to Storm
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/388990
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:stormify-cve into launchpad:master.
diff --git a/lib/lp/bugs/browser/tests/test_cve.py b/lib/lp/bugs/browser/tests/test_cve.py
index ee760e3..8c4f024 100644
--- a/lib/lp/bugs/browser/tests/test_cve.py
+++ b/lib/lp/bugs/browser/tests/test_cve.py
@@ -1,8 +1,10 @@
-# Copyright 2012-2018 Canonical Ltd. This software is licensed under the
+# Copyright 2012-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""CVE related tests."""
+from __future__ import absolute_import, print_function, unicode_literals
+
from functools import partial
from operator import attrgetter
import re
diff --git a/lib/lp/bugs/doc/cve-update.txt b/lib/lp/bugs/doc/cve-update.txt
index 542e457..5ae2697 100644
--- a/lib/lp/bugs/doc/cve-update.txt
+++ b/lib/lp/bugs/doc/cve-update.txt
@@ -21,11 +21,13 @@ the case.
>>> import transaction
>>> from lp.services.config import config
-OK. So now lets import the first XML database. First, lets se how many CVE
-entries are in the database.
+OK. So now let's import the first XML database. First, let's see how many
+CVE entries are in the database.
- >>> from lp.bugs.model.cve import Cve
- >>> print(Cve.select().count())
+ >>> from zope.component import getUtility
+ >>> from lp.bugs.interfaces.cve import ICveSet
+ >>> cve_set = getUtility(ICveSet)
+ >>> print(cve_set.getAll().count())
10
>>> script = os.path.join(config.root, 'cronscripts', 'update-cve.py')
@@ -81,14 +83,14 @@ Now run the cronscript.
And let's make sure we got the right number of CVE entries.
>>> transaction.commit()
- >>> print(Cve.select().count())
+ >>> print(cve_set.getAll().count())
18
We will make a note of the CVE modification time of 1999-0002. When we
update it later, we can use this modification time to check that its
modification time is being updated correctly.
- >>> c = Cve.bySequence('2005-2734')
+ >>> c = cve_set['2005-2734']
>>> mod_time = c.datemodified
And while we are here, make a note of the number of references for that CVE
@@ -137,13 +139,14 @@ Now, let's run an import of the update db.
Let's make sure we got the new CVE's.
>>> transaction.commit()
- >>> print(Cve.select().count())
+ >>> print(cve_set.getAll().count())
21
And let's make sure the modification time of 2005-2734 was updated, as were
the number of comments.
- >>> c.sync()
+ >>> from storm.store import Store
+ >>> Store.of(c).autoreload()
>>> print(mod_time < c.datemodified)
True
>>> print(c.references.count())
diff --git a/lib/lp/bugs/mail/tests/test_commands.py b/lib/lp/bugs/mail/tests/test_commands.py
index fe6d796..00721f8 100644
--- a/lib/lp/bugs/mail/tests/test_commands.py
+++ b/lib/lp/bugs/mail/tests/test_commands.py
@@ -1,6 +1,8 @@
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
+from __future__ import absolute_import, print_function, unicode_literals
+
from lazr.lifecycle.interfaces import (
IObjectCreatedEvent,
IObjectModifiedEvent,
diff --git a/lib/lp/bugs/model/cve.py b/lib/lp/bugs/model/cve.py
index d5329ff..05b5fa8 100644
--- a/lib/lp/bugs/model/cve.py
+++ b/lib/lp/bugs/model/cve.py
@@ -1,6 +1,8 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
__all__ = [
@@ -10,13 +12,16 @@ __all__ = [
import operator
-from sqlobject import (
- SQLMultipleJoin,
- SQLObjectNotFound,
- StringCol,
+import pytz
+import six
+from storm.locals import (
+ DateTime,
+ Desc,
+ Int,
+ ReferenceSet,
+ Store,
+ Unicode,
)
-from storm.expr import In
-from storm.store import Store
from zope.component import getUtility
from zope.interface import implementer
@@ -35,29 +40,36 @@ from lp.bugs.model.buglinktarget import BugLinkTargetMixin
from lp.bugs.model.cvereference import CveReference
from lp.services.database import bulk
from lp.services.database.constants import UTC_NOW
-from lp.services.database.datetimecol import UtcDateTimeCol
-from lp.services.database.enumcol import EnumCol
+from lp.services.database.enumcol import DBEnum
from lp.services.database.interfaces import IStore
-from lp.services.database.sqlbase import SQLBase
+from lp.services.database.stormbase import StormBase
from lp.services.database.stormexpr import fti_search
from lp.services.xref.interfaces import IXRefSet
from lp.services.xref.model import XRef
@implementer(ICve, IBugLinkTarget)
-class Cve(SQLBase, BugLinkTargetMixin):
+class Cve(StormBase, BugLinkTargetMixin):
"""A CVE database record."""
- _table = 'Cve'
+ __storm_table__ = 'Cve'
+
+ id = Int(primary=True)
- sequence = StringCol(notNull=True, alternateID=True)
- status = EnumCol(dbName='status', schema=CveStatus, notNull=True)
- description = StringCol(notNull=True)
- datecreated = UtcDateTimeCol(notNull=True, default=UTC_NOW)
- datemodified = UtcDateTimeCol(notNull=True, default=UTC_NOW)
+ sequence = Unicode(allow_none=False)
+ status = DBEnum(name='status', enum=CveStatus, allow_none=False)
+ description = Unicode(allow_none=False)
+ datecreated = DateTime(tzinfo=pytz.UTC, allow_none=False, default=UTC_NOW)
+ datemodified = DateTime(tzinfo=pytz.UTC, allow_none=False, default=UTC_NOW)
- references = SQLMultipleJoin(
- 'CveReference', joinColumn='cve', orderBy='id')
+ references = ReferenceSet(
+ id, 'CveReference.cve_id', order_by='CveReference.id')
+
+ def __init__(self, sequence, status, description):
+ super(Cve, self).__init__()
+ self.sequence = sequence
+ self.status = status
+ self.description = description
@property
def url(self):
@@ -89,7 +101,7 @@ class Cve(SQLBase, BugLinkTargetMixin):
def removeReference(self, ref):
assert ref.cve == self
- CveReference.delete(ref.id)
+ Store.of(ref).remove(ref)
def createBugLink(self, bug, props=None):
"""See BugLinkTargetMixin."""
@@ -97,18 +109,18 @@ class Cve(SQLBase, BugLinkTargetMixin):
props = {}
# XXX: Should set creator.
getUtility(IXRefSet).create(
- {(u'cve', self.sequence): {(u'bug', unicode(bug.id)): props}})
+ {(u'cve', self.sequence): {
+ (u'bug', six.text_type(bug.id)): props}})
def deleteBugLink(self, bug):
"""See BugLinkTargetMixin."""
getUtility(IXRefSet).delete(
- {(u'cve', self.sequence): [(u'bug', unicode(bug.id))]})
+ {(u'cve', self.sequence): [(u'bug', six.text_type(bug.id))]})
@implementer(ICveSet)
class CveSet:
"""The full set of ICve's."""
- table = Cve
def __init__(self, bug=None):
"""See ICveSet."""
@@ -120,40 +132,42 @@ class CveSet:
sequence = sequence[4:]
if not valid_cve(sequence):
return None
- try:
- return Cve.bySequence(sequence)
- except SQLObjectNotFound:
- return None
+ return IStore(Cve).find(Cve, sequence=sequence).one()
def getAll(self):
"""See ICveSet."""
- return Cve.select(orderBy="-datemodified")
+ return IStore(Cve).find(Cve).order_by(Desc(Cve.datemodified))
def __iter__(self):
"""See ICveSet."""
- return iter(Cve.select())
+ return iter(IStore(Cve).find(Cve))
def new(self, sequence, description, status=CveStatus.CANDIDATE):
"""See ICveSet."""
- return Cve(sequence=sequence, status=status,
+ cve = Cve(sequence=sequence, status=status,
description=description)
+ IStore(Cve).add(cve)
+ return cve
def latest(self, quantity=5):
"""See ICveSet."""
- return Cve.select(orderBy='-datecreated', limit=quantity)
+ return IStore(Cve).find(Cve).order_by(
+ Desc(Cve.datecreated)).config(limit=quantity)
def latest_modified(self, quantity=5):
"""See ICveSet."""
- return Cve.select(orderBy='-datemodified', limit=quantity)
+ return IStore(Cve).find(Cve).order_by(
+ Desc(Cve.datemodified)).config(limit=quantity)
def search(self, text):
"""See ICveSet."""
- return Cve.select(
- fti_search(Cve, text), distinct=True, orderBy='-datemodified')
+ return IStore(Cve).find(Cve, fti_search(Cve, text)).order_by(
+ Desc(Cve.datemodified)).config(distinct=True)
def inText(self, text):
"""See ICveSet."""
# let's look for matching entries
+ store = IStore(Cve)
cves = set()
for match in CVEREF_PATTERN.finditer(text):
# let's get the core CVE data
@@ -168,6 +182,7 @@ class CveSet:
"are reading this, then this CVE entry is probably "
"erroneous, since this text should be replaced by "
"the official CVE description automatically.")
+ store.add(cve)
cves.add(cve)
return sorted(cves, key=lambda a: a.sequence)
@@ -194,7 +209,7 @@ class CveSet:
store = Store.of(bugtasks[0])
xrefs = getUtility(IXRefSet).findFromMany(
- [(u'bug', unicode(bug.id)) for bug in bugs], types=[u'cve'])
+ [(u'bug', six.text_type(bug.id)) for bug in bugs], types=[u'cve'])
bugcve_ids = set()
for bug_key in xrefs:
for cve_key in xrefs[bug_key]:
@@ -203,7 +218,7 @@ class CveSet:
bugcve_ids = list(sorted(bugcve_ids))
cves = store.find(
- Cve, In(Cve.sequence, [seq for _, seq in bugcve_ids]))
+ Cve, Cve.sequence.is_in([seq for _, seq in bugcve_ids]))
if cve_mapper is None:
cvemap = dict((cve.sequence, cve) for cve in cves)
diff --git a/lib/lp/bugs/model/cvereference.py b/lib/lp/bugs/model/cvereference.py
index afc357a..129b5a1 100644
--- a/lib/lp/bugs/model/cvereference.py
+++ b/lib/lp/bugs/model/cvereference.py
@@ -1,27 +1,37 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
__all__ = ['CveReference']
-from sqlobject import (
- ForeignKey,
- StringCol,
+from storm.locals import (
+ Int,
+ Reference,
+ Unicode,
)
from zope.interface import implementer
from lp.bugs.interfaces.cvereference import ICveReference
-from lp.services.database.sqlbase import SQLBase
+from lp.services.database.stormbase import StormBase
@implementer(ICveReference)
-class CveReference(SQLBase):
+class CveReference(StormBase):
"""A CVE reference to some other tracking system."""
- _table = 'CveReference'
+ __storm_table__ = 'CveReference'
- # db field names
- cve = ForeignKey(dbName='cve', foreignKey='Cve', notNull=True)
- source = StringCol(notNull=True)
- content = StringCol(notNull=True)
- url = StringCol(notNull=False, default=None)
+ id = Int(primary=True)
+
+ cve_id = Int(name='cve', allow_none=False)
+ cve = Reference(cve_id, 'Cve.id')
+ source = Unicode(allow_none=False)
+ content = Unicode(allow_none=False)
+ url = Unicode(allow_none=True, default=None)
+
+ def __init__(self, cve, source, content, url=None):
+ super(CveReference, self).__init__()
+ self.cve = cve
+ self.source = source
+ self.content = content
+ self.url = url
diff --git a/lib/lp/bugs/model/tests/test_bugtasksearch.py b/lib/lp/bugs/model/tests/test_bugtasksearch.py
index 60524b8..b5455f6 100644
--- a/lib/lp/bugs/model/tests/test_bugtasksearch.py
+++ b/lib/lp/bugs/model/tests/test_bugtasksearch.py
@@ -1,6 +1,8 @@
-# Copyright 2010-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
from datetime import (
@@ -11,6 +13,7 @@ from operator import attrgetter
import unittest
import pytz
+import six
from storm.expr import Or
from testtools.matchers import Equals
from testtools.testcase import ExpectedException
@@ -2494,7 +2497,8 @@ def test_suite():
loader = unittest.TestLoader()
for bug_target_search_type_class in (
PreloadBugtaskTargets, NoPreloadBugtaskTargets, QueryBugIDs):
- class_name = 'Test%s' % bug_target_search_type_class.__name__
+ class_name = six.ensure_str(
+ 'Test%s' % bug_target_search_type_class.__name__)
class_bases = (
bug_target_search_type_class, ProductTarget, OnceTests,
SearchTestBase, TestCaseWithFactory)
@@ -2502,9 +2506,10 @@ def test_suite():
suite.addTest(loader.loadTestsFromTestCase(test_class))
for target_mixin in bug_targets_mixins:
- class_name = 'Test%s%s' % (
- bug_target_search_type_class.__name__,
- target_mixin.__name__)
+ class_name = six.ensure_str(
+ 'Test%s%s' % (
+ bug_target_search_type_class.__name__,
+ target_mixin.__name__))
mixins = [
target_mixin, bug_target_search_type_class]
class_bases = (
diff --git a/lib/lp/bugs/scripts/cveimport.py b/lib/lp/bugs/scripts/cveimport.py
index b202974..3730fef 100644
--- a/lib/lp/bugs/scripts/cveimport.py
+++ b/lib/lp/bugs/scripts/cveimport.py
@@ -1,10 +1,12 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""A set of functions related to the ability to parse the XML CVE database,
extract details of known CVE entries, and ensure that all of the known
CVE's are fully registered in Launchpad."""
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
import gzip
@@ -13,6 +15,7 @@ import time
import defusedxml.cElementTree as cElementTree
import requests
+import six
from zope.component import getUtility
from zope.event import notify
from zope.interface import implementer
@@ -42,11 +45,11 @@ CVEDB_NS = '{http://cve.mitre.org/cve/downloads/1.0}'
def getText(elem):
"""Get the text content of the given element"""
- text = elem.text or ""
+ text = six.ensure_text(elem.text or "")
for e in elem:
text += getText(e)
if e.tail:
- text += e.tail
+ text += six.ensure_text(e.tail)
return text.strip()
@@ -71,8 +74,10 @@ def handle_references(cve_node, cve, log):
# work through the refs in the xml dump
for ref_node in cve_node.findall('.//%sref' % CVEDB_NS):
- refsrc = ref_node.get("source")
+ refsrc = six.ensure_text(ref_node.get("source"))
refurl = ref_node.get("url")
+ if refurl is not None:
+ refurl = six.ensure_text(refurl)
reftxt = getText(ref_node)
# compare it to each of the known references
was_there_previously = False
@@ -105,9 +110,9 @@ def handle_references(cve_node, cve, log):
def update_one_cve(cve_node, log):
"""Update the state of a single CVE item."""
# get the sequence number
- sequence = cve_node.get('seq')
+ sequence = six.ensure_text(cve_node.get('seq'))
# establish its status
- status = cve_node.get('type')
+ status = six.ensure_text(cve_node.get('type'))
# get the description
description = getText(cve_node.find(CVEDB_NS + 'desc'))
if not description:
diff --git a/lib/lp/bugs/tests/test_bug.py b/lib/lp/bugs/tests/test_bug.py
index fd833b8..2d76d10 100644
--- a/lib/lp/bugs/tests/test_bug.py
+++ b/lib/lp/bugs/tests/test_bug.py
@@ -1,8 +1,10 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Tests for lp.bugs.model.Bug."""
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
from datetime import timedelta
diff --git a/lib/lp/bugs/tests/test_bugchanges.py b/lib/lp/bugs/tests/test_bugchanges.py
index 48c74b0..7312716 100644
--- a/lib/lp/bugs/tests/test_bugchanges.py
+++ b/lib/lp/bugs/tests/test_bugchanges.py
@@ -1,8 +1,10 @@
-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Tests for recording changes done to a bug."""
+from __future__ import absolute_import, print_function, unicode_literals
+
from lazr.lifecycle.event import ObjectCreatedEvent
from testtools.matchers import (
MatchesStructure,
diff --git a/lib/lp/bugs/tests/test_buglinktarget.py b/lib/lp/bugs/tests/test_buglinktarget.py
index 2286ffb..ccccc38 100644
--- a/lib/lp/bugs/tests/test_buglinktarget.py
+++ b/lib/lp/bugs/tests/test_buglinktarget.py
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Test harness for running the buglinktarget.txt interface test
@@ -7,6 +7,8 @@ This module will run the interface test against the CVE, Specification,
Question, and BranchMergeProposal implementations of that interface.
"""
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
__all__ = []
diff --git a/lib/lp/bugs/tests/test_cve.py b/lib/lp/bugs/tests/test_cve.py
index 52cfd44..60d2d61 100644
--- a/lib/lp/bugs/tests/test_cve.py
+++ b/lib/lp/bugs/tests/test_cve.py
@@ -1,8 +1,10 @@
-# Copyright 2012 Canonical Ltd. This software is licensed under the
+# Copyright 2012-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""CVE related tests."""
+from __future__ import absolute_import, print_function, unicode_literals
+
from zope.component import getUtility
from lp.bugs.interfaces.bugtasksearch import BugTaskSearchParams
diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
index 4d93342..4d92b7e 100644
--- a/lib/lp/testing/factory.py
+++ b/lib/lp/testing/factory.py
@@ -4578,7 +4578,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
cvestate=CveStatus.CANDIDATE):
"""Create a new CVE record."""
if description is None:
- description = self.getUniqueString()
+ description = self.getUniqueUnicode()
return getUtility(ICveSet).new(sequence, description, cvestate)
def makePublisherConfig(self, distribution=None, root_dir=None,
diff --git a/lib/lp/testing/tests/test_factory.py b/lib/lp/testing/tests/test_factory.py
index 46db466..341a990 100644
--- a/lib/lp/testing/tests/test_factory.py
+++ b/lib/lp/testing/tests/test_factory.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2020 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Tests for the Launchpad object factory."""
@@ -630,20 +630,20 @@ class TestFactory(TestCaseWithFactory):
# makeCVE
def test_makeCVE_returns_cve(self):
- cve = self.factory.makeCVE(sequence='2000-1234')
+ cve = self.factory.makeCVE(sequence=u'2000-1234')
self.assertThat(cve, ProvidesAndIsProxied(ICve))
def test_makeCVE_uses_sequence(self):
- cve = self.factory.makeCVE(sequence='2000-1234')
- self.assertEqual('2000-1234', cve.sequence)
+ cve = self.factory.makeCVE(sequence=u'2000-1234')
+ self.assertEqual(u'2000-1234', cve.sequence)
def test_makeCVE_uses_description(self):
- cve = self.factory.makeCVE(sequence='2000-1234', description='foo')
- self.assertEqual('foo', cve.description)
+ cve = self.factory.makeCVE(sequence=u'2000-1234', description=u'foo')
+ self.assertEqual(u'foo', cve.description)
def test_makeCVE_uses_cve_status(self):
cve = self.factory.makeCVE(
- sequence='2000-1234', cvestate=CveStatus.DEPRECATED)
+ sequence=u'2000-1234', cvestate=CveStatus.DEPRECATED)
self.assertEqual(CveStatus.DEPRECATED, cve.status)
# dir() support.
