launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #19468
[Merge] lp:~wgrant/launchpad/xref-model into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/xref-model into lp:launchpad with lp:~wgrant/launchpad/xref-db as a prerequisite.
Commit message:
Add lp.services.xref for generic cross-references between artifacts.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/xref-model/+merge/272588
Add lp.services.xref for generic cross-references between artifacts.
Schema is at <https://code.launchpad.net/~wgrant/launchpad/xref-db/+merge/272587>.
It's likely that we'll end up with wrappers that automatically resolve known objects to/from their tuple representations, but for now the by-ID API is surprisingly unonerous.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/xref-model into lp:launchpad.
=== modified file 'lib/lp/services/configure.zcml'
--- lib/lp/services/configure.zcml 2015-05-22 07:12:35 +0000
+++ lib/lp/services/configure.zcml 2015-09-28 12:44:55 +0000
@@ -34,4 +34,5 @@
<include package=".webhooks" />
<include package=".webservice" />
<include package=".worlddata" />
+ <include package=".xref" />
</configure>
=== added directory 'lib/lp/services/xref'
=== added file 'lib/lp/services/xref/__init__.py'
--- lib/lp/services/xref/__init__.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/__init__.py 2015-09-28 12:44:55 +0000
@@ -0,0 +1,13 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Generic cross references between artifacts.
+
+Provides infrastructure for generic information references between
+artifacts, easing weak coupling of apps.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = []
=== added file 'lib/lp/services/xref/configure.zcml'
--- lib/lp/services/xref/configure.zcml 1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/configure.zcml 2015-09-28 12:44:55 +0000
@@ -0,0 +1,13 @@
+<!-- Copyright 2015 Canonical Ltd. This software is licensed under the
+ GNU Affero General Public License version 3 (see the file LICENSE).
+-->
+
+<configure xmlns="http://namespaces.zope.org/zope";>
+
+ <securedutility
+ class="lp.services.xref.model.XRefSet"
+ provides="lp.services.xref.interfaces.IXRefSet">
+ <allow interface="lp.services.xref.interfaces.IXRefSet"/>
+ </securedutility>
+
+</configure>
=== added file 'lib/lp/services/xref/interfaces.py'
--- lib/lp/services/xref/interfaces.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/interfaces.py 2015-09-28 12:44:55 +0000
@@ -0,0 +1,62 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'IXRefSet',
+ ]
+
+from zope.interface import Interface
+
+
+class IXRefSet(Interface):
+ """Manager of cross-references between objects.
+
+ Each participant in an xref has an "object ID": a tuple of
+ (str type, str id).
+
+ All xrefs are currently between local objects, so links always exist
+ in both directions, but this can't be assumed to hold in future.
+ """
+
+ def create(xrefs):
+ """Create cross-references.
+
+ Back-links are automatically created.
+
+ :param xrefs: A dict of
+ {from_object_id: {to_object_id:
+ {'creator': `IPerson`, 'metadata': value}}}.
+ The creator and metadata keys are optional.
+ """
+
+ def findFromMany(object_ids, types=None):
+ """Find all cross-references from multiple objects.
+
+ :param object_ids: A collection of object IDs.
+ :param types: An optional collection of the types to include.
+ :return: A dict of
+ {from_object_id: {to_object_id:
+ {'creator': `IPerson`, 'metadata': value}}}.
+ The creator and metadata keys are optional.
+ """
+
+ def delete(xrefs):
+ """Delete cross-references.
+
+ Back-links are automatically deleted.
+
+ :param xrefs: A dict of {from_object_id: [to_object_id]}.
+ """
+
+ def findFrom(object_id, types=None):
+ """Find all cross-references from an object.
+
+ :param object_id: An object ID.
+ :param types: An optional collection of the types to include.
+ :return: A dict of
+ {to_object_id: {'creator': `IPerson`, 'metadata': value}}.
+ The creator and metadata keys are optional.
+ """
=== added file 'lib/lp/services/xref/model.py'
--- lib/lp/services/xref/model.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/model.py 2015-09-28 12:44:55 +0000
@@ -0,0 +1,131 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ "XRefSet",
+ ]
+
+import pytz
+from storm.expr import (
+ And,
+ Or,
+ )
+from storm.properties import (
+ DateTime,
+ Int,
+ JSON,
+ Unicode,
+ )
+from storm.references import Reference
+from zope.interface import implementer
+
+from lp.services.database import bulk
+from lp.services.database.constants import UTC_NOW
+from lp.services.database.interfaces import IStore
+from lp.services.database.stormbase import StormBase
+from lp.services.xref.interfaces import IXRefSet
+
+
+class XRef(StormBase):
+ """Cross-reference between two objects.
+
+ For references to local objects (there is currently no other kind),
+ another reference in the opposite direction exists.
+
+ The to_id_int and from_id_int columns exist for efficient SQL joins.
+ They are set automatically when the ID looks like an integer.
+ """
+
+ __storm_table__ = 'XRef'
+ __storm_primary__ = "to_type", "to_id", "from_type", "from_id"
+
+ to_type = Unicode(allow_none=False)
+ to_id = Unicode(allow_none=False)
+ to_id_int = Int() # For efficient joins.
+ from_type = Unicode(allow_none=False)
+ from_id = Unicode(allow_none=False)
+ from_id_int = Int() # For efficient joins.
+ creator_id = Int(name="creator")
+ creator = Reference(creator_id, "Person.id")
+ date_created = DateTime(name='date_created', tzinfo=pytz.UTC)
+ metadata = JSON()
+
+
+def _int_or_none(s):
+ if s.isdigit():
+ return int(s)
+ else:
+ return None
+
+
+@implementer(IXRefSet)
+class XRefSet:
+
+ def create(self, xrefs):
+ # All references are currently to local objects, so add
+ # backlinks as well to keep queries in both directions quick.
+ # The *_id_int columns are also set if the ID looks like an int.
+ rows = []
+ for from_, tos in xrefs.items():
+ for to, props in tos.items():
+ rows.append((
+ from_[0], from_[1], _int_or_none(from_[1]),
+ to[0], to[1], _int_or_none(to[1]),
+ props.get('creator'), props.get('date_created', UTC_NOW),
+ props.get('metadata')))
+ rows.append((
+ to[0], to[1], _int_or_none(to[1]),
+ from_[0], from_[1], _int_or_none(from_[1]),
+ props.get('creator'), props.get('date_created', UTC_NOW),
+ props.get('metadata')))
+ bulk.create(
+ (XRef.from_type, XRef.from_id, XRef.from_id_int,
+ XRef.to_type, XRef.to_id, XRef.to_id_int,
+ XRef.creator, XRef.date_created, XRef.metadata), rows)
+
+ def delete(self, xrefs):
+ # Delete both directions.
+ pairs = []
+ for from_, tos in xrefs.items():
+ for to in tos:
+ pairs.extend([(from_, to), (to, from_)])
+
+ IStore(XRef).find(
+ XRef,
+ Or(*[
+ And(XRef.from_type == pair[0][0],
+ XRef.from_id == pair[0][1],
+ XRef.to_type == pair[1][0],
+ XRef.to_id == pair[1][1])
+ for pair in pairs])
+ ).remove()
+
+ def findFromMany(self, object_ids, types=None):
+ from lp.registry.model.person import Person
+
+ object_ids = list(object_ids)
+ if not object_ids:
+ return {}
+
+ store = IStore(XRef)
+ rows = list(store.using(XRef).find(
+ (XRef.from_type, XRef.from_id, XRef.to_type, XRef.to_id,
+ XRef.creator_id, XRef.date_created, XRef.metadata),
+ Or(*[
+ And(XRef.from_type == id[0], XRef.from_id == id[1])
+ for id in object_ids]),
+ XRef.to_type.is_in(types) if types is not None else True))
+ bulk.load(Person, [row[4] for row in rows])
+ result = {}
+ for row in rows:
+ result.setdefault((row[0], row[1]), {})[(row[2], row[3])] = {
+ "creator": store.get(Person, row[4]) if row[4] else None,
+ "date_created": row[5],
+ "metadata": row[6]}
+ return result
+
+ def findFrom(self, object_id, types=None):
+ return self.findFromMany([object_id], types=types).get(object_id, {})
=== added directory 'lib/lp/services/xref/tests'
=== added file 'lib/lp/services/xref/tests/__init__.py'
--- lib/lp/services/xref/tests/__init__.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/tests/__init__.py 2015-09-28 12:44:55 +0000
@@ -0,0 +1,7 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = []
=== added file 'lib/lp/services/xref/tests/test_model.py'
--- lib/lp/services/xref/tests/test_model.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/tests/test_model.py 2015-09-28 12:44:55 +0000
@@ -0,0 +1,159 @@
+# Copyright 2015 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+import datetime
+
+import pytz
+from testtools.matchers import Equals
+from zope.component import getUtility
+
+from lp.services.database.interfaces import IStore
+from lp.services.xref.interfaces import IXRefSet
+from lp.services.xref.model import XRef
+from lp.testing import (
+ StormStatementRecorder,
+ TestCaseWithFactory,
+ )
+from lp.testing.layers import DatabaseFunctionalLayer
+from lp.testing.matchers import HasQueryCount
+
+
+class TestXRefSet(TestCaseWithFactory):
+
+ layer = DatabaseFunctionalLayer
+
+ def test_create_sets_date_created(self):
+ # date_created defaults to now, but can be overridden.
+ old = datetime.datetime.strptime('2005-01-01', '%Y-%m-%d').replace(
+ tzinfo=pytz.UTC)
+ now = IStore(XRef).execute(
+ "SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'"
+ ).get_one()[0].replace(tzinfo=pytz.UTC)
+ getUtility(IXRefSet).create({
+ ('a', '1'): {('b', 'foo'): {}},
+ ('a', '2'): {('b', 'bar'): {'date_created': old}}})
+ rows = IStore(XRef).find(
+ (XRef.from_id, XRef.to_id, XRef.date_created),
+ XRef.from_type == 'a')
+ self.assertContentEqual(
+ [('1', 'foo', now), ('2', 'bar', old)], rows)
+
+ def test_create_sets_int_columns(self):
+ # The string ID columns have integers equivalents for quick and
+ # easy joins to integer PKs. They're set automatically when the
+ # string ID looks like an integer.
+ getUtility(IXRefSet).create({
+ ('a', '1234'): {('b', 'foo'): {}, ('b', '2468'): {}},
+ ('a', '12ab'): {('b', '1234'): {}, ('b', 'foo'): {}}})
+ rows = IStore(XRef).find(
+ (XRef.from_type, XRef.from_id, XRef.from_id_int, XRef.to_type,
+ XRef.to_id, XRef.to_id_int),
+ XRef.from_type == 'a')
+ self.assertContentEqual(
+ [('a', '1234', 1234, 'b', 'foo', None),
+ ('a', '1234', 1234, 'b', '2468', 2468),
+ ('a', '12ab', None, 'b', '1234', 1234),
+ ('a', '12ab', None, 'b', 'foo', None)
+ ],
+ rows)
+
+ def test_findFrom(self):
+ creator = self.factory.makePerson()
+ now = IStore(XRef).execute(
+ "SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'"
+ ).get_one()[0].replace(tzinfo=pytz.UTC)
+ getUtility(IXRefSet).create({
+ ('a', 'bar'): {
+ ('b', 'foo'): {'creator': creator, 'metadata': {'test': 1}}},
+ ('b', 'foo'): {
+ ('a', 'baz'): {'creator': creator, 'metadata': {'test': 2}}},
+ })
+
+ with StormStatementRecorder() as recorder:
+ bar_refs = getUtility(IXRefSet).findFrom(('a', 'bar'))
+ self.assertThat(recorder, HasQueryCount(Equals(2)))
+ self.assertEqual(
+ {('b', 'foo'): {
+ 'creator': creator, 'date_created': now,
+ 'metadata': {'test': 1}}},
+ bar_refs)
+
+ with StormStatementRecorder() as recorder:
+ foo_refs = getUtility(IXRefSet).findFrom(('b', 'foo'))
+ self.assertThat(recorder, HasQueryCount(Equals(2)))
+ self.assertEqual(
+ {('a', 'bar'): {
+ 'creator': creator, 'date_created': now,
+ 'metadata': {'test': 1}},
+ ('a', 'baz'): {
+ 'creator': creator, 'date_created': now,
+ 'metadata': {'test': 2}}},
+ foo_refs)
+
+ with StormStatementRecorder() as recorder:
+ bar_refs = getUtility(IXRefSet).findFrom(('a', 'baz'))
+ self.assertThat(recorder, HasQueryCount(Equals(2)))
+ self.assertEqual(
+ {('b', 'foo'): {
+ 'creator': creator, 'date_created': now,
+ 'metadata': {'test': 2}}},
+ bar_refs)
+
+ with StormStatementRecorder() as recorder:
+ bar_baz_refs = getUtility(IXRefSet).findFromMany(
+ [('a', 'bar'), ('a', 'baz')])
+ self.assertThat(recorder, HasQueryCount(Equals(2)))
+ self.assertEqual(
+ {('a', 'bar'): {
+ ('b', 'foo'): {
+ 'creator': creator, 'date_created': now,
+ 'metadata': {'test': 1}}},
+ ('a', 'baz'): {
+ ('b', 'foo'): {
+ 'creator': creator, 'date_created': now,
+ 'metadata': {'test': 2}}}},
+ bar_baz_refs)
+
+ def test_findFrom_types(self):
+ # findFrom can look for only particular types of related
+ # objects.
+ getUtility(IXRefSet).create({
+ ('a', '1'): {('a', '2'): {}, ('b', '3'): {}},
+ ('b', '4'): {('a', '5'): {}, ('c', '6'): {}},
+ })
+ self.assertContentEqual(
+ [('a', '2')],
+ getUtility(IXRefSet).findFrom(('a', '1'), types=['a', 'c']).keys())
+ self.assertContentEqual(
+ [('a', '5'), ('c', '6')],
+ getUtility(IXRefSet).findFrom(('b', '4'), types=['a', 'c']).keys())
+
+ # Asking for no types or types that don't exist finds nothing.
+ self.assertContentEqual(
+ [],
+ getUtility(IXRefSet).findFrom(('b', '4'), types=[]).keys())
+ self.assertContentEqual(
+ [],
+ getUtility(IXRefSet).findFrom(('b', '4'), types=['d']).keys())
+
+ def test_findFromMany_none(self):
+ self.assertEqual({}, getUtility(IXRefSet).findFromMany([]))
+
+ def test_delete(self):
+ getUtility(IXRefSet).create({
+ ('a', 'bar'): {('b', 'foo'): {}},
+ ('b', 'foo'): {('a', 'baz'): {}},
+ })
+ self.assertContentEqual(
+ [('a', 'bar'), ('a', 'baz')],
+ getUtility(IXRefSet).findFrom(('b', 'foo')).keys())
+ with StormStatementRecorder() as recorder:
+ getUtility(IXRefSet).delete({('b', 'foo'): [('a', 'bar')]})
+ self.assertThat(recorder, HasQueryCount(Equals(1)))
+ self.assertEqual(
+ [('a', 'baz')],
+ getUtility(IXRefSet).findFrom(('b', 'foo')).keys())
=== modified file 'lib/lp/testing/tests/test_standard_test_template.py'
--- lib/lp/testing/tests/test_standard_test_template.py 2015-01-30 10:13:51 +0000
+++ lib/lp/testing/tests/test_standard_test_template.py 2015-09-28 12:44:55 +0000
@@ -3,6 +3,8 @@
"""XXX: Module docstring goes here."""
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
# or TestCaseWithFactory
=== modified file 'lib/lp/testing/tests/test_standard_yuixhr_test_template.py'
--- lib/lp/testing/tests/test_standard_yuixhr_test_template.py 2015-01-30 10:13:51 +0000
+++ lib/lp/testing/tests/test_standard_yuixhr_test_template.py 2015-09-28 12:44:55 +0000
@@ -4,6 +4,8 @@
"""{Describe your test suite here}.
"""
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
__all__ = []
=== modified file 'standard_template.py'
--- standard_template.py 2015-01-30 10:13:51 +0000
+++ standard_template.py 2015-09-28 12:44:55 +0000
@@ -3,5 +3,7 @@
"""XXX: Module docstring goes here."""
+from __future__ import absolute_import, print_function, unicode_literals
+
__metaclass__ = type
__all__ = []
Follow ups
