launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #22324
[Merge] lp:~maxiberta/launchpad/bing-search into lp:launchpad
Maximiliano Bertacchini has proposed merging lp:~maxiberta/launchpad/bing-search into lp:launchpad.
Commit message:
Add basic Bing Custom Search site search support.
Requested reviews:
Colin Watson (cjwatson)
For more details, see:
https://code.launchpad.net/~maxiberta/launchpad/bing-search/+merge/341549
Add basic Bing Custom Search support.
Should be pretty unobtrusive, while adding a basic site search implementation around Bing Custom Search (shamelessly copied from the Google site search implementation). Might need some more testing; and could use some code deduplication.
--
Your team Launchpad code reviewers is subscribed to branch lp:launchpad.
=== modified file 'Makefile'
--- Makefile 2018-02-01 20:56:23 +0000
+++ Makefile 2018-03-28 21:29:05 +0000
@@ -61,6 +61,7 @@
# NB: It's important PIP_BIN only mentions things genuinely produced by pip.
PIP_BIN = \
$(PY) \
+ bin/bingtestservice \
bin/build-twisted-plugin-cache \
bin/combine-css \
bin/googletestservice \
@@ -284,7 +285,7 @@
bin/test -f $(TESTFLAGS) $(TESTOPTS)
run: build inplace stop
- bin/run -r librarian,google-webservice,memcached,rabbitmq,txlongpoll \
+ bin/run -r librarian,bing-webservice,google-webservice,memcached,rabbitmq,txlongpoll \
-i $(LPCONFIG)
run-testapp: LPCONFIG=testrunner-appserver
@@ -297,12 +298,12 @@
start-gdb: build inplace stop support_files run.gdb
nohup gdb -x run.gdb --args bin/run -i $(LPCONFIG) \
- -r librarian,google-webservice
+ -r librarian,bing-webservice,google-webservice
> ${LPCONFIG}-nohup.out 2>&1 &
run_all: build inplace stop
bin/run \
- -r librarian,sftp,forker,mailman,codebrowse,google-webservice,\
+ -r librarian,sftp,forker,mailman,codebrowse,bing-webservice,google-webservice,\
memcached,rabbitmq,txlongpoll -i $(LPCONFIG)
run_codebrowse: compile
=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf 2018-02-02 15:29:38 +0000
+++ configs/development/launchpad-lazr.conf 2018-03-28 21:29:05 +0000
@@ -79,13 +79,22 @@
error_dir: /var/tmp/lperr
[google]
-# Development and the testrunner should use the stub service be default.
+# Development and the testrunner should use the stub service by default.
site: http://launchpad.dev:8092/cse
client_id: ABCDEF2323
[google_test_service]
launch: True
+[bing]
+# Development and the testrunner should use the stub service by default.
+site: http://launchpad.dev:8093/bingcustomsearch/v7.0/search
+subscription_key: abcdef01234567890abcdef012345678
+custom_config_id: 1234567890
+
+[bing_test_service]
+launch: True
+
[gpghandler]
host: keyserver.launchpad.dev
public_host: keyserver.launchpad.dev
=== modified file 'configs/testrunner-appserver/launchpad-lazr.conf'
--- configs/testrunner-appserver/launchpad-lazr.conf 2016-10-11 15:28:25 +0000
+++ configs/testrunner-appserver/launchpad-lazr.conf 2018-03-28 21:29:05 +0000
@@ -14,6 +14,9 @@
[google_test_service]
launch: False
+[bing_test_service]
+launch: False
+
[launchpad]
openid_provider_root: http://testopenid.dev:8085/
=== modified file 'configs/testrunner/launchpad-lazr.conf'
--- configs/testrunner/launchpad-lazr.conf 2018-01-26 22:18:38 +0000
+++ configs/testrunner/launchpad-lazr.conf 2018-03-28 21:29:05 +0000
@@ -91,6 +91,9 @@
[google]
site: http://launchpad.dev:8092/cse
+[bing]
+site: http://launchpad.dev:8093/bingcustomsearch/v7.0/search
+
[gpghandler]
upload_keys: True
host: localhost
=== added directory 'lib/lp/app/browser/doc'
=== added directory 'lib/lp/app/browser/doc.moved'
=== renamed file 'lib/lp/app/browser/tests/base-layout.txt' => 'lib/lp/app/browser/doc/base-layout.txt'
--- lib/lp/app/browser/tests/base-layout.txt 2016-03-14 00:45:42 +0000
+++ lib/lp/app/browser/doc/base-layout.txt 2018-03-28 21:29:05 +0000
@@ -25,7 +25,7 @@
>>> class MainSideView(LaunchpadView):
... """A simple view to test base-layout."""
... __launchpad_facetname__ = 'overview'
- ... template = ViewPageTemplateFile('testfiles/main-side.pt')
+ ... template = ViewPageTemplateFile('../tests/testfiles/main-side.pt')
... page_title = 'Test base-layout: main_side'
The main_side layout uses all the defined features of the base-layout
@@ -57,7 +57,7 @@
>>> class MainOnlyView(LaunchpadView):
... """A simple view to test base-layout."""
... __launchpad_facetname__ = 'overview'
- ... template = ViewPageTemplateFile('testfiles/main-only.pt')
+ ... template = ViewPageTemplateFile('../tests/testfiles/main-only.pt')
... page_title = 'Test base-layout: main_only'
>>> view = MainOnlyView(user, request)
@@ -80,7 +80,7 @@
>>> class SearchlessView(LaunchpadView):
... """A simple view to test base-layout."""
... __launchpad_facetname__ = 'overview'
- ... template = ViewPageTemplateFile('testfiles/searchless.pt')
+ ... template = ViewPageTemplateFile('../tests/testfiles/searchless.pt')
... page_title = 'Test base-layout: searchless'
>>> view = SearchlessView(user, request)
=== added file 'lib/lp/app/browser/doc/launchpad-search-pages-bing.txt'
--- lib/lp/app/browser/doc/launchpad-search-pages-bing.txt 1970-01-01 00:00:00 +0000
+++ lib/lp/app/browser/doc/launchpad-search-pages-bing.txt 2018-03-28 21:29:05 +0000
@@ -0,0 +1,725 @@
+Launchpad search page
+=====================
+
+Users can search for Launchpad objects and pages from the search form
+located on all pages. The search is performed and displayed by the
+LaunchpadSearchView.
+
+ >>> from zope.component import getMultiAdapter, getUtility
+ >>> from lp.services.webapp.interfaces import ILaunchpadRoot
+ >>> from lp.services.webapp.servers import LaunchpadTestRequest
+
+ >>> root = getUtility(ILaunchpadRoot)
+ >>> request = LaunchpadTestRequest()
+ >>> search_view = getMultiAdapter((root, request), name="+search")
+ >>> search_view.initialize()
+ >>> search_view
+ <....SimpleViewClass from .../templates/launchpad-search.pt ...>
+
+
+Page title and heading
+----------------------
+
+The page title and heading suggest to the user to search launchpad
+when there is no search text.
+
+ >>> print search_view.text
+ None
+ >>> search_view.page_title
+ 'Search Launchpad'
+ >>> search_view.page_heading
+ 'Search Launchpad'
+
+When text is not None, the title indicates what was searched.
+
+ >>> def getSearchView(form):
+ ... search_param_list = []
+ ... for name in sorted(form):
+ ... value = form[name]
+ ... search_param_list.append('%s=%s' % (name, value))
+ ... query_string = '&'.join(search_param_list)
+ ... request = LaunchpadTestRequest(
+ ... SERVER_URL='https://launchpad.dev/+search',
+ ... QUERY_STRING=query_string, form=form, PATH_INFO='/+search')
+ ... search_view = getMultiAdapter((root, request), name="+search")
+ ... search_view.initialize()
+ ... return search_view
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'albatross'})
+
+ >>> search_view.text
+ u'albatross'
+ >>> search_view.page_title
+ u'Pages matching "albatross" in Launchpad'
+ >>> search_view.page_heading
+ u'Pages matching "albatross" in Launchpad'
+
+
+No matches
+----------
+
+There were no matches for 'albatross'.
+
+ >>> search_view.has_matches
+ False
+
+When search text is not submitted there are no matches. Search text is
+required to perform a search. Note that field.actions.search is not a
+required param to call the Search Action. The view always calls the
+search action.
+
+ >>> search_view = getSearchView(form={})
+
+ >>> print search_view.text
+ None
+ >>> search_view.has_matches
+ False
+
+
+Bug and Question Searches
+-------------------------
+
+When a numeric token can be extracted from the submitted search text,
+the view tries to match a bug and question. Bugs and questions are
+matched by their id.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': '5'})
+ >>> search_view._getNumericToken(search_view.text)
+ u'5'
+ >>> search_view.has_matches
+ True
+ >>> search_view.bug.title
+ u'Firefox install instructions should be complete'
+ >>> search_view.question.title
+ u'Installation failed'
+
+Bugs and questions are matched independent of each other. The number
+extracted may only match one kind of object. For example, there are
+more bugs than questions.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': '15'})
+ >>> search_view._getNumericToken(search_view.text)
+ u'15'
+ >>> search_view.has_matches
+ True
+ >>> search_view.bug.title
+ u'Nonsensical bugs are useless'
+ >>> print search_view.question
+ None
+
+Private bugs are not matched if the user does not have permission to
+see them. For example, Sample Person can see a private bug that they
+created because they are the owner.
+
+ >>> from lp.services.webapp.interfaces import ILaunchBag
+ >>> from lp.app.enums import InformationType
+
+ >>> login('test@xxxxxxxxxxxxx')
+ >>> sample_person = getUtility(ILaunchBag).user
+ >>> private_bug = factory.makeBug(
+ ... owner=sample_person, information_type=InformationType.USERDATA)
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': private_bug.id})
+ >>> search_view.bug.private
+ True
+
+But anonymous and unprivileged users cannot see the private bug.
+
+ >>> login(ANONYMOUS)
+ >>> search_view = getSearchView(
+ ... form={'field.text': private_bug.id})
+ >>> print search_view.bug
+ None
+
+The text and punctuation in the search text is ignored, and only the
+first group of numbers is matched. For example a user searches for three
+questions by number ('Question #15, #7, and 5.'). Only the first number
+is used, and it matches a bug, not a question. The second and third
+numbers do match questions, but they are not used.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'Question #15, #7, and 5.'})
+ >>> search_view._getNumericToken(search_view.text)
+ u'15'
+ >>> search_view.has_matches
+ True
+ >>> search_view.bug.title
+ u'Nonsensical bugs are useless'
+ >>> print search_view.question
+ None
+
+It is not an error to search for a non-existent bug or question.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': '55555'})
+ >>> search_view._getNumericToken(search_view.text)
+ u'55555'
+ >>> search_view.has_matches
+ False
+ >>> print search_view.bug
+ None
+ >>> print search_view.question
+ None
+
+There is no error if a number cannot be extracted from the search text.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'fifteen'})
+ >>> print search_view._getNumericToken(
+ ... search_view.text)
+ None
+ >>> search_view.has_matches
+ False
+ >>> print search_view.bug
+ None
+ >>> print search_view.question
+ None
+
+Bugs and questions are only returned for the first page of search,
+when the start param is 0.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': '5',
+ ... 'start': '20'})
+ >>> search_view.has_matches
+ False
+ >>> print search_view.bug
+ None
+ >>> print search_view.question
+ None
+
+
+
+Projects and Persons and Teams searches
+---------------------------------------
+
+When a Launchpad name can be made from the search text, the view tries
+to match the name to a pillar or person. a pillar is a distribution,
+product, or project group. A person is a person or a team.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'launchpad'})
+ >>> search_view._getNameToken(search_view.text)
+ u'launchpad'
+ >>> search_view.has_matches
+ True
+ >>> search_view.pillar.displayname
+ u'Launchpad'
+ >>> search_view.person_or_team.displayname
+ u'Launchpad Developers'
+
+A launchpad name is constructed from the search text. The letters are
+converted to lowercase. groups of spaces and punctuation are replaced
+with a hyphen.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'Gnome Terminal'})
+ >>> search_view._getNameToken(search_view.text)
+ u'gnome-terminal'
+ >>> search_view.has_matches
+ True
+ >>> search_view.pillar.displayname
+ u'GNOME Terminal'
+ >>> print search_view.person_or_team
+ None
+
+Since our pillars can have aliases, it's also possible to look up a pillar
+by any of its aliases.
+
+ >>> from lp.registry.interfaces.product import IProductSet
+ >>> firefox = getUtility(IProductSet)['firefox']
+ >>> login('foo.bar@xxxxxxxxxxxxx')
+ >>> firefox.setAliases(['iceweasel'])
+ >>> login(ANONYMOUS)
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'iceweasel'})
+ >>> search_view._getNameToken(search_view.text)
+ u'iceweasel'
+ >>> search_view.has_matches
+ True
+ >>> search_view.pillar.displayname
+ u'Mozilla Firefox'
+
+This is a harder example that illustrates that text that is clearly not
+the name of a pillar will none-the-less be tried. See the `Page searches`
+section for how this kind of search can return matches.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': "YAHOO! webservice's Python API."})
+ >>> search_view._getNameToken(search_view.text)
+ u'yahoo-webservices-python-api.'
+ >>> search_view.has_matches
+ False
+ >>> print search_view.pillar
+ None
+ >>> print search_view.person_or_team
+ None
+
+Leading and trailing punctuation and whitespace are stripped.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': "~name12"})
+ >>> search_view._getNameToken(search_view.text)
+ u'name12'
+ >>> search_view.has_matches
+ True
+ >>> print search_view.pillar
+ None
+ >>> search_view.person_or_team.displayname
+ u'Sample Person'
+
+Pillars, persons and teams are only returned for the first page of
+search, when the start param is 0.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'launchpad',
+ ... 'start': '20'})
+ >>> search_view.has_matches
+ True
+ >>> print search_view.bug
+ None
+ >>> print search_view.question
+ None
+ >>> print search_view.pillar
+ None
+
+Deactivated pillars and non-valid persons and teams cannot be exact
+matches. For example, the python-gnome2-dev product will not match a
+pillar, nor will nsv match Nicolas Velin's unclaimed account.
+
+ >>> from lp.registry.interfaces.person import IPersonSet
+
+ >>> python_gnome2 = getUtility(IProductSet).getByName('python-gnome2-dev')
+ >>> python_gnome2.active
+ False
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'python-gnome2-dev',
+ ... 'start': '0'})
+ >>> search_view._getNameToken(search_view.text)
+ u'python-gnome2-dev'
+ >>> print search_view.pillar
+ None
+
+ >>> nsv = getUtility(IPersonSet).getByName('nsv')
+ >>> nsv.displayname
+ u'Nicolas Velin'
+ >>> nsv.is_valid_person_or_team
+ False
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'nsv',
+ ... 'start': '0'})
+ >>> search_view._getNameToken(search_view.text)
+ u'nsv'
+ >>> print search_view.person_or_team
+ None
+
+Private pillars are not matched if the user does not have permission to see
+them. For example, Sample Person can see a private project that they created
+because they are the owner.
+
+ >>> from lp.registry.interfaces.product import License
+
+ >>> login('test@xxxxxxxxxxxxx')
+ >>> private_product = factory.makeProduct(
+ ... owner=sample_person, information_type=InformationType.PROPRIETARY,
+ ... licenses=[License.OTHER_PROPRIETARY])
+ >>> private_product_name = private_product.name
+
+ >>> search_view = getSearchView(form={'field.text': private_product_name})
+ >>> search_view.pillar.private
+ True
+
+But anonymous and unprivileged users cannot see the private project.
+
+ >>> login(ANONYMOUS)
+ >>> search_view = getSearchView(form={'field.text': private_product_name})
+ >>> print search_view.pillar
+ None
+
+
+Shipit CD searches
+------------------
+
+The has_shipit property will be True when the search looks like the user
+is searching for Shipit CDs. There is no correct object in Launchpad to
+display. The page template decides how to handle when has_shipit is
+True.
+
+The match is based on an intersection to the words in the search text
+and the shipit_keywords. The comparison is case-insensitive, has_shipit
+is True when 2 or more words match.
+
+ >>> sorted(search_view.shipit_keywords)
+ ['cd', 'cds', 'disc', 'dvd', 'dvds', 'edubuntu', 'free', 'get', 'kubuntu',
+ 'mail', 'send', 'ship', 'shipit', 'ubuntu']
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'ubuntu CDs',
+ ... 'start': '0'})
+ >>> search_view.has_shipit
+ True
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'shipit',
+ ... 'start': '0'})
+ >>> search_view.has_shipit
+ False
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'get Kubuntu cds',
+ ... 'start': '0'})
+ >>> search_view.has_shipit
+ True
+
+There are shipit_anti_keywords too, words that indicate the search is
+not for free CDs from Shipit. Search that have any of these word will
+set has_shipit to False.
+
+ >>> sorted(search_view.shipit_anti_keywords)
+ ['burn', 'burning', 'enable', 'error', 'errors', 'image', 'iso',
+ 'read', 'rip', 'write']
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'ubuntu CD write',
+ ... 'start': '0'})
+ >>> search_view.has_shipit
+ False
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': 'shipit error',
+ ... 'start': '0'})
+ >>> search_view.has_shipit
+ False
+
+
+The shipit FAQ URL is provides by the view for the template to use.
+
+ >>> search_view.shipit_faq_url
+ 'http://www.ubuntu.com/getubuntu/shipit-faq'
+
+
+Page searches
+-------------
+
+The view uses the BingSearchService to locate pages that match the
+search terms.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': " bug"})
+ >>> search_view.text
+ u'bug'
+ >>> search_view.has_matches
+ True
+ >>> search_view.pages
+ <...SiteSearchBatchNavigator ...>
+
+The BingSearchService may not be available due to connectivity problems.
+The view's has_page_service attribute reports when the search was performed
+with Bing page matches.
+
+ >>> search_view.has_page_service
+ True
+
+The batch navigation heading is created by the view. The heading
+property returns a 2-tuple of singular and plural heading. There
+is a heading when there are only Bing page matches...
+
+ >>> search_view.has_exact_matches
+ False
+ >>> search_view.batch_heading
+ (u'page matching "bug"', u'pages matching "bug"')
+
+...and a heading for when there are exact matches and Bing page
+matches.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': " launchpad"})
+ >>> search_view.has_exact_matches
+ True
+ >>> search_view.batch_heading
+ (u'other page matching "launchpad"', u'other pages matching "launchpad"')
+
+The SiteSearchBatchNavigator behaves like most BatchNavigators, except that
+its batch size is always 20. The size restriction conforms to Google's
+maximum number of results that can be returned per request.
+
+ >>> search_view.start
+ 0
+ >>> search_view.pages.currentBatch().size
+ 20
+ >>> pages = list(search_view.pages.currentBatch())
+ >>> len(pages)
+ 20
+ >>> for page in pages[0:5]:
+ ... page.title
+ u'Launchpad Bugs'
+ u'Bugs in Ubuntu Linux'
+ u'Bugs related to Sample Person'
+ u'Bug #1 in Mozilla Firefox: Firefox does not support SVG'
+ u'Question #232632 : Questions : OpenStack Heat'
+
+The batch navigator provides access to the other batches. There are two
+batches of pages that match the search text 'bugs'. The navigator
+provides a link to the next batch, which also happens to be the last
+batch.
+
+ >>> search_view.pages.nextBatchURL()
+ '...start=20'
+ >>> search_view.pages.lastBatchURL()
+ '...start=20'
+
+The second batch has only five matches in it, even though the batch size
+is 20. That is because there were only 25 matching pages.
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': "bug",
+ ... 'start': '20'})
+ >>> search_view.start
+ 20
+ >>> search_view.text
+ u'bug'
+ >>> search_view.has_matches
+ True
+
+ >>> search_view.pages.currentBatch().size
+ 20
+ >>> pages = list(search_view.pages.currentBatch())
+ >>> len(pages)
+ 5
+ >>> for page in pages:
+ ... page.title
+ u'Bugs - Launchpad Help'
+ u'Of Bugs and Statuses - Launchpad Blog'
+ u'Mahara 1.8.0'
+ u'Mighty Box in Launchpad'
+ u'Bug tracking - Launchpad Bugs'
+
+ >>> search_view.pages.nextBatchURL()
+ ''
+ >>> search_view.pages.lastBatchURL()
+ ''
+
+The PageMatch object has a title, url, and summary. The title and url
+are used for making links to the pages. The summary contains markup
+showing the matching terms in context of the page text.
+
+ >>> print range(20)
+ [0, 1, ..., 18, 19]
+ >>> page = pages[0]
+ >>> page
+ <...PageMatch ...>
+ >>> page.title
+ u'Bugs - Launchpad Help'
+ >>> page.url
+ 'https://help.launchpad.net/Bugs'
+ >>> page.summary # doctest: +ELLIPSIS
+ u"Launchpad Help > Bugs . Use Launchpad's bug tracker for your project..."
+
+See `google-searchservice.txt` for more information about the
+BingSearchService and PageMatch objects.
+
+
+No page matches
+---------------
+
+When an empty PageMatches object is returned by the BingSearchService to
+the view, there are no matches to show.
+
+ >>> search_view = getSearchView(form={'field.text': 'no-meaningful'})
+ >>> search_view.has_matches
+ False
+
+
+Unintelligible searches
+-----------------------
+
+When a user searches for a malformed string, we don't OOPS, but show an
+error. Also disable warnings, since we are tossing around malformed Unicode.
+
+ >>> import warnings
+ >>> with warnings.catch_warnings():
+ ... warnings.simplefilter('ignore')
+ ... search_view = getSearchView(
+ ... form={'field.text': '\xfe\xfckr\xfc'})
+ >>> html = search_view()
+ >>> 'Can not convert your search term' in html
+ True
+
+
+Bad Bing response handling
+----------------------------
+
+Connectivity problems can cause missing or incomplete responses from
+Bing. The LaunchpadSearchView will display the other searches and
+show a message explaining that the user can search again to find
+matching pages.
+
+ >>> search_view = getSearchView(form={'field.text': 'gnomebaker'})
+ >>> search_view.has_matches
+ True
+ >>> search_view.pillar.displayname
+ u'gnomebaker'
+ >>> search_view.has_page_service
+ False
+
+The view provides the requested URL so that the template can make a
+link to try the search again
+
+ >>> print search_view.url
+ https://launchpad.dev/+search?field.text=gnomebaker
+
+
+SearchFormView and SearchFormPrimaryView
+----------------------------------------
+
+Two companion views are used to help render the global search form.
+They define the required attributes to render the form in the
+correct state.
+
+The LaunchpadSearchFormView provides the minimum information to display
+the form, but cannot handled the submitted data. It appends a suffix
+('-secondary') to the id= and name= of the form and inputs, to prevent
+them from conflicting with the other form. The search text is not the
+default value of the text field; 'bug' was submitted above, but is not
+present in the rendered form.
+
+ >>> search_form_view = getMultiAdapter(
+ ... (search_view, request), name='+search-form')
+ >>> search_form_view.initialize()
+ >>> search_form_view.id_suffix
+ '-secondary'
+ >>> print search_form_view.render()
+ <form action="http://launchpad.dev/+search"; method="get"
+ accept-charset="UTF-8" id="sitesearch-secondary"
+ name="sitesearch-secondary">
+ <div>
+ <input class="textType" type="text" size="36"
+ id="field.text-secondary" name="field.text" />
+ <input class="button" type="submit" value="Search"
+ id="field.text-secondary" name="field.actions.search-secondary" />
+ </div>
+ </form>
+
+LaunchpadPrimarySearchFormView can handle submitted form by deferring to
+its context (the LaunchpadSearchView) for the needed information. The
+view does not append a suffix to the form and input ids. The search
+field's value is 'bug', as was submitted above.
+
+ >>> search_form_view = getMultiAdapter(
+ ... (search_view, request), name='+primary-search-form')
+ >>> search_form_view.initialize()
+ >>> search_form_view.id_suffix
+ ''
+ >>> print search_form_view.render()
+ <form action="http://launchpad.dev/+search"; method="get"
+ accept-charset="UTF-8" id="sitesearch"
+ name="sitesearch">
+ <div>
+ <input class="textType" type="text" size="36"
+ id="field.text" value="gnomebaker" name="field.text" />
+ <input class="button" type="submit" value="Search"
+ id="field.text" name="field.actions.search" />
+ </div>
+ </form>
+
+WindowedList and SiteSearchBatchNavigator
+-------------------------------------
+
+The LaunchpadSearchView uses two helper classes to work with
+PageMatches.
+
+The PageMatches object returned by the BingSearchService contains 20
+or fewer PageMatches of what could be thousands of matches. Bing
+requires client's to make repeats request to step though the batches of
+matches. The Windowed list is a list that contains only a subset of its
+reported size. It is used to make batches in the SiteSearchBatchNavigator.
+
+For example, the last batch of the 'bug' search contained 5 of the 25
+matching pages. The WindowList claims to be 25 items in length, but
+the first 20 items are None. Only the last 5 items are PageMatches.
+
+ >>> from lp.app.browser.root import WindowedList
+ >>> from lp.services.sitesearch import BingSearchService
+
+ >>> bing_search = BingSearchService()
+ >>> page_matches = bing_search.search(terms='bug', start=20)
+ >>> results = WindowedList(
+ ... page_matches, page_matches.start, page_matches.total)
+ >>> len(results)
+ 25
+ >>> print results[0]
+ None
+ >>> results[24].title
+ u'Bug tracking - Launchpad Bugs'
+ >>> results[18, 22]
+ [None, None, <...PageMatch ...>, <...PageMatch ...>]
+
+The SiteSearchBatchNavigator restricts the batch size to 20. the 'batch'
+parameter that comes from the URL is ignored. For example, setting
+the 'batch' parameter to 100 has no affect upon the Bing search
+or on the navigator object.
+
+ >>> from lp.app.browser.root import SiteSearchBatchNavigator
+
+ >>> SiteSearchBatchNavigator.batch_variable_name
+ 'batch'
+
+ >>> search_view = getSearchView(
+ ... form={'field.text': "bug",
+ ... 'start': '0',
+ ... 'batch': '100',})
+
+ >>> navigator = search_view.pages
+ >>> navigator.currentBatch().size
+ 20
+ >>> len(navigator.currentBatch())
+ 20
+ >>> navigator.nextBatchURL()
+ '...start=20'
+
+Even if the PageMatch object to have an impossibly large size, the
+navigator conforms to Google's maximum size of 20.
+
+ >>> matches = list(range(0, 100))
+ >>> page_matches._matches = matches
+ >>> page_matches.start = 0
+ >>> page_matches.total = 100
+ >>> navigator = SiteSearchBatchNavigator(
+ ... page_matches, search_view.request, page_matches.start, size=100)
+ >>> navigator.currentBatch().size
+ 20
+ >>> len(navigator.currentBatch())
+ 20
+ >>> navigator.nextBatchURL()
+ '...start=20'
+
+The PageMatches object can be smaller than 20, for instance, pages
+without titles are skipped when parsing the Bing Search JSON. The size
+of the batch is still 20, but when the items in the batch are iterated,
+the true size can be seen. For example there could be only 3 matches in
+the PageMatches object, so only 3 are yielded. The start of the next
+batch is 20, which is the start of the next batch from Bing.
+
+ >>> matches = list(range(0, 3))
+ >>> page_matches._matches = matches
+ >>> navigator = SiteSearchBatchNavigator(
+ ... page_matches, search_view.request, page_matches.start, size=100)
+ >>> batch = navigator.currentBatch()
+ >>> batch.size
+ 20
+ >>> len(batch)
+ 20
+ >>> batch.endNumber()
+ 3
+ >>> for item in batch:
+ ... print item
+ 0
+ 1
+ 2
+ >>> navigator.nextBatchURL()
+ '...start=20'
=== renamed file 'lib/lp/app/browser/tests/launchpad-search-pages.txt' => 'lib/lp/app/browser/doc/launchpad-search-pages-google.txt'
=== renamed file 'lib/lp/app/browser/tests/launchpadform-view.txt' => 'lib/lp/app/browser/doc/launchpadform-view.txt'
=== renamed file 'lib/lp/app/browser/tests/menu.txt' => 'lib/lp/app/browser/doc/menu.txt'
=== renamed file 'lib/lp/app/browser/tests/root-views.txt' => 'lib/lp/app/browser/doc/root-views.txt'
=== renamed file 'lib/lp/app/browser/tests/watermark.txt' => 'lib/lp/app/browser/doc/watermark.txt'
=== modified file 'lib/lp/app/browser/root.py'
--- lib/lp/app/browser/root.py 2018-03-27 14:31:36 +0000
+++ lib/lp/app/browser/root.py 2018-03-28 21:29:05 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Browser code for the Launchpad root page."""
@@ -510,18 +510,21 @@
def searchPages(self, query_terms, start=0):
"""Return the up to 20 pages that match the query_terms, or None.
- :param query_terms: The unescaped terms to query Google.
+ :param query_terms: The unescaped terms to query for.
:param start: The index of the page that starts the set of pages.
- :return: A GooglBatchNavigator or None.
+ :return: A SiteSearchBatchNavigator or None.
"""
if query_terms in [None, '']:
return None
- google_search = getUtility(ISearchService)
+ search_engine = getFeatureFlag("sitesearch.engine.name")
+ # Default to the Google search engine.
+ search_engine = search_engine or "google"
+ site_search = getUtility(ISearchService, name=search_engine)
try:
- page_matches = google_search.search(
+ page_matches = site_search.search(
terms=query_terms, start=start)
except SiteSearchResponseError:
- # There was a connectivity or Google service issue that means
+ # There was a connectivity or search service issue that means
# there is no data available at this moment.
self.has_page_service = False
return None
=== modified file 'lib/lp/app/browser/tests/test_views.py'
--- lib/lp/app/browser/tests/test_views.py 2011-12-28 17:03:06 +0000
+++ lib/lp/app/browser/tests/test_views.py 2018-03-28 21:29:05 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""
@@ -7,10 +7,11 @@
import logging
import os
-import unittest
+from lp.services.features.testing import FeatureFixture
+from lp.services.testing import build_test_suite
from lp.testing.layers import (
- DatabaseFunctionalLayer,
+ BingLaunchpadFunctionalLayer,
GoogleLaunchpadFunctionalLayer,
)
from lp.testing.systemdocs import (
@@ -21,32 +22,46 @@
here = os.path.dirname(os.path.realpath(__file__))
+bing_flag = FeatureFixture({'sitesearch.engine.name': 'bing'})
+google_flag = FeatureFixture({'sitesearch.engine.name': 'google'})
+
+
+def setUp_bing(test):
+ setUp(test)
+ bing_flag.setUp()
+
+
+def setUp_google(test):
+ setUp(test)
+ google_flag.setUp()
+
+
+def tearDown_bing(test):
+ bing_flag.cleanUp()
+ tearDown(test)
+
+
+def tearDown_google(test):
+ google_flag.cleanUp()
+ tearDown(test)
+
# The default layer of view tests is the DatabaseFunctionalLayer. Tests
# that require something special like the librarian or mailman must run
# on a layer that sets those services up.
-special_test_layer = {
- 'launchpad-search-pages.txt': GoogleLaunchpadFunctionalLayer,
+special = {
+ 'launchpad-search-pages-bing.txt': LayeredDocFileSuite(
+ '../doc/launchpad-search-pages-bing.txt',
+ setUp=setUp_bing, tearDown=tearDown_bing,
+ layer=BingLaunchpadFunctionalLayer,
+ stdout_logging_level=logging.WARNING),
+ 'launchpad-search-pages-google.txt': LayeredDocFileSuite(
+ '../doc/launchpad-search-pages-google.txt',
+ setUp=setUp_google, tearDown=tearDown_google,
+ layer=GoogleLaunchpadFunctionalLayer,
+ stdout_logging_level=logging.WARNING),
}
def test_suite():
- suite = unittest.TestSuite()
- testsdir = os.path.abspath(here)
-
- # Add tests using default setup/teardown
- filenames = [filename
- for filename in os.listdir(testsdir)
- if filename.endswith('.txt')]
- # Sort the list to give a predictable order.
- filenames.sort()
- for filename in filenames:
- path = filename
- layer = special_test_layer.get(path, DatabaseFunctionalLayer)
- one_test = LayeredDocFileSuite(
- path, setUp=setUp, tearDown=tearDown, layer=layer,
- stdout_logging_level=logging.WARNING
- )
- suite.addTest(one_test)
-
- return suite
+ return build_test_suite(here, special)
=== modified file 'lib/lp/app/stories/launchpad-root/site-search.txt'
--- lib/lp/app/stories/launchpad-root/site-search.txt 2016-01-26 15:47:37 +0000
+++ lib/lp/app/stories/launchpad-root/site-search.txt 2018-03-28 21:29:05 +0000
@@ -1,9 +1,9 @@
Site-wide Search
================
-Launchpad features a site-wide search that combines Google's site-
-specific search with Launchpad's prominent objects (projects, bugs,
-teams, etc.).
+Launchpad features a site-wide search that combines an external search
+engine's site-specific search with Launchpad's prominent objects (projects,
+bugs, teams, etc.).
# Our very helpful function for printing all the page results.
@@ -81,9 +81,9 @@
3: Firefox is slow and consumes too much RAM
...
-An arbitrary search returns a list of Google search results. The user
-searches for "bug" and sees a listing of matching pages. The navigation
-states that the page is showing 1 through 20 of 25 total results.
+An arbitrary search returns a list of the search engine's search results.
+The user searches for "bug" and sees a listing of matching pages. The
+navigation states that the page is showing 1 through 20 of 25 total results.
# Use our pre-defined search results for the 'bug' search.
@@ -248,9 +248,10 @@
Searches when there is no page service
--------------------------------------
-Google may not be available when the search is performed. This is often
-caused by temporary connectivity problems. A message is displayed that
-explains that the search can be performed again to find matching pages.
+The search provider may not be available when the search is performed.
+This is often caused by temporary connectivity problems. A message is
+displayed that explains that the search can be performed again to find
+matching pages.
>>> search_for('gnomebaker')
>>> print find_tag_by_id(anon_browser.contents, 'no-page-service')
=== modified file 'lib/lp/scripts/runlaunchpad.py'
--- lib/lp/scripts/runlaunchpad.py 2018-03-16 14:50:01 +0000
+++ lib/lp/scripts/runlaunchpad.py 2018-03-28 21:29:05 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -26,7 +26,10 @@
pidfile_path,
)
from lp.services.rabbit.server import RabbitServer
-from lp.services.sitesearch import googletestservice
+from lp.services.sitesearch import (
+ bingtestservice,
+ googletestservice,
+ )
from lp.services.txlongpoll.server import TxLongPollServer
@@ -153,6 +156,16 @@
self.addCleanup(stop_process, googletestservice.start_as_process())
+class BingWebService(Service):
+
+ @property
+ def should_launch(self):
+ return config.bing_test_service.launch
+
+ def launch(self):
+ self.addCleanup(stop_process, bingtestservice.start_as_process())
+
+
class MemcachedService(Service):
"""A local memcached service for developer environments."""
@@ -280,6 +293,7 @@
'sftp': TacFile('sftp', 'daemons/sftp.tac', 'codehosting'),
'forker': ForkingSessionService(),
'mailman': MailmanService(),
+ 'bing-webservice': BingWebService(),
'codebrowse': CodebrowseService(),
'google-webservice': GoogleWebService(),
'memcached': MemcachedService(),
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2018-03-16 14:02:16 +0000
+++ lib/lp/services/config/schema-lazr.conf 2018-03-28 21:29:05 +0000
@@ -791,7 +791,43 @@
# url_rewrite_exceptions is a list of launchpad.net domains that must
# not be rewritten.
# datatype: string of space separated domains
-# Example: help.launchpad.net login.launchapd.net
+# Example: help.launchpad.net login.launchpad.net
+url_rewrite_exceptions: help.launchpad.net
+
+[bing_test_service]
+# Run a web service stub that simulates the Bing search service.
+
+# Where are our canned JSON responses stored?
+canned_response_directory: lib/lp/services/sitesearch/tests/data/
+
+# Which file maps service URLs to the JSON that the server returns?
+mapfile: lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt
+
+# Where should the service log files live?
+log: logs/bing-stub.log
+
+# Do we actually want to run the service?
+launch: False
+
+[bing]
+# site is the host and path that search requests are made to.
+# eg. https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search
+# datatype: string, a url to a host
+site: https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search
+
+# subscription_key is the Cognitive Services subscription key for
+# Bing Custom Search API.
+# datatype: string
+subscription_key:
+
+# custom_config_id is the id that identifies the custom search instance.
+# datatype: string
+custom_config_id:
+
+# url_rewrite_exceptions is a list of launchpad.net domains that must
+# not be rewritten.
+# datatype: string of space separated domains
+# Example: help.launchpad.net login.launchpad.net
url_rewrite_exceptions: help.launchpad.net
[gpghandler]
=== modified file 'lib/lp/services/features/flags.py'
--- lib/lp/services/features/flags.py 2016-10-14 16:16:18 +0000
+++ lib/lp/services/features/flags.py 2018-03-28 21:29:05 +0000
@@ -234,6 +234,12 @@
'disabled',
'Named authorization tokens for archives',
''),
+ ('sitesearch.engine.name',
+ 'space delimited',
+ 'Name of the site search engine backend ("google" or "bing").',
+ 'google',
+ 'Site search engine',
+ ''),
])
# The set of all flag names that are documented.
=== modified file 'lib/lp/services/sitesearch/__init__.py'
--- lib/lp/services/sitesearch/__init__.py 2018-03-27 15:47:35 +0000
+++ lib/lp/services/sitesearch/__init__.py 2018-03-28 21:29:05 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Interfaces for searching and working with results."""
@@ -6,11 +6,13 @@
__metaclass__ = type
__all__ = [
+ 'BingSearchService',
'GoogleSearchService',
'PageMatch',
'PageMatches',
]
+import json
import urllib
from urlparse import (
parse_qsl,
@@ -163,7 +165,7 @@
class GoogleSearchService:
"""See `ISearchService`.
- A search service that search Google for launchpad.net pages.
+ A search service that searches Google for launchpad.net pages.
"""
_default_values = {
@@ -333,3 +335,160 @@
raise GoogleWrongGSPVersion(
"Could not get any PageMatches from the GSP XML response.")
return PageMatches(page_matches, start, total)
+
+
+@implementer(ISearchService)
+class BingSearchService:
+ """See `ISearchService`.
+
+ A search service that searches Bing for launchpad.net pages.
+ """
+
+ _default_values = {
+ # XXX: maxiberta 2018-03-26: Set `mkt` based on the current request.
+ 'customConfig': None,
+ 'mkt': 'en-US',
+ 'count': 20,
+ 'offset': 0,
+ 'q': None,
+ }
+
+ @property
+ def subscription_key(self):
+ """The subscription key issued by Bing Custom Search."""
+ return config.bing.subscription_key
+
+ @property
+ def custom_config_id(self):
+ """The custom search instance as configured in Bing Custom Search."""
+ return config.bing.custom_config_id
+
+ @property
+ def site(self):
+ """The URL to the Bing Custom Search service.
+
+ The URL is probably
+ https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search.
+ """
+ return config.bing.site
+
+ def search(self, terms, start=0):
+ """See `ISearchService`.
+
+ The `subscription_key` and `custom_config_id` are used in the
+ search request. Search returns 20 or fewer results for each query.
+ For terms that match more than 20 results, the start param can be
+ used over multiple queries to get successive sets of results.
+
+ :return: `ISearchResults` (PageMatches).
+ :raise: `SiteSearchResponseError` if the json response is incomplete or
+ cannot be parsed.
+ """
+ search_url = self.create_search_url(terms, start=start)
+ search_headers = self.create_search_headers()
+ request = get_current_browser_request()
+ timeline = get_request_timeline(request)
+ action = timeline.start("bing-search-api", search_url)
+ try:
+ response = urlfetch(search_url, headers=search_headers)
+ except (TimeoutError, requests.RequestException) as error:
+ raise SiteSearchResponseError(
+ "The response errored: %s" % str(error))
+ finally:
+ action.finish()
+ page_matches = self._parse_bing_response(response.content, start)
+ return page_matches
+
+ def _checkParameter(self, name, value, is_int=False):
+ """Check that a parameter value is not None or an empty string."""
+ if value in (None, ''):
+ raise ValueError("Missing value for parameter '%s'." % name)
+ if is_int:
+ try:
+ int(value)
+ except ValueError:
+ raise ValueError(
+ "Value for parameter '%s' is not an int." % name)
+
+ def create_search_url(self, terms, start=0):
+ """Return a Bing Custom Search search url."""
+ self._checkParameter('q', terms)
+ self._checkParameter('offset', start, is_int=True)
+ self._checkParameter('customConfig', self.custom_config_id)
+ search_params = dict(self._default_values)
+ search_params['q'] = terms.encode('utf8')
+ search_params['offset'] = start
+ search_params['customConfig'] = self.custom_config_id
+ query_string = urllib.urlencode(sorted(search_params.items()))
+ return self.site + '?' + query_string
+
+ def create_search_headers(self):
+ """Return a dict with Bing Custom Search compatible request headers."""
+ self._checkParameter('subscription_key', self.subscription_key)
+ return {
+ 'Ocp-Apim-Subscription-Key': self.subscription_key,
+ }
+
+ def _parse_bing_response(self, bing_json, start=0):
+ """Return a `PageMatches` object.
+
+ :param bing_json: A string containing Bing Custom Search API v7 JSON.
+ :return: `ISearchResults` (PageMatches).
+ :raise: `SiteSearchResponseError` if the json response is incomplete or
+ cannot be parsed.
+ """
+ try:
+ bing_doc = json.loads(bing_json)
+ except (TypeError, ValueError):
+ raise SiteSearchResponseError(
+ "The response was incomplete, no JSON.")
+
+ try:
+ response_type = bing_doc['_type']
+ except (AttributeError, KeyError, ValueError):
+ raise SiteSearchResponseError(
+ "Could not get the '_type' from the Bing JSON response.")
+
+ if response_type == 'ErrorResponse':
+ try:
+ errors = [error['message'] for error in bing_doc['errors']]
+ raise SiteSearchResponseError(
+ "Error response from Bing: %s" % '; '.join(errors))
+ except (AttributeError, KeyError, TypeError, ValueError):
+ raise SiteSearchResponseError(
+ "Unable to parse the Bing JSON error response.")
+ elif response_type != 'SearchResponse':
+ raise SiteSearchResponseError(
+ "Unknown Bing JSON response type: '%s'." % response_type)
+
+ page_matches = []
+ total = 0
+ try:
+ results = bing_doc['webPages']['value']
+ except (AttributeError, KeyError, ValueError):
+ # Bing did not match any pages. Return an empty PageMatches.
+ return PageMatches(page_matches, start, total)
+
+ try:
+ total = int(bing_doc['webPages']['totalEstimatedMatches'])
+ except (AttributeError, KeyError, ValueError):
+ # The datatype is not what PageMatches requires.
+ raise SiteSearchResponseError(
+ "Could not get the total from the Bing JSON response.")
+ if total < 0:
+ # See bug 683115.
+ total = 0
+ for result in results:
+ url = result.get('url')
+ title = result.get('name')
+ summary = result.get('snippet')
+ if None in (url, title, summary):
+ # There is not enough data to create a PageMatch object.
+ # This can be caused by an empty title or summary which
+ # has been observed for pages that are from vhosts that
+ # should not be indexed.
+ continue
+ summary = summary.replace('<br>', '')
+ page_matches.append(PageMatch(title, url, summary))
+
+ return PageMatches(page_matches, start, total)
=== added file 'lib/lp/services/sitesearch/bingtestservice.py'
--- lib/lp/services/sitesearch/bingtestservice.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/bingtestservice.py 2018-03-28 21:29:05 +0000
@@ -0,0 +1,79 @@
+#!/usr/bin/python
+#
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""
+This script runs a simple HTTP server. The server returns JSON files
+when given certain user-configurable URLs.
+"""
+
+import logging
+import os
+
+from six.moves.BaseHTTPServer import HTTPServer
+
+from lp.services.config import config
+from lp.services.osutils import ensure_directory_exists
+from lp.services.pidfile import make_pidfile
+from lp.services.sitesearch import testservice
+
+
+# Set up basic logging.
+log = logging.getLogger(__name__)
+
+# The default service name, used by the Launchpad service framework.
+service_name = 'bing-webservice'
+
+
+class BingRequestHandler(testservice.RequestHandler):
+ default_content_type = 'text/xml; charset=UTF-8'
+ log = log
+ mapfile = config.bing_test_service.mapfile
+ content_dir = config.bing_test_service.canned_response_directory
+
+
+def start_as_process():
+ return testservice.start_as_process('bingtestservice')
+
+
+def get_service_endpoint():
+ """Return the host and port that the service is running on."""
+ return testservice.hostpair(config.bing.site)
+
+
+def service_is_available():
+ host, port = get_service_endpoint()
+ return testservice.service_is_available(host, port)
+
+
+def wait_for_service():
+ host, port = get_service_endpoint()
+ return testservice.wait_for_service(host, port)
+
+
+def kill_running_process():
+ global service_name
+ host, port = get_service_endpoint()
+ return testservice.kill_running_process(service_name, host, port)
+
+
+def main():
+ """Run the HTTP server."""
+ # Redirect our service output to a log file.
+ global log
+ ensure_directory_exists(os.path.dirname(config.bing_test_service.log))
+ filelog = logging.FileHandler(config.bing_test_service.log)
+ log.addHandler(filelog)
+ log.setLevel(logging.DEBUG)
+
+ # To support service shutdown we need to create a PID file that is
+ # understood by the Launchpad services framework.
+ global service_name
+ make_pidfile(service_name)
+
+ host, port = get_service_endpoint()
+ server = HTTPServer((host, port), BingRequestHandler)
+
+ log.info("Starting HTTP Bing webservice server on port %s", port)
+ server.serve_forever()
=== modified file 'lib/lp/services/sitesearch/configure.zcml'
--- lib/lp/services/sitesearch/configure.zcml 2018-03-16 14:02:16 +0000
+++ lib/lp/services/sitesearch/configure.zcml 2018-03-28 21:29:05 +0000
@@ -1,4 +1,4 @@
-<!-- Copyright 2009-2010 Canonical Ltd. This software is licensed under the
+<!-- Copyright 2009-2018 Canonical Ltd. This software is licensed under the
GNU Affero General Public License version 3 (see the file LICENSE).
-->
@@ -16,8 +16,17 @@
</class>
<securedutility
+ name="google"
class="lp.services.sitesearch.GoogleSearchService"
provides="lp.services.sitesearch.interfaces.ISearchService">
<allow interface="lp.services.sitesearch.interfaces.ISearchService" />
</securedutility>
+
+ <securedutility
+ name="bing"
+ class="lp.services.sitesearch.BingSearchService"
+ provides="lp.services.sitesearch.interfaces.ISearchService">
+ <allow interface="lp.services.sitesearch.interfaces.ISearchService" />
+ </securedutility>
+
</configure>
=== added file 'lib/lp/services/sitesearch/doc/bing-searchservice.txt'
--- lib/lp/services/sitesearch/doc/bing-searchservice.txt 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/doc/bing-searchservice.txt 2018-03-28 21:29:05 +0000
@@ -0,0 +1,438 @@
+===================
+Bing Search Service
+===================
+
+The BingSearchService is a Bing Custom Search client.
+Given one or more terms, it will retrieve a JSON
+summary of the matching launchpad.net pages.
+
+We silence logging of new HTTP connections from requests throughout.
+
+ >>> from fixtures import FakeLogger
+ >>> logger = FakeLogger()
+ >>> logger.setUp()
+
+
+BingSearchService
+=================
+
+The BingSearchService implements the ISearchService interface.
+
+ >>> from zope.component import getUtility
+ >>> from zope.interface.verify import verifyObject
+ >>> from lp.services.sitesearch.interfaces import (
+ ... ISearchService)
+
+ >>> bing_search = getUtility(ISearchService, name="bing")
+ >>> verifyObject(ISearchService, bing_search)
+ True
+ >>> bing_search
+ <...BingSearchService ...>
+
+
+--------------------------
+BingSearchService search()
+--------------------------
+
+The search method accepts a string argument of terms and an optional int
+argument of start. The terms are the same as the text that would be
+entered in Bing search form; the terms should not be escaped.
+
+ >>> from lp.services.sitesearch.interfaces import (
+ ... ISearchResults)
+
+ >>> first_page_matches = bing_search.search(terms='bug')
+ >>> first_page_matches
+ <...PageMatches ...>
+
+The start parameter specifies the index (starting at 0) of the first
+result returned in the overall set of matches. Since 20 results are
+returned, to get the second batch of matches, you would use start=20.
+
+ >>> second_page_matches = bing_search.search(terms='bug', start=20)
+ >>> second_page_matches
+ <...PageMatches ...>
+
+
+PageMatches
+===========
+
+The PageMatches object returned by BingSearchService.search()
+implements ISearchResults.
+
+ >>> verifyObject(ISearchResults, first_page_matches)
+ True
+
+The 'total' attribute is the total number of matches that the search
+found. If that number is higher than 20, it means that multiple requests
+would be needed to retrieve the entire result set.
+
+The 'start' attribute is the index of the first returned item within the
+entire collection of matches. The 'length' attribute contains the number
+of returned results.
+
+The first search for 'bugs' returned a subset of items in the
+ISearchResult. There are 25 total items, but the results contains the
+first 20 matches (because they start at index 0).
+
+ >>> first_page_matches.start
+ 0
+ >>> first_page_matches.total
+ 25
+ >>> len(first_page_matches)
+ 20
+
+The second search for 'bugs' returned the remainder of the 25 matches.
+They start from index 20.
+
+ >>> verifyObject(ISearchResults, second_page_matches)
+ True
+
+ >>> second_page_matches.start
+ 20
+ >>> second_page_matches.total
+ 25
+ >>> len(second_page_matches)
+ 5
+
+An item can be retrieved from an ISearchResults object using its
+index. All the items in the collection can be iterated.
+
+ >>> second_page_matches[1].url
+ 'http://blog.launchpad.dev/general/of-bugs-and-statuses'
+
+ >>> for page_match in second_page_matches:
+ ... page_match.url
+ 'https://help.launchpad.net/Bugs'
+ 'http://blog.launchpad.dev/general/of-bugs-and-statuses'
+ 'http://launchpad.dev/mahara/+milestone/1.8.0'
+ 'http://launchpad.dev/mb'
+ 'http://launchpad.dev/bugs'
+
+An empty PageMatches is returns if there are no results.
+
+ >>> no_page_matches = bing_search.search(terms='fnord')
+ >>> no_page_matches.start
+ 0
+ >>> no_page_matches.total
+ 0
+ >>> len(no_page_matches)
+ 0
+
+
+PageMatch
+=========
+
+The PageMatch object represents a single result from a search result
+set. It is created by passing a title, url, and a summary. It is
+an implementation of ISearchResult.
+
+ >>> from lp.services.sitesearch.interfaces import ISearchResult
+ >>> from lp.services.sitesearch import PageMatch
+
+ >>> page_match = PageMatch(
+ ... u'Unicode Titles in Launchpad',
+ ... 'https://launchpad.net/unicode-titles',
+ ... u'Unicode Titles is a modest project dedicated to using Unicode.')
+
+ >>> verifyObject(ISearchResult, page_match)
+ True
+
+The title and summary attributes contain the same text that
+initialized the object.
+
+ >>> page_match.title
+ u'Unicode Titles in Launchpad'
+ >>> page_match.summary
+ u'Unicode Titles is a modest project dedicated to using Unicode.'
+
+The URL's domain is rewitten to the so that links from launchpad.net are
+mapped to the local environment.
+
+ >>> page_match.url
+ 'http://launchpad.dev/unicode-titles'
+
+
+Search configuration
+====================
+
+The bing search service is configured by the bing section in
+lazr.config. All requests are made to Bing's site, but the
+configuration may set a testing site.
+
+ >>> from lp.services.config import config
+ >>> from lp.services.sitesearch import BingSearchService
+
+ >>> bing_search = BingSearchService()
+ >>> config.bing.site == bing_search.site
+ True
+ >>> bing_search.site
+ 'http://launchpad.dev:.../bingcustomsearch/v7.0/search'
+
+The subscription_key is the Cognitive Services subscription key for
+Bing Custom Search API.
+
+ >>> config.bing.subscription_key == bing_search.subscription_key
+ True
+ >>> bing_search.subscription_key
+ 'abcdef01234567890abcdef012345678'
+
+The custom_config_id is the id that identifies the custom search instance.
+
+ >>> config.bing.custom_config_id == bing_search.custom_config_id
+ True
+ >>> bing_search.custom_config_id
+ 1234567890
+
+Several default query parameters are constant. They are kept in the
+_default_values dict. The q (terms) and start params are provided at
+the time of the search.
+
+ >>> for key, value in sorted(bing_search._default_values.items()):
+ ... print key, ':', repr(value)
+ count : 20
+ customConfig : None
+ mkt : 'en-US'
+ offset : 0
+ q : None
+
+create_search_url()
+===================
+
+The search url used inside the search() method is created by
+create_search_url(). It accepts two optional arguments: terms and start.
+An error is raised if any of the parameters are None.
+
+ >>> bing_search.create_search_url('')
+ Traceback (most recent call last):
+ ...
+ ValueError: Missing value for parameter 'q'.
+
+ >>> bing_search.create_search_url(None)
+ Traceback (most recent call last):
+ ...
+ ValueError: Missing value for parameter 'q'.
+
+ >>> bing_search.create_search_url('bugs', start='true')
+ Traceback (most recent call last):
+ ...
+ ValueError: Value for parameter 'offset' is not an int.
+
+The term parameter in this example can be defined by passing the term
+argument to the method. The argument is url encoded and used as the
+value for the 'q' (query) parameter.
+
+ >>> bing_search.create_search_url(terms='svg +bugs').replace('&', ' ')
+ 'http://launchpad.dev:.../bingcustomsearch/v7.0/search?count=20 customConfig=1234567890 mkt=en-US offset=0 q=svg+%2Bbugs'
+
+Unicode characters are escaped correctly in the bing request URL.
+
+ >>> bing_search.create_search_url(terms=u'Carlos Perell\xf3 Mar\xedn')
+ 'http://launchpad.dev:.../...offset=0&q=Carlos+Perell%C3%B3+Mar%C3%ADn'
+
+The start parameter can be changed by passing a start int argument.
+
+ >>> bing_search.create_search_url(terms='svg +bugs', start=20)
+ 'http://launchpad.dev:.../...offset=20&q=svg+%2Bbugs'
+
+
+Bing Search response parsing
+============================
+
+The BingSearchService's _parse_bing_response() expects a JSON response to
+create the PageMatch and PageMatches objects. An error is raised when
+the JSON document cannot be parsed into objects.
+
+The PageMatches's total attribute comes from the `webPages.totalEstimatedMatches`
+JSON element. When it cannot be found and the value cast to an int,
+an error is raised. If Bing were to redefine the meaning of the
+element to use a '~' to indicate an approximate total, an error would
+be raised.
+
+ >>> from os import path
+
+ >>> base_path = path.normpath(path.join(
+ ... path.dirname(__file__), '..', 'tests', 'data'))
+ >>> json_file_name = path.join(
+ ... base_path, 'bingsearchservice-incompatible-matches.json')
+ >>> with open(json_file_name, 'r') as json_file:
+ ... data = json_file.read()
+ >>> print data
+ {...
+ "totalEstimatedMatches": "~25"...
+
+ >>> bing_search._parse_bing_response(data)
+ Traceback (most recent call last):
+ ...
+ SiteSearchResponseError: Could not get the total from the
+ Bing JSON response.
+
+On the other hand, if the total is ever less than zero (see bug 683115),
+this is expected: we simply return a total of 0.
+
+ >>> json_file_name = path.join(
+ ... base_path, 'bingsearchservice-negative-total.json')
+ >>> with open(json_file_name, 'r') as json_file:
+ ... data = json_file.read()
+ >>> print data
+ {...
+ "totalEstimatedMatches": -25...
+
+ >>> bing_search._parse_bing_response(data).total
+ 0
+
+A PageMatch requires a title, url, and a summary. If those elements cannot
+be found, a PageMatch cannot be made. A missing title ('name') indicates
+a bad page on Launchpad, so it is ignored. In this example, the first match
+is missing a title, so only the second page is present in the PageMatches.
+
+ >>> json_file_name = path.join(
+ ... base_path, 'bingsearchservice-missing-title.json')
+ >>> with open(json_file_name, 'r') as json_file:
+ ... data = json_file.read()
+ >>> page_matches = bing_search._parse_bing_response(data)
+ >>> len(page_matches)
+ 1
+ >>> page_matches[0].title
+ u'GCleaner in Launchpad'
+ >>> page_matches[0].url
+ 'http://launchpad.dev/gcleaner'
+
+When a match is missing a summary ('snippet'), the match is skipped because
+there is no information about why it matched. This appears to relate to
+pages that are in the index, but should be removed. In this example
+taken from real data, the links are to the same page on different
+vhosts. The edge vhost has no summary, so it is skipped.
+
+ >>> json_file_name = path.join(
+ ... base_path, 'bingsearchservice-missing-summary.json')
+ >>> with open(json_file_name, 'r') as json_file:
+ ... data = json_file.read()
+ >>> page_matches = bing_search._parse_bing_response(data)
+ >>> len(page_matches)
+ 1
+ >>> page_matches[0].title
+ u'BugExpiry - Launchpad Help'
+ >>> page_matches[0].url
+ 'https://help.launchpad.net/BugExpiry'
+
+When the URL ('url') cannot be found the match is skipped. There are no
+examples of this. We do not want this hypothetical situation to give
+users a bad experience.
+
+ >>> json_file_name = path.join(
+ ... base_path, 'bingsearchservice-missing-url.json')
+ >>> with open(json_file_name, 'r') as json_file:
+ ... data = json_file.read()
+ >>> page_matches = bing_search._parse_bing_response(data)
+ >>> len(page_matches)
+ 1
+ >>> page_matches[0].title
+ u'LongoMatch in Launchpad'
+ >>> page_matches[0].url
+ 'http://launchpad.dev/longomatch'
+
+If no matches are found in the response, and there are 20 or fewer results,
+an Empty PageMatches is returned. This happens when the results are missing
+titles and summaries. This is not considered to be a problem because the
+small number implies that Bing did a poor job of indexing pages or indexed
+the wrong Launchpad server. In this example, there is only one match, but
+the results is missing a title so there is not enough information to make
+a PageMatch.
+
+ >>> json_file_name = path.join(
+ ... base_path, 'bingsearchservice-no-meaningful-results.json')
+ >>> with open(json_file_name, 'r') as json_file:
+ ... data = json_file.read()
+ >>> page_matches = bing_search._parse_bing_response(data)
+ >>> len(page_matches)
+ 0
+
+
+-------------
+URL rewriting
+-------------
+
+The URL scheme used in the rewritten URL is configured in
+config.bing.url_rewrite_scheme. The hostname is set in the shared
+key config.vhost.mainsite.hostname.
+
+ >>> config.vhosts.use_https
+ False
+ >>> page_match.url_rewrite_scheme
+ 'http'
+
+ >>> config.vhost.mainsite.hostname == page_match.url_rewrite_hostname
+ True
+ >>> page_match.url_rewrite_hostname
+ 'launchpad.dev'
+
+URLs are rewritten to map public URL to the private hostname.
+The vhost name is preserved when the URL is rewritten.
+
+ >>> page_match = PageMatch(
+ ... u'Bug #456 in Unicode title: "testrunner hates Unicode"',
+ ... 'https://bugs.launchpad.net/unicode-titles/+bug/456',
+ ... u'The Zope testrunner likes ASCII more than Unicode.')
+ >>> page_match.url
+ 'http://bugs.launchpad.dev/unicode-titles/+bug/456'
+
+A URL's trailing slash is removed; Launchpad does not use trailing
+slashes.
+
+ >>> page_match = PageMatch(
+ ... u'Ubuntu in Launchpad',
+ ... 'https://launchpad.net/ubuntu/',
+ ... u'Ubuntu also includes more software than any other operating')
+ >>> page_match.url
+ 'http://launchpad.dev/ubuntu'
+
+There is a list of URLs that are not rewritten configured in
+config.bing.url_rewrite_exceptions. For example, help.launchpad.net
+is only run in one environment, so links to that site will be preserved.
+
+ >>> config.bing.url_rewrite_exceptions
+ 'help.launchpad.net'
+ >>> page_match.url_rewrite_exceptions
+ ['help.launchpad.net']
+
+ >>> page_match = PageMatch(
+ ... u'OpenID',
+ ... 'https://help.launchpad.net/OpenID',
+ ... u'Launchpad uses OpenID.')
+ >>> page_match.url
+ 'https://help.launchpad.net/OpenID'
+
+
+-----------------------------
+Graceful handling of timeouts
+-----------------------------
+
+The external service (Bing Search Engine) may not be available, or
+is not responding quickly because there are network issues. In these
+cases a TimeoutError is issued.
+
+ >>> from socket import socket
+ >>> from textwrap import dedent
+ >>> from lp.services.timeout import (
+ ... get_default_timeout_function, set_default_timeout_function)
+
+ >>> server = socket()
+ >>> server.bind(('127.0.0.01', 0))
+ >>> server.listen(1)
+ >>> config.push('timeout_data', dedent("""
+ ... [bing]
+ ... site: http://%s:%d/cse
+ ... """ % server.getsockname()))
+ >>> old_timeout_function = get_default_timeout_function()
+ >>> set_default_timeout_function(lambda: 0.1)
+ >>> bing_search.search(terms='bug')
+ Traceback (most recent call last):
+ ...
+ SiteSearchResponseError: ... timeout exceeded.
+
+ # Restore the configuration and the timeout state.
+ >>> timeout_data = config.pop('timeout_data')
+ >>> set_default_timeout_function(old_timeout_function)
+
+ >>> logger.cleanUp()
=== modified file 'lib/lp/services/sitesearch/doc/google-searchservice.txt'
--- lib/lp/services/sitesearch/doc/google-searchservice.txt 2018-03-27 15:47:35 +0000
+++ lib/lp/services/sitesearch/doc/google-searchservice.txt 2018-03-28 21:29:05 +0000
@@ -23,7 +23,7 @@
>>> from lp.services.sitesearch.interfaces import (
... ISearchService)
- >>> google_search = getUtility(ISearchService)
+ >>> google_search = getUtility(ISearchService, name="google")
>>> verifyObject(ISearchService, google_search)
True
>>> google_search
=== added file 'lib/lp/services/sitesearch/googletestservice.py'
--- lib/lp/services/sitesearch/googletestservice.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/googletestservice.py 2018-03-28 21:29:05 +0000
@@ -0,0 +1,79 @@
+#!/usr/bin/python
+#
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""
+This script runs a simple HTTP server. The server returns XML files
+when given certain user-configurable URLs.
+"""
+
+import logging
+import os
+
+from six.moves.BaseHTTPServer import HTTPServer
+
+from lp.services.config import config
+from lp.services.osutils import ensure_directory_exists
+from lp.services.pidfile import make_pidfile
+from lp.services.sitesearch import testservice
+
+
+# Set up basic logging.
+log = logging.getLogger(__name__)
+
+# The default service name, used by the Launchpad service framework.
+service_name = 'google-webservice'
+
+
+class GoogleRequestHandler(testservice.RequestHandler):
+ default_content_type = 'text/xml; charset=UTF-8'
+ log = log
+ mapfile = config.google_test_service.mapfile
+ content_dir = config.google_test_service.canned_response_directory
+
+
+def start_as_process():
+ return testservice.start_as_process('googletestservice')
+
+
+def get_service_endpoint():
+ """Return the host and port that the service is running on."""
+ return testservice.hostpair(config.google.site)
+
+
+def service_is_available():
+ host, port = get_service_endpoint()
+ return testservice.service_is_available(host, port)
+
+
+def wait_for_service():
+ host, port = get_service_endpoint()
+ return testservice.wait_for_service(host, port)
+
+
+def kill_running_process():
+ global service_name
+ host, port = get_service_endpoint()
+ return testservice.kill_running_process(service_name, host, port)
+
+
+def main():
+ """Run the HTTP server."""
+ # Redirect our service output to a log file.
+ global log
+ ensure_directory_exists(os.path.dirname(config.google_test_service.log))
+ filelog = logging.FileHandler(config.google_test_service.log)
+ log.addHandler(filelog)
+ log.setLevel(logging.DEBUG)
+
+ # To support service shutdown we need to create a PID file that is
+ # understood by the Launchpad services framework.
+ global service_name
+ make_pidfile(service_name)
+
+ host, port = get_service_endpoint()
+ server = HTTPServer((host, port), GoogleRequestHandler)
+
+ log.info("Starting HTTP Google webservice server on port %s", port)
+ server.serve_forever()
=== modified file 'lib/lp/services/sitesearch/interfaces.py'
--- lib/lp/services/sitesearch/interfaces.py 2018-03-26 21:06:51 +0000
+++ lib/lp/services/sitesearch/interfaces.py 2018-03-28 21:29:05 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Interfaces for searching and working with results."""
=== added file 'lib/lp/services/sitesearch/tests/bingserviceharness.py'
--- lib/lp/services/sitesearch/tests/bingserviceharness.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/bingserviceharness.py 2018-03-28 21:29:05 +0000
@@ -0,0 +1,107 @@
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""
+Fixtures for running the Bing test webservice.
+"""
+
+__metaclass__ = type
+
+__all__ = ['BingServiceTestSetup']
+
+
+import errno
+import os
+import signal
+
+from lp.services.sitesearch import bingtestservice
+
+
+class BingServiceTestSetup:
+ """Set up the Bing web service stub for use in functional tests.
+ """
+
+ # XXX gary 2008-12-06 bug=305858: Spurious test failures discovered on
+ # buildbot, builds 40 and 43. The locations of the failures are marked
+ # below with " # SPURIOUS FAILURE". To reinstate, add the text below back
+ # to the docstring above. Note that the test that uses this setup,
+ # bing-service-stub.txt, is also disabled. See test_doc.py.
+ """
+ >>> from lp.services.sitesearch.bingtestservice import (
+ ... service_is_available)
+ >>> from lp.services.config import config
+
+ >>> assert not service_is_available() # Sanity check. # SPURIOUS FAILURE
+
+ >>> BingServiceTestSetup().setUp()
+
+ After setUp is called, a Bing test service instance is running.
+
+ >>> assert service_is_available()
+ >>> assert BingServiceTestSetup.service is not None
+
+ After tearDown is called, the service is shut down.
+
+ >>> BingServiceTestSetup().tearDown()
+
+ >>> assert not service_is_available()
+ >>> assert BingServiceTestSetup.service is None
+
+ The fixture can be started and stopped multiple time in succession:
+
+ >>> BingServiceTestSetup().setUp()
+ >>> assert service_is_available()
+
+ Having a service instance already running doesn't prevent a new
+ service from starting. The old instance is killed off and replaced
+ by the new one.
+
+ >>> old_pid = BingServiceTestSetup.service.pid
+ >>> BingServiceTestSetup().setUp() # SPURIOUS FAILURE
+ >>> BingServiceTestSetup.service.pid != old_pid
+ True
+
+ Tidy up.
+
+ >>> BingServiceTestSetup().tearDown()
+ >>> assert not service_is_available()
+
+ """
+
+ service = None # A reference to our running service.
+
+ def setUp(self):
+ self.startService()
+
+ def tearDown(self):
+ self.stopService()
+
+ @classmethod
+ def startService(cls):
+ """Start the webservice."""
+ bingtestservice.kill_running_process()
+ cls.service = bingtestservice.start_as_process()
+ assert cls.service, "The Search service process did not start."
+ try:
+ bingtestservice.wait_for_service()
+ except RuntimeError:
+ # The service didn't start itself soon enough. We must
+ # make sure to kill any errant processes that may be
+ # hanging around.
+ cls.stopService()
+ raise
+
+ @classmethod
+ def stopService(cls):
+ """Shut down the webservice instance."""
+ if cls.service:
+ try:
+ os.kill(cls.service.pid, signal.SIGTERM)
+ except OSError as error:
+ if error.errno != errno.ESRCH:
+ raise
+ # The process with the given pid doesn't exist, so there's
+ # nothing to kill or wait for.
+ else:
+ cls.service.wait()
+ cls.service = None
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-1.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-1.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-1.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,384 @@
+{
+ "_type": "SearchResponse",
+ "instrumentation": {
+ "pingUrlBase": "https://www.bingapis.com/api/ping?IG=4C3A184771914F6BB81F8541E7EE7695&CID=13045103CBA569B80D475AB7CA03687C&ID=";,
+ "pageLoadPingUrl": "https://www.bingapis.com/api/ping/pageload?IG=4C3A184771914F6BB81F8541E7EE7695&CID=13045103CBA569B80D475AB7CA03687C&Type=Event.CPT&DATA=0";
+ },
+ "queryContext": {
+ "originalQuery": "bug"
+ },
+ "webPages": {
+ "webSearchUrl": "https://www.bing.com/search?q=bug";,
+ "webSearchUrlPingSuffix": "DevEx,5462.1",
+ "totalEstimatedMatches": 25,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";,
+ "name": "Launchpad Bugs",
+ "url": "https://launchpad.net/~ubuntu-bugs";,
+ "urlPingSuffix": "DevEx,5080.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/~ubuntu-bugs";,
+ "snippet": "The purpose of this team is to have a mailing list that receives all Ubuntu bug mail. If you wish to receive copies of all bug traffic, instead subscribe to the ...",
+ "dateLastCrawled": "2018-03-10T00:32:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";,
+ "name": "Bugs in Ubuntu Linux",
+ "url": "https://launchpad.net/gcleaner";,
+ "urlPingSuffix": "DevEx,5095.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/gcleaner";,
+ "snippet": "GCleaner is a beautiful and fast system cleaner for Elementary OS and Ubuntu or based. Writen in Vala, GTK+, Granite and GLib/GIO for the purpose that the users feel ...",
+ "dateLastCrawled": "2018-02-21T13:17:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";,
+ "name": "Bugs related to Sample Person",
+ "url": "https://help.launchpad.net/BugExpiry";,
+ "urlPingSuffix": "DevEx,5110.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/BugExpiry";,
+ "snippet": "Want to know more about Bug Statuses? Check out our BugStatuses page. What is bug expiry? Today, projects and distributions have the option of allowing old ...",
+ "dateLastCrawled": "2018-01-13T13:58:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";,
+ "name": "Bug #1 in Mozilla Firefox: Firefox does not support SVG",
+ "url": "https://launchpad.net/longomatch";,
+ "urlPingSuffix": "DevEx,5125.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/longomatch";,
+ "snippet": "LongoMatch is a sports video analyse tool for coaches to assist them on making live video reports from a match. It creates a database with the most important plays of ...",
+ "dateLastCrawled": "2018-02-16T21:54:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";,
+ "name": "Question #232632 : Questions : OpenStack Heat",
+ "url": "https://answers.launchpad.net/heat/+question/232632";,
+ "urlPingSuffix": "DevEx,5140.1",
+ "datePublished": "2013-07-18T00:00:00.0000000",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://answers.launchpad.net/heat/+question/232632";,
+ "snippet": "Using grizzly version(2013.1.2) from heat/openstack. Failed to create stack, got error: 'NoneType' object has no attribute 'rstrip'. See below my runtime ...",
+ "dateLastCrawled": "2018-02-20T23:45:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.5";,
+ "name": "Sandpad in Launchpad",
+ "url": "https://launchpad.net/sandpad";,
+ "urlPingSuffix": "DevEx,5154.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/sandpad";,
+ "snippet": "Sandpad is an standalone wlua application for making quick scratch programs and sandboxes in Lua. It uses IUP 3 for the user interface.",
+ "dateLastCrawled": "2018-03-03T17:54:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.6";,
+ "name": "Eventum in Launchpad",
+ "url": "https://launchpad.net/eventum";,
+ "urlPingSuffix": "DevEx,5169.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/eventum";,
+ "snippet": "Eventum is a user-friendly and flexible issue tracking system that can be used by a support department to track incoming technical support requests, or by a software ...",
+ "dateLastCrawled": "2018-03-07T19:52:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.7";,
+ "name": "Inkscape 0.48.2 \"0.48.2\" - Launchpad",
+ "url": "https://launchpad.net/inkscape/+milestone/0.48.2";,
+ "urlPingSuffix": "DevEx,5182.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/inkscape/+milestone/0.48.2";,
+ "snippet": "After you've downloaded a file, you can verify its authenticity using its MD5 sum or signature. (How do I verify a download?",
+ "dateLastCrawled": "2018-02-24T19:01:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.8";,
+ "name": "Code Hosting - Launchpad tour",
+ "url": "https://launchpad.net/+tour/branch-hosting-tracking";,
+ "urlPingSuffix": "DevEx,5195.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/+tour/branch-hosting-tracking";,
+ "snippet": "Code hosting and review. Launchpad and Bazaar make it easy for anyone to get your project's code, make their own changes with full version control, and then propose ...",
+ "dateLastCrawled": "2018-03-10T23:01:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.9";,
+ "name": "gdiskdump project files : gdiskdump",
+ "url": "https://launchpad.net/gdiskdump/+download";,
+ "urlPingSuffix": "DevEx,5208.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/gdiskdump/+download";,
+ "snippet": "added advanced settings and estimated Time for process to finish. bug fixes for some languages.",
+ "dateLastCrawled": "2018-02-19T11:52:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.10";,
+ "name": "Bug #1747091 “epicsTimeGetEvent() / generalTime bug ...",
+ "url": "https://code.launchpad.net/bugs/1747091";,
+ "urlPingSuffix": "DevEx,5220.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://code.launchpad.net/bugs/1747091";,
+ "snippet": "When changes in epicsGeneralTime.c were made (fetch time provider's eventTime into a local copy) an inconsistency resulted.",
+ "dateLastCrawled": "2018-02-04T02:49:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.11";,
+ "name": "Bug #96441 “timer updates use too much CPU” : Bugs : Jokosher",
+ "url": "https://launchpad.net/jokosher/+bug/96441";,
+ "urlPingSuffix": "DevEx,5232.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/jokosher/+bug/96441";,
+ "snippet": "The information about this bug in Launchpad is automatically pulled daily from the remote bug. This information was last pulled ...",
+ "dateLastCrawled": "2018-02-17T16:53:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.12";,
+ "name": "Series 17.10 : Mahara",
+ "url": "https://launchpad.net/mahara/17.10";,
+ "urlPingSuffix": "DevEx,5247.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/mahara/17.10";,
+ "snippet": "All bugs Latest bugs reported. Bug #1752688: MariaDB fails to upgrade - unable to CAST as JSON Reported on 2018-03-01 Bug #1752442: Problems with group forums / topics",
+ "dateLastCrawled": "2018-03-12T13:48:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.13";,
+ "name": "sap in Launchpad",
+ "url": "https://launchpad.net/sap+";,
+ "urlPingSuffix": "DevEx,5260.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/sap+";,
+ "snippet": "Sap is a simple audio player written in Vala and utilizing gstreamer for audio playback and ncurses for user interaction.",
+ "dateLastCrawled": "2018-03-12T05:18:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.14";,
+ "name": "Code/BugAndBlueprintLinks - Launchpad Help",
+ "url": "https://help.launchpad.net/Code/BugAndBlueprintLinks";,
+ "urlPingSuffix": "DevEx,5275.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/Code/BugAndBlueprintLinks";,
+ "snippet": "Linking code to bug reports and blueprints. Launchpad is much like a fancy china dinner service: you can get a great deal of use and contentment from just one or two ...",
+ "dateLastCrawled": "2018-01-26T14:46:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.15";,
+ "name": "One Hundred Papercuts in Launchpad",
+ "url": "https://launchpad.net/hundredpapercuts";,
+ "urlPingSuffix": "DevEx,5289.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/hundredpapercuts";,
+ "snippet": "Downloads. One Hundred Papercuts does not have any download files registered with Launchpad. •",
+ "dateLastCrawled": "2018-03-01T20:05:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.16";,
+ "name": "Little Software Stats in Launchpad",
+ "url": "https://launchpad.net/lilsoftstats/";,
+ "urlPingSuffix": "DevEx,5304.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/lilsoftstats";,
+ "snippet": "Little Software Stats is the first free and open source program that will allow software developers to keep track of how their software is being used.",
+ "dateLastCrawled": "2018-03-07T21:46:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.17";,
+ "name": "Take the tour - Launchpad tour",
+ "url": "https://launchpad.net/+tour";,
+ "urlPingSuffix": "DevEx,5317.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/+tour";,
+ "snippet": "Launchpad helps people to work together on free software by making it easy to share code, bug reports, translations and ideas.",
+ "dateLastCrawled": "2018-02-25T04:37:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.18";,
+ "name": "EOEC in Launchpad",
+ "url": "https://launchpad.net/eoec";,
+ "urlPingSuffix": "DevEx,5332.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/eoec";,
+ "snippet": "Barcode example published as standalone extension on 2009-01-15 After adding a number of extra features to the Barcode example of EuroOffice ... EOEC ...",
+ "dateLastCrawled": "2018-03-09T21:39:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.19";,
+ "name": "Bug #1210747 “taskdef name not specified” : Bugs : Radkarte",
+ "url": "https://launchpad.net/radkarte/+bug/1210747";,
+ "urlPingSuffix": "DevEx,5345.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/radkarte/+bug/1210747";,
+ "snippet": "Sorry for the late response, somehow the notification mail did not work. I use the \"ant-contrib\" package for the ID calculation to prepare the build-file for easy ...",
+ "dateLastCrawled": "2018-01-11T22:23:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ },
+ "rankingResponse": {
+ "mainline": {
+ "items": [
+ {
+ "answerType": "WebPages",
+ "resultIndex": 0,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 1,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 2,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 3,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 4,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 5,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.5";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 6,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.6";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 7,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.7";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 8,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.8";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 9,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.9";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 10,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.10";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 11,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.11";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 12,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.12";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 13,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.13";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 14,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.14";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 15,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.15";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 16,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.16";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 17,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.17";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 18,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.18";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 19,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.19";
+ }
+ }
+ ]
+ }
+ }
+}
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-2.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-2.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-2.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,151 @@
+{
+ "_type": "SearchResponse",
+ "instrumentation": {
+ "pingUrlBase": "https://www.bingapis.com/api/ping?IG=41744312F5E645D58DE6733982EDC72A&CID=222E331334AA69A5318C38A7350C683B&ID=";,
+ "pageLoadPingUrl": "https://www.bingapis.com/api/ping/pageload?IG=41744312F5E645D58DE6733982EDC72A&CID=222E331334AA69A5318C38A7350C683B&Type=Event.CPT&DATA=0";
+ },
+ "queryContext": {
+ "originalQuery": "bug"
+ },
+ "webPages": {
+ "webSearchUrl": "https://www.bing.com/search?q=bug";,
+ "webSearchUrlPingSuffix": "DevEx,5530.1",
+ "totalEstimatedMatches": 25,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";,
+ "name": "Bugs - Launchpad Help",
+ "url": "https://help.launchpad.net/Bugs";,
+ "urlPingSuffix": "DevEx,5103.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/Bugs";,
+ "snippet": "Launchpad Help > Bugs . Use Launchpad's bug tracker for your project. Bug heat: a computed estimate of a bug's significance. Learn about automatic bug expiry",
+ "deepLinks": [
+ {
+ "name": "Bugs/EmailInterface",
+ "url": "https://help.launchpad.net/Bugs/EmailInterface";,
+ "urlPingSuffix": "DevEx,5093.1",
+ "snippet": "Overview. Launchpad's bug tracker sends you email about the bugs you're interested in. If you see something that requires your attention - for example, you want to ..."
+ },
+ {
+ "name": "Bugs/PluginAPISpec",
+ "url": "https://help.launchpad.net/Bugs/PluginAPISpec";,
+ "urlPingSuffix": "DevEx,5094.1",
+ "snippet": "Overview. We want Launchpad to share bug reports, comments, statuses and other information with as many bug trackers as possible. We've already produced plugins that ..."
+ },
+ {
+ "name": "Bugs/Subscriptions",
+ "url": "https://help.launchpad.net/Bugs/Subscriptions";,
+ "urlPingSuffix": "DevEx,5095.1",
+ "snippet": "Overview. Launchpad uses notification emails and Atom feeds to help you stay on top of the bugs that interest you. Bug mail. There are three ways to get bug ..."
+ },
+ {
+ "name": "Bugs/YourProject",
+ "url": "https://help.launchpad.net/Bugs/YourProject";,
+ "urlPingSuffix": "DevEx,5096.1",
+ "snippet": "Overview. Launchpad's bug tracker is unique: it can track how one bug affects different communities. When you share free software, you share its bugs."
+ },
+ {
+ "name": "Bugs/Expiry",
+ "url": "https://help.launchpad.net/Bugs/Expiry";,
+ "urlPingSuffix": "DevEx,5097.1",
+ "snippet": "This gives you three benefits: you can view a report of all bugs that are due to expiry and so deal with any that need your attention ; once the bug's expired ..."
+ }
+ ],
+ "dateLastCrawled": "2018-02-20T23:45:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";,
+ "name": "Of Bugs and Statuses - Launchpad Blog",
+ "url": "https://blog.launchpad.net/general/of-bugs-and-statuses";,
+ "urlPingSuffix": "DevEx,5149.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://blog.launchpad.net/general/of-bugs-and-statuses";,
+ "snippet": "This past week’s Bug Janitor thread has made it clear that we need better descriptions of what our bug statuses actually mean. While it’s true that many project ...",
+ "dateLastCrawled": "2018-03-09T07:40:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";,
+ "name": "Mahara 1.8.0",
+ "url": "https://launchpad.net/mahara/+milestone/1.8.0";,
+ "urlPingSuffix": "DevEx,5178.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/mahara/+milestone/1.8.0";,
+ "snippet": "Mahara 1.8.0 Release Notes. This is a stable release of Mahara 1.8. Stable releases are fit for general use. If you find a bug, please report it to the tracker:",
+ "dateLastCrawled": "2018-03-04T10:53:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";,
+ "name": "Mighty Box in Launchpad",
+ "url": "https://launchpad.net/mb";,
+ "urlPingSuffix": "DevEx,5193.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/mb";,
+ "snippet": "All bugs Latest bugs reported. Bug #766411: The settings dialog box is not available on Windows 7 Reported on 2011-04-19 Bug #766410: App crashes when navigating file ...",
+ "dateLastCrawled": "2018-03-10T07:57:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";,
+ "name": "Bug tracking - Launchpad Bugs",
+ "url": "https://launchpad.net/bugs";,
+ "urlPingSuffix": "DevEx,5208.1",
+ "about": [
+ {
+ "name": "Launchpad"
+ }
+ ],
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/bugs";,
+ "snippet": "Statistics. 1755185 bugs reported across 12480 projects including 122769 links to 2796 bug trackers; 148160 bugs are shared across multiple projects",
+ "dateLastCrawled": "2018-03-14T01:18:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ },
+ "rankingResponse": {
+ "mainline": {
+ "items": [
+ {
+ "answerType": "WebPages",
+ "resultIndex": 0,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 1,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 2,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 3,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 4,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";
+ }
+ }
+ ]
+ }
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-error.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-error.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-error.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,20 @@
+{
+ "_type": "ErrorResponse",
+ "errors": [
+ {
+ "code": "InvalidRequest",
+ "subCode": "ParameterInvalidValue",
+ "message": "Parameter has invalid value.",
+ "parameter": "count",
+ "value": "-1"
+ },
+ {
+ "code": "InvalidRequest",
+ "subCode": "ParameterInvalidValue",
+ "message": "Parameter has invalid value.",
+ "parameter": "offset",
+ "value": "a"
+ }
+ ]
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-incompatible-matches.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-incompatible-matches.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-incompatible-matches.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,8 @@
+{
+ "_type": "SearchResponse",
+ "webPages": {
+ "totalEstimatedMatches": "~25",
+ "value": []
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-incomplete-response.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-incomplete-response.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-incomplete-response.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,1 @@
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt 2018-03-28 21:29:05 +0000
@@ -0,0 +1,26 @@
+# This file defines a mapping of Bing search service URLs to the JSON
+# files that should be returned by them.
+#
+# The format is 'url JSONfile'. Blank lines and lines starting with '#'
+# are ignored.
+#
+# The special URL, '*', is returned for all un-mapped URLs.
+
+* bingsearchservice-no-results.json
+
+/bingcustomsearch/v7.0/search?count=20&customConfig=1234567890&mkt=en-US&offset=0&q=bug bingsearchservice-bugs-1.json
+
+/bingcustomsearch/v7.0/search?count=20&customConfig=1234567890&mkt=en-US&offset=20&q=bug bingsearchservice-bugs-2.json
+
+/bingcustomsearch/v7.0/search?count=20&customConfig=1234567890&mkt=en-US&offset=0&q=launchpad bingsearchservice-bugs-1.json
+
+/bingcustomsearch/v7.0/search?count=20&customConfig=1234567890&mkt=en-US&offset=20&q=launchpad bingsearchservice-bugs-2.json
+
+/bingcustomsearch/v7.0/search?count=20&customConfig=1234567890&mkt=en-US&offset=0&q=gnomebaker bingsearchservice-incomplete-response.json
+
+/bingcustomsearch/v7.0/search?count=20&customConfig=1234567890&mkt=en-US&offset=0&q=no-meaningful bingsearchservice-no-meaningful-results.json
+
+/bingcustomsearch/v7.0/search?count=20&customConfig=1234567890&mkt=en-US&offset=0&q=errors-please bingsearchservice-error.json
+
+# This stub service is also used to impersonate the Blog feed
+/blog-feed blog.launchpad.net-feed.json
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-summary.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-summary.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-summary.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,30 @@
+{
+ "_type": "SearchResponse",
+ "webPages": {
+ "totalEstimatedMatches": -25,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";,
+ "name": "Ubuntu Bugs in Launchpad",
+ "url": "https://launchpad.net/~ubuntu-bugs";,
+ "urlPingSuffix": "DevEx,5080.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/~ubuntu-bugs";,
+ "dateLastCrawled": "2018-03-10T00:32:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";,
+ "name": "BugExpiry - Launchpad Help",
+ "url": "https://help.launchpad.net/BugExpiry";,
+ "urlPingSuffix": "DevEx,5110.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/BugExpiry";,
+ "snippet": "Want to know more about Bug Statuses? Check out our BugStatuses page. What is bug expiry? Today, projects and distributions have the option of allowing old ...",
+ "dateLastCrawled": "2018-01-13T13:58:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-title.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-title.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-title.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,30 @@
+{
+ "_type": "SearchResponse",
+ "webPages": {
+ "totalEstimatedMatches": -25,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";,
+ "url": "https://launchpad.net/~ubuntu-bugs";,
+ "urlPingSuffix": "DevEx,5080.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/~ubuntu-bugs";,
+ "snippet": "The purpose of this team is to have a mailing list that receives all Ubuntu bug mail. If you wish to receive copies of all bug traffic, instead subscribe to the ...",
+ "dateLastCrawled": "2018-03-10T00:32:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";,
+ "name": "GCleaner in Launchpad",
+ "url": "https://launchpad.net/gcleaner";,
+ "urlPingSuffix": "DevEx,5095.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/gcleaner";,
+ "snippet": "GCleaner is a beautiful and fast system cleaner for Elementary OS and Ubuntu or based. Writen in Vala, GTK+, Granite and GLib/GIO for the purpose that the users feel ...",
+ "dateLastCrawled": "2018-02-21T13:17:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-url.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-url.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-missing-url.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,30 @@
+{
+ "_type": "SearchResponse",
+ "webPages": {
+ "totalEstimatedMatches": -25,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";,
+ "name": "Ubuntu Bugs in Launchpad",
+ "urlPingSuffix": "DevEx,5080.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/~ubuntu-bugs";,
+ "snippet": "The purpose of this team is to have a mailing list that receives all Ubuntu bug mail. If you wish to receive copies of all bug traffic, instead subscribe to the ...",
+ "dateLastCrawled": "2018-03-10T00:32:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";,
+ "name": "LongoMatch in Launchpad",
+ "url": "https://launchpad.net/longomatch";,
+ "urlPingSuffix": "DevEx,5125.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/longomatch";,
+ "snippet": "LongoMatch is a sports video analyse tool for coaches to assist them on making live video reports from a match. It creates a database with the most important plays of ...",
+ "dateLastCrawled": "2018-02-16T21:54:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-negative-total.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-negative-total.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-negative-total.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,8 @@
+{
+ "_type": "SearchResponse",
+ "webPages": {
+ "totalEstimatedMatches": -25,
+ "value": []
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-no-meaningful-results.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-no-meaningful-results.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-no-meaningful-results.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,19 @@
+{
+ "_type": "SearchResponse",
+ "webPages": {
+ "totalEstimatedMatches": 25,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";,
+ "url": "https://help.launchpad.net/BugExpiry";,
+ "urlPingSuffix": "DevEx,5110.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/BugExpiry";,
+ "snippet": "Want to know more about Bug Statuses? Check out our BugStatuses page. What is bug expiry? Today, projects and distributions have the option of allowing old ...",
+ "dateLastCrawled": "2018-01-13T13:58:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-no-results.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-no-results.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-no-results.json 2018-03-28 21:29:05 +0000
@@ -0,0 +1,12 @@
+{
+ "_type": "SearchResponse",
+ "instrumentation": {
+ "pingUrlBase": "https://www.bingapis.com/api/ping?IG=74D50DC66A774E74AF486941DDF2C89C&CID=2023A1AFC6086AE52F08AA1BC7476B1F&ID=";,
+ "pageLoadPingUrl": "https://www.bingapis.com/api/ping/pageload?IG=74D50DC66A774E74AF486941DDF2C89C&CID=2023A1AFC6086AE52F08AA1BC7476B1F&Type=Event.CPT&DATA=0";
+ },
+ "queryContext": {
+ "originalQuery": "AELymgURIr4plE6V5qUaesxj1S8kUSFxCVrVLNU_OeCogh9Q7W5be6lEGMcbb0q6WTDgLL7zsnlnYGLvVrsdxgx3AamFm0M6ARaxerSLvSf-1JQHrOLuhsQ"
+ },
+ "rankingResponse": {}
+}
+
=== added file 'lib/lp/services/sitesearch/tests/test_bing.py'
--- lib/lp/services/sitesearch/tests/test_bing.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/test_bing.py 2018-03-28 21:29:05 +0000
@@ -0,0 +1,118 @@
+# Copyright 2011-2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test the bing search service."""
+
+__metaclass__ = type
+
+
+from fixtures import MockPatch
+from requests.exceptions import (
+ ConnectionError,
+ HTTPError,
+ )
+
+from lp.services.sitesearch import BingSearchService
+from lp.services.sitesearch.interfaces import SiteSearchResponseError
+from lp.services.timeout import TimeoutError
+from lp.testing import TestCase
+from lp.testing.layers import (
+ BingLaunchpadFunctionalLayer,
+ LaunchpadFunctionalLayer,
+ )
+
+
+class TestBingSearchService(TestCase):
+ """Test BingSearchService."""
+
+ layer = LaunchpadFunctionalLayer
+
+ def setUp(self):
+ super(TestBingSearchService, self).setUp()
+ self.search_service = BingSearchService()
+
+ def test_search_converts_HTTPError(self):
+ # The method converts HTTPError to SiteSearchResponseError.
+ args = ('url', 500, 'oops', {}, None)
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch', side_effect=HTTPError(*args)))
+ self.assertRaises(
+ SiteSearchResponseError, self.search_service.search, 'fnord')
+
+ def test_search_converts_ConnectionError(self):
+ # The method converts ConnectionError to SiteSearchResponseError.
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch',
+ side_effect=ConnectionError('oops')))
+ self.assertRaises(
+ SiteSearchResponseError, self.search_service.search, 'fnord')
+
+ def test_search_converts_TimeoutError(self):
+ # The method converts TimeoutError to SiteSearchResponseError.
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch',
+ side_effect=TimeoutError('oops')))
+ self.assertRaises(
+ SiteSearchResponseError, self.search_service.search, 'fnord')
+
+ def test_parse_bing_response_TypeError(self):
+ # The method converts TypeError to SiteSearchResponseError.
+ self.assertRaises(
+ SiteSearchResponseError,
+ self.search_service._parse_bing_response, None)
+
+ def test_parse_bing_response_ValueError(self):
+ # The method converts ValueError to SiteSearchResponseError.
+ self.assertRaises(
+ SiteSearchResponseError,
+ self.search_service._parse_bing_response, '')
+
+ def test_parse_bing_response_KeyError(self):
+ # The method converts KeyError to SiteSearchResponseError.
+ self.assertRaises(
+ SiteSearchResponseError,
+ self.search_service._parse_bing_response, '{}')
+
+
+class FunctionalTestBingSearchService(TestCase):
+ """Test BingSearchService."""
+
+ layer = BingLaunchpadFunctionalLayer
+
+ def setUp(self):
+ super(FunctionalTestBingSearchService, self).setUp()
+ self.search_service = BingSearchService()
+
+ def test_search_with_results(self):
+ matches = self.search_service.search('bug')
+ self.assertEqual(0, matches.start)
+ self.assertEqual(25, matches.total)
+ self.assertEqual(20, len(matches))
+
+ def test_search_with_results_and_offset(self):
+ matches = self.search_service.search('bug', start=20)
+ self.assertEqual(20, matches.start)
+ self.assertEqual(25, matches.total)
+ self.assertEqual(5, len(matches))
+
+ def test_search_no_results(self):
+ matches = self.search_service.search('fnord')
+ self.assertEqual(0, matches.start)
+ self.assertEqual(0, matches.total)
+ self.assertEqual(0, len(matches))
+
+ def test_search_no_meaningful_results(self):
+ matches = self.search_service.search('no-meaningful')
+ self.assertEqual(0, matches.start)
+ self.assertEqual(25, matches.total)
+ self.assertEqual(0, len(matches))
+
+ def test_search_incomplete_response(self):
+ self.assertRaises(
+ SiteSearchResponseError,
+ self.search_service.search, 'gnomebaker')
+
+ def test_search_error_response(self):
+ self.assertRaises(
+ SiteSearchResponseError,
+ self.search_service.search, 'errors-please')
=== added file 'lib/lp/services/sitesearch/tests/test_bingharness.py'
--- lib/lp/services/sitesearch/tests/test_bingharness.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/test_bingharness.py 2018-03-28 21:29:05 +0000
@@ -0,0 +1,10 @@
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import doctest
+
+
+def test_suite():
+ return doctest.DocTestSuite(
+ 'lp.services.sitesearch.tests.bingserviceharness',
+ optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS)
=== added file 'lib/lp/services/sitesearch/tests/test_bingservice.py'
--- lib/lp/services/sitesearch/tests/test_bingservice.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/test_bingservice.py 2018-03-28 21:29:05 +0000
@@ -0,0 +1,38 @@
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""
+Unit tests for the Bing test service stub.
+"""
+
+__metaclass__ = type
+
+
+import os
+import unittest
+
+from lp.services.osutils import process_exists
+from lp.services.pidfile import pidfile_path
+from lp.services.sitesearch import bingtestservice
+
+
+class TestServiceUtilities(unittest.TestCase):
+ """Test the service's supporting functions."""
+
+ def test_stale_pid_file_cleanup(self):
+ """The service should be able to clean up invalid PID files."""
+ bogus_pid = 9999999
+ self.assertFalse(
+ process_exists(bogus_pid),
+ "There is already a process with PID '%d'." % bogus_pid)
+
+ # Create a stale/bogus PID file.
+ filepath = pidfile_path(bingtestservice.service_name)
+ with file(filepath, 'w') as pidfile:
+ pidfile.write(str(bogus_pid))
+
+ # The PID clean-up code should silently remove the file and return.
+ bingtestservice.kill_running_process()
+ self.assertFalse(
+ os.path.exists(filepath),
+ "The PID file '%s' should have been deleted." % filepath)
=== modified file 'lib/lp/services/sitesearch/tests/test_doc.py'
--- lib/lp/services/sitesearch/tests/test_doc.py 2012-01-01 02:58:52 +0000
+++ lib/lp/services/sitesearch/tests/test_doc.py 2018-03-28 21:29:05 +0000
@@ -9,6 +9,7 @@
from lp.services.testing import build_test_suite
from lp.testing.layers import (
+ BingLaunchpadFunctionalLayer,
DatabaseFunctionalLayer,
GoogleLaunchpadFunctionalLayer,
)
@@ -23,6 +24,10 @@
special = {
+ 'bing-searchservice.txt': LayeredDocFileSuite(
+ '../doc/bing-searchservice.txt',
+ setUp=setUp, tearDown=tearDown,
+ layer=BingLaunchpadFunctionalLayer,),
'google-searchservice.txt': LayeredDocFileSuite(
'../doc/google-searchservice.txt',
setUp=setUp, tearDown=tearDown,
=== modified file 'lib/lp/services/sitesearch/tests/test_google.py'
--- lib/lp/services/sitesearch/tests/test_google.py 2018-03-26 21:06:51 +0000
+++ lib/lp/services/sitesearch/tests/test_google.py 2018-03-28 21:29:05 +0000
@@ -5,8 +5,8 @@
__metaclass__ = type
-from contextlib import contextmanager
+from fixtures import MockPatch
from requests.exceptions import (
ConnectionError,
HTTPError,
@@ -19,26 +19,6 @@
from lp.testing.layers import LaunchpadFunctionalLayer
-@contextmanager
-def urlfetch_exception(test_error, *args):
- """Raise an error during the execution of urlfetch.
-
- This function replaces urlfetch() with a function that
- raises an error.
- """
-
- def raise_exception(url):
- raise test_error(*args)
-
- from lp.services import timeout
- original_urlfetch = timeout.urlfetch
- timeout.urlfetch = raise_exception
- try:
- yield
- finally:
- timeout.urlfetch = original_urlfetch
-
-
class TestGoogleSearchService(TestCase):
"""Test GoogleSearchService."""
@@ -51,35 +31,43 @@
def test_search_converts_HTTPError(self):
# The method converts HTTPError to SiteSearchResponseError.
args = ('url', 500, 'oops', {}, None)
- with urlfetch_exception(HTTPError, *args):
- self.assertRaises(
- SiteSearchResponseError, self.search_service.search, 'fnord')
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch', side_effect=HTTPError(*args)))
+ self.assertRaises(
+ SiteSearchResponseError, self.search_service.search, 'fnord')
def test_search_converts_ConnectionError(self):
# The method converts ConnectionError to SiteSearchResponseError.
- with urlfetch_exception(ConnectionError, 'oops'):
- self.assertRaises(
- SiteSearchResponseError, self.search_service.search, 'fnord')
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch',
+ side_effect=ConnectionError('oops')))
+ self.assertRaises(
+ SiteSearchResponseError, self.search_service.search, 'fnord')
def test_search_converts_TimeoutError(self):
# The method converts TimeoutError to SiteSearchResponseError.
- with urlfetch_exception(TimeoutError, 'oops'):
- self.assertRaises(
- SiteSearchResponseError, self.search_service.search, 'fnord')
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch',
+ side_effect=TimeoutError('oops')))
+ self.assertRaises(
+ SiteSearchResponseError, self.search_service.search, 'fnord')
def test___parse_google_search_protocol_SyntaxError(self):
# The method converts SyntaxError to SiteSearchResponseError.
- with urlfetch_exception(SyntaxError, 'oops'):
- self.assertRaises(
- SiteSearchResponseError,
- self.search_service._parse_google_search_protocol, '')
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch',
+ side_effect=SyntaxError('oops')))
+ self.assertRaises(
+ SiteSearchResponseError,
+ self.search_service._parse_google_search_protocol, '')
def test___parse_google_search_protocol_IndexError(self):
# The method converts IndexError to SiteSearchResponseError.
- with urlfetch_exception(IndexError, 'oops'):
- data = (
- '<?xml version="1.0" encoding="UTF-8" standalone="no"?>'
- '<GSP VER="3.2"></GSP>')
- self.assertRaises(
- SiteSearchResponseError,
- self.search_service._parse_google_search_protocol, data)
+ self.useFixture(MockPatch(
+ 'lp.services.sitesearch.urlfetch', side_effect=IndexError('oops')))
+ data = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="no"?>'
+ '<GSP VER="3.2"></GSP>')
+ self.assertRaises(
+ SiteSearchResponseError,
+ self.search_service._parse_google_search_protocol, data)
=== renamed file 'lib/lp/services/sitesearch/googletestservice.py' => 'lib/lp/services/sitesearch/testservice.py'
--- lib/lp/services/sitesearch/googletestservice.py 2018-03-26 21:22:03 +0000
+++ lib/lp/services/sitesearch/testservice.py 2018-03-28 21:29:05 +0000
@@ -1,54 +1,37 @@
#!/usr/bin/python
#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""
-This script runs a simple HTTP server. The server returns XML files
+This script runs a simple HTTP server. The server returns files
when given certain user-configurable URLs.
"""
import errno
-import logging
import os
import signal
import socket
import subprocess
import time
-from six.moves.BaseHTTPServer import (
- BaseHTTPRequestHandler,
- HTTPServer,
- )
+from six.moves.BaseHTTPServer import BaseHTTPRequestHandler
-from lp.services.config import config
-from lp.services.osutils import (
- ensure_directory_exists,
- remove_if_exists,
- )
+from lp.services.osutils import remove_if_exists
from lp.services.pidfile import (
get_pid,
- make_pidfile,
pidfile_path,
)
from lp.services.webapp.url import urlsplit
-# Set up basic logging.
-log = logging.getLogger(__name__)
-
-# The default service name, used by the Launchpad service framework.
-service_name = 'google-webservice'
-
-
-class GoogleRequestHandler(BaseHTTPRequestHandler):
- """Return an XML file depending on the requested URL."""
-
- default_content_type = 'text/xml; charset=UTF-8'
+
+class RequestHandler(BaseHTTPRequestHandler):
+ """Return a file depending on the requested URL."""
def do_GET(self):
"""See BaseHTTPRequestHandler in the Python Standard Library."""
- urlmap = url_to_xml_map()
+ urlmap = url_to_file_map(self.mapfile)
if self.path in urlmap:
self.return_file(urlmap[self.path])
else:
@@ -65,8 +48,7 @@
self.send_header('Content-Type', self.default_content_type)
self.end_headers()
- content_dir = config.google_test_service.canned_response_directory
- filepath = os.path.join(content_dir, filename)
+ filepath = os.path.join(self.content_dir, filename)
content_body = file(filepath).read()
self.wfile.write(content_body)
@@ -78,12 +60,11 @@
(self.address_string(),
self.log_date_time_string(),
format % args))
- log.info(message)
-
-
-def url_to_xml_map():
- """Return our URL-to-XML mapping as a dictionary."""
- mapfile = config.google_test_service.mapfile
+ self.log.info(message)
+
+
+def url_to_file_map(mapfile):
+ """Return our URL-to-file mapping as a dictionary."""
mapping = {}
for line in file(mapfile):
if line.startswith('#') or len(line.strip()) == 0:
@@ -95,12 +76,7 @@
return mapping
-def get_service_endpoint():
- """Return the host and port that the service is running on."""
- return hostpair(config.google.site)
-
-
-def service_is_available(timeout=2.0):
+def service_is_available(host, port, timeout=2.0):
"""Return True if the service is up and running.
:param timeout: BLOCK execution for at most 'timeout' seconds
@@ -108,7 +84,6 @@
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(timeout) # Block for 'timeout' seconds.
- host, port = get_service_endpoint()
try:
try:
sock.connect((host, port))
@@ -120,14 +95,13 @@
sock.close() # Clean up.
-def wait_for_service(timeout=15.0):
+def wait_for_service(host, port, timeout=15.0):
"""Poll the service and BLOCK until we can connect to it.
:param timeout: The socket should timeout after this many seconds.
Refer to the socket module documentation in the Standard Library
for possible timeout values.
"""
- host, port = get_service_endpoint()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(timeout) # Block for at most X seconds.
@@ -150,7 +124,7 @@
sock.close() # Clean up.
-def wait_for_service_shutdown(seconds_to_wait=10.0):
+def wait_for_service_shutdown(host, port, seconds_to_wait=10.0):
"""Poll the service until it shuts down.
Raises a RuntimeError if the service doesn't shut down within the allotted
@@ -160,8 +134,6 @@
:param seconds_to_wait: The number of seconds to wait for the socket to
open up.
"""
- host, port = get_service_endpoint()
-
start = time.time() # Record when we started polling.
try:
while True:
@@ -194,7 +166,7 @@
return (host, port)
-def start_as_process():
+def start_as_process(service_binary_name):
"""Run this file as a stand-alone Python script.
Returns a subprocess.Popen object. (See the `subprocess` module in
@@ -203,7 +175,7 @@
script = os.path.join(
os.path.dirname(__file__),
os.pardir, os.pardir, os.pardir, os.pardir, 'bin',
- 'googletestservice')
+ service_binary_name)
# Make sure we aren't using the parent stdin and stdout to avoid spam
# and have fewer things that can go wrong shutting down the process.
proc = subprocess.Popen(
@@ -213,9 +185,8 @@
return proc
-def kill_running_process():
+def kill_running_process(service_name, host, port):
"""Find and kill any running web service processes."""
- global service_name
try:
pid = get_pid(service_name)
except IOError:
@@ -233,7 +204,7 @@
# becomes available. Failing to do so causes a race condition
# between freeing the socket in the killed process, and
# opening it in the current one.
- wait_for_service_shutdown()
+ wait_for_service_shutdown(host, port)
except os.error as err:
if err.errno == errno.ESRCH:
# Whoops, we got a 'No such process' error. The PID file
@@ -243,24 +214,3 @@
remove_if_exists(pidfile_path(service_name))
else:
raise
-
-
-def main():
- """Run the HTTP server."""
- # Redirect our service output to a log file.
- global log
- ensure_directory_exists(os.path.dirname(config.google_test_service.log))
- filelog = logging.FileHandler(config.google_test_service.log)
- log.addHandler(filelog)
- log.setLevel(logging.DEBUG)
-
- # To support service shutdown we need to create a PID file that is
- # understood by the Launchpad services framework.
- global service_name
- make_pidfile(service_name)
-
- host, port = get_service_endpoint()
- server = HTTPServer((host, port), GoogleRequestHandler)
-
- log.info("Starting HTTP Google webservice server on port %s", port)
- server.serve_forever()
=== modified file 'lib/lp/testing/layers.py'
--- lib/lp/testing/layers.py 2018-03-16 14:55:41 +0000
+++ lib/lp/testing/layers.py 2018-03-28 21:29:05 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Layers used by Launchpad tests.
@@ -23,6 +23,8 @@
'AppServerLayer',
'AuditorLayer',
'BaseLayer',
+ 'BingLaunchpadFunctionalLayer',
+ 'BingServiceLayer',
'DatabaseFunctionalLayer',
'DatabaseLayer',
'FunctionalLayer',
@@ -126,6 +128,9 @@
from lp.services.osutils import kill_by_pidfile
from lp.services.rabbit.server import RabbitServer
from lp.services.scripts import execute_zcml_for_scripts
+from lp.services.sitesearch.tests.bingserviceharness import (
+ BingServiceTestSetup,
+ )
from lp.services.sitesearch.tests.googleserviceharness import (
GoogleServiceTestSetup,
)
@@ -1259,6 +1264,31 @@
pass
+class BingServiceLayer(BaseLayer):
+ """Tests for Bing web service integration."""
+
+ @classmethod
+ def setUp(cls):
+ bing = BingServiceTestSetup()
+ bing.setUp()
+
+ @classmethod
+ def tearDown(cls):
+ BingServiceTestSetup().tearDown()
+
+ @classmethod
+ def testSetUp(self):
+ # We need to override BaseLayer.testSetUp(), or else we will
+ # get a LayerIsolationError.
+ pass
+
+ @classmethod
+ def testTearDown(self):
+ # We need to override BaseLayer.testTearDown(), or else we will
+ # get a LayerIsolationError.
+ pass
+
+
class DatabaseFunctionalLayer(DatabaseLayer, FunctionalLayer):
"""Provides the database and the Zope3 application server environment."""
@@ -1383,6 +1413,31 @@
pass
+class BingLaunchpadFunctionalLayer(LaunchpadFunctionalLayer,
+ BingServiceLayer):
+ """Provides Bing service in addition to LaunchpadFunctionalLayer."""
+
+ @classmethod
+ @profiled
+ def setUp(cls):
+ pass
+
+ @classmethod
+ @profiled
+ def tearDown(cls):
+ pass
+
+ @classmethod
+ @profiled
+ def testSetUp(cls):
+ pass
+
+ @classmethod
+ @profiled
+ def testTearDown(cls):
+ pass
+
+
class ZopelessDatabaseLayer(ZopelessLayer, DatabaseLayer):
"""Testing layer for unit tests with no need for librarian.
@@ -1541,7 +1596,8 @@
return self.request._orig_env
-class PageTestLayer(LaunchpadFunctionalLayer, GoogleServiceLayer):
+class PageTestLayer(LaunchpadFunctionalLayer,
+ BingServiceLayer, GoogleServiceLayer):
"""Environment for page tests.
"""
=== modified file 'setup.py'
--- setup.py 2018-03-16 15:14:34 +0000
+++ setup.py 2018-03-28 21:29:05 +0000
@@ -284,6 +284,8 @@
},
entry_points=dict(
console_scripts=[ # `console_scripts` is a magic name to setuptools
+ 'bingtestservice = '
+ 'lp.services.sitesearch.bingtestservice:main',
'build-twisted-plugin-cache = '
'lp.services.twistedsupport.plugincache:main',
'combine-css = lp.scripts.utilities.js.combinecss:main',
Follow ups
