launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/lpbuildbot/xenial-fixes into lp:lpbuildbot.

Commit message:
Various fixes to work with xenial's buildbot.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~cjwatson/lpbuildbot/xenial-fixes/+merge/404066
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/lpbuildbot/xenial-fixes into lp:lpbuildbot.

=== modified file 'lpbuildbot/poller.py'
--- lpbuildbot/poller.py	2020-07-27 17:08:18 +0000
+++ lpbuildbot/poller.py	2021-06-11 11:46:19 +0000
@@ -1,7 +1,7 @@
-import os
+import itertools
+import re
 
 from twisted.internet import defer
-from twisted.internet.utils import getProcessOutput
 from twisted.python import log
 
 from buildbot.changes.gitpoller import GitPoller as _GitPoller
@@ -20,67 +20,81 @@
     @defer.inlineCallbacks
     def _get_commit_files(self, rev):
         args = [
-            'log', rev, '--name-only', '%s~..%s' % (rev, rev), r'--format=%n']
-        git_output = yield getProcessOutput(
-            self.gitbin, args, path=self.workdir,
-            env={'PATH': os.environ['PATH']}, errortoo=False)
-        fileList = sorted(set(git_output.split()))
-        defer.returnValue(fileList)
-
-    @defer.inlineCallbacks
-    def _get_commit_name(self, rev):
-        args = ['log', rev, '--no-walk', r'--format=%aN <%aE>']
-        git_output = yield getProcessOutput(
-            self.gitbin, args, path=self.workdir,
-            env={'PATH': os.environ['PATH']}, errortoo=False)
-        stripped_output = git_output.strip().decode(self.encoding)
-        if len(stripped_output) == 0:
-            raise EnvironmentError('could not get commit name for rev')
-        defer.returnValue(stripped_output)
-
-    @defer.inlineCallbacks
-    def _process_changes(self, unused_output):
-        revListArgs = [
-            'log', '--first-parent',
-            '%s..origin/%s' % (self.branch, self.branch), r'--format=%H']
+            '--name-only', '--no-walk', r'--format=%n',
+            '%s~..%s' % (rev, rev), '--']
+        git_output = yield self._dovccmd('log', args, path=self.workdir)
+
+        def decode_file(file):
+            # git use octal char sequences in quotes when non ASCII
+            match = re.match('^"(.*)"$', file)
+            if match:
+                file = match.groups()[0].decode('string_escape')
+            return self._decode(file)
+
+        fileList = [
+            decode_file(file) for file in itertools.ifilter(
+                lambda s: len(s), git_output.splitlines())]
+        defer.returnValue(sorted(set(fileList)))
+
+    @defer.inlineCallbacks
+    def _process_changes(self, newRev, branch):
+        """
+        Read changes since last change.
+
+        - Read list of commit hashes.
+        - Extract details from each commit.
+        - Add changes to database.
+        """
+
+        # initial run, don't parse all history
+        if not self.lastRev:
+            return
+        if newRev in self.lastRev.values():
+            # TODO: no new changes on this branch
+            # should we just use the lastRev again, but with a different
+            # branch?
+            pass
+
+        # get the change list
+        revListArgs = ([r'--first-parent', r'--format=%H', r'%s' % newRev] +
+                       [r'^%s' % rev for rev in self.lastRev.values()] +
+                       [r'--'])
         self.changeCount = 0
-        results = yield getProcessOutput(
-            self.gitbin, revListArgs, path=self.workdir,
-            env={'PATH': os.environ['PATH']}, errortoo=False)
+        results = yield self._dovccmd('log', revListArgs, path=self.workdir)
 
         # process oldest change first
         revList = results.split()
-        if not revList:
-            return
-
         revList.reverse()
         self.changeCount = len(revList)
+        self.lastRev[branch] = newRev
 
-        log.msg('gitpoller: processing %d changes: %s in "%s"'
-                % (self.changeCount, revList, self.workdir))
+        log.msg('gitpoller: processing %d changes: %s from "%s"'
+                % (self.changeCount, revList, self.repourl))
 
         for rev in revList:
-            results = yield defer.DeferredList([
+            dl = defer.DeferredList([
                 self._get_commit_timestamp(rev),
-                self._get_commit_name(rev),
+                self._get_commit_author(rev),
                 self._get_commit_files(rev),
                 self._get_commit_comments(rev),
             ], consumeErrors=True)
 
+            results = yield dl
+
             # check for failures
-            failures = [ r[1] for r in results if not r[0] ]
+            failures = [r[1] for r in results if not r[0]]
             if failures:
                 # just fail on the first error; they're probably all related!
                 raise failures[0]
 
-            timestamp, name, files, comments = [ r[1] for r in results ]
+            timestamp, author, files, comments = [r[1] for r in results]
             yield self.master.addChange(
-                author=name,
+                author=author,
                 revision=rev,
                 files=files,
                 comments=comments,
                 when_timestamp=epoch2datetime(timestamp),
-                branch=self.branch,
+                branch=self._removeHeads(branch),
                 category=self.category,
                 project=self.project,
                 repository=self.repourl,

=== modified file 'lpbuildbot/schedulers/aggregating.py'
--- lpbuildbot/schedulers/aggregating.py	2021-04-08 11:30:41 +0000
+++ lpbuildbot/schedulers/aggregating.py	2021-06-11 11:46:19 +0000
@@ -78,7 +78,7 @@
     @defer.inlineCallbacks
     def gotChange(self, change, important):
         yield self.master.db.schedulers.classifyChanges(
-            self.schedulerid, {change.number: important})
+            self.objectid, {change.number: important})
         yield self._maybeAddBuildset()
 
     def _getTimerName(self):
@@ -91,7 +91,7 @@
     def _getChangeClassifications(self):
         timer_name = self._getTimerName()
         classifications = yield self.getChangeClassificationsForTimer(
-            self.schedulerid, timer_name)
+            self.objectid, timer_name)
         defer.returnValue(classifications)
 
     @util.deferredLocked('_stable_timers_lock')
@@ -158,7 +158,7 @@
             bsid, _ = yield self.addBuildsetForChanges(
                 reason=reason, changeids=changeids, builderNames=builderNames)
             yield self.master.db.schedulers.flushChangeClassifications(
-                self.schedulerid, less_than=changeids[-1] + 1)
+                self.objectid, less_than=changeids[-1] + 1)
         else:  # build forced, so we have no changes
             bsid, _ = yield self.addBuildsetForLatest(
                 reason=reason, branch=self._branch, builderNames=builderNames)

=== modified file 'lpbuildbot/web.py'
--- lpbuildbot/web.py	2021-04-08 11:30:41 +0000
+++ lpbuildbot/web.py	2021-06-11 11:46:19 +0000
@@ -1,10 +1,13 @@
 from buildbot.status import html
+from buildbot.status.web.base import AccessorMixin
 
 from lpbuildbot.schedulers.aggregating import AggregatingScheduler
 
 from twisted.internet import defer
-from twisted.web.resource import Resource
-from twisted.web.error import ErrorPage
+from twisted.web.resource import (
+    ErrorPage,
+    Resource,
+    )
 from twisted.web.server import NOT_DONE_YET
 
 
@@ -46,7 +49,7 @@
     real_detail = '<p>%s</p><p><a href="force">Try again</a></p>' % detail
     return ErrorPage(400, "Something went wrong", real_detail).render(request)
 
-class ForceResource(Resource):
+class ForceResource(Resource, AccessorMixin):
 
     def __init__(self, forceableBranchBuilderNames):
         self.forceableBranchBuilderNames = forceableBranchBuilderNames
@@ -82,9 +85,10 @@
 
     @defer.inlineCallbacks
     def _submitBuild(self, request, repository, branch, builder):
+        master = self.getBuildmaster(request)
         reason = 'build forced'
         relevant_schedulers = [
-            scheduler for scheduler in self.master.allSchedulers()
+            scheduler for scheduler in master.allSchedulers()
             if builder in scheduler.listBuilderNames() and
                isinstance(scheduler, AggregatingScheduler)]
         if relevant_schedulers:
@@ -92,10 +96,10 @@
                 yield scheduler.scheduleBuild(
                     reason=reason, builderNames=[builder])
         else:
-            ssid = yield self.master.db.sourcestamps.addSourceStamp(
+            ssid = yield master.db.sourcestamps.addSourceStamp(
                 branch=branch, revision=None, repository=repository,
                 project='')
-            yield self.master.addBuildset(
+            yield master.addBuildset(
                 ssid=ssid, reason=reason, properties={},
                 builderNames=[builder])
         request.write(success_html)

=== modified file 'master.cfg'
--- master.cfg	2021-03-30 13:24:32 +0000
+++ master.cfg	2021-06-11 11:46:19 +0000
@@ -43,7 +43,9 @@
         branch='master', workdir=os.path.join(poller_workdir, 'devel'),
         pollinterval=60),
     GitPoller(
-        'https://git.launchpad.net/launchpad',
+        # Artificially-different URL to work around
+        # https://lists.buildbot.net/pipermail/devel/2013-January/009390.html.
+        'https://git.launchpad.net/~launchpad/launchpad',
         branch='db-devel', workdir=os.path.join(poller_workdir, 'db-devel'),
         pollinterval=60),
     ]
@@ -65,6 +67,20 @@
     branch='db-devel',
     treeStableTimer=3*60, treeStableCount=3))
 
+from buildbot import scheduler
+c['schedulers'].append(scheduler.Triggerable(
+    name='lp-devel-bionic',
+    builderNames=["lp-devel-bionic"]))
+c['schedulers'].append(scheduler.Triggerable(
+    name='lp-devel-bionic-py3',
+    builderNames=["lp-devel-bionic-py3"]))
+c['schedulers'].append(scheduler.Triggerable(
+    name='lp-db-devel-bionic',
+    builderNames=["lp-db-devel-bionic"]))
+c['schedulers'].append(scheduler.Triggerable(
+    name='lp-db-devel-bionic-py3',
+    builderNames=["lp-db-devel-bionic-py3"]))
+
 ####### BUILDERS
 
 # the 'builders' list defines the Builders. Each one is configured with a

=== modified file 'templates/root.html'
--- templates/root.html	2018-05-02 09:49:40 +0000
+++ templates/root.html	2021-06-11 11:46:19 +0000
@@ -46,7 +46,7 @@
   <li class="{{ item_class.next() }}"><a href="about">About</a> this Buildbot</li>
 </ul>
 
-{%- if authz.advertiseAction('cleanShutdown') -%}
+{%- if authz.advertiseAction('cleanShutdown', request) -%}
 {%- if shutting_down -%}
 Master is shutting down<br/>
 {{ forms.cancel_clean_shutdown(cancel_shutdown_url, authz) }}