launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #04462
[Merge] lp:~wgrant/launchpad/murder-poppy into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/murder-poppy into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/murder-poppy/+merge/70125
The old poppy is dead.
soyuz-upload.txt was the only test that used it. It has been ported to poppy-sftp.
--
https://code.launchpad.net/~wgrant/launchpad/murder-poppy/+merge/70125
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/murder-poppy into lp:launchpad.
=== removed file 'daemons/poppy-upload.py'
--- daemons/poppy-upload.py 2010-04-27 19:48:39 +0000
+++ daemons/poppy-upload.py 1970-01-01 00:00:00 +0000
@@ -1,10 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-import sys
-from lp.poppy.daemon import main
-
-if __name__ == '__main__':
- sys.exit(main())
=== removed file 'lib/lp/poppy/daemon.py'
--- lib/lp/poppy/daemon.py 2011-02-03 03:49:22 +0000
+++ lib/lp/poppy/daemon.py 1970-01-01 00:00:00 +0000
@@ -1,65 +0,0 @@
-# Copyright 2010 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""The Poppy daemon."""
-
-__metaclass__ = type
-__all__ = [
- 'main',
- ]
-
-import optparse
-import warnings
-
-from canonical.launchpad.scripts import (
- logger,
- logger_options,
- )
-from lp.poppy.hooks import Hooks
-from lp.poppy.server import run_server
-
-# XXX: 2010-04-26, Salgado, bug=570246: Silence python2.6 deprecation
-# warnings.
-warnings.filterwarnings(
- 'ignore', '.*(md5|sha|sets)', DeprecationWarning,
- )
-
-
-def main():
- parser = optparse.OptionParser()
- logger_options(parser)
-
- parser.add_option("--cmd", action="store", metavar="CMD",
- help="Run CMD after each upload completion")
-
- parser.add_option("--allow-user", action="store", metavar="USER",
- default='ubuntu',
- help="Username allowed to log in.")
-
- parser.add_option("--permissions", action="store", metavar="PERMS",
- default='g+rwxs',
- help="Permissions to chmod the targetfsroot with "
- "before letting go of the directory.")
-
- options, args = parser.parse_args()
-
- log = logger(options, "poppy-upload")
-
- if len(args) != 2:
- print "usage: poppy-upload.py rootuploaddirectory port"
- return 1
-
- root, port = args
- host = "0.0.0.0"
- ident = "Launchpad upload server"
- numthreads = 4
-
- hooks = Hooks(root, log, allow_user=options.allow_user, cmd=options.cmd,
- perms=options.permissions)
-
- run_server(host, int(port), ident, numthreads,
- hooks.new_client_hook, hooks.client_done_hook,
- hooks.auth_verify_hook)
- return 0
-
-
=== removed file 'lib/lp/poppy/server.py'
--- lib/lp/poppy/server.py 2010-08-20 20:31:18 +0000
+++ lib/lp/poppy/server.py 1970-01-01 00:00:00 +0000
@@ -1,160 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-__metaclass__ = type
-
-import asyncore
-import tempfile
-from time import time
-
-from zope.server.ftp.server import (
- FTPServerChannel,
- STORChannel as OriginalSTORChannel,
- )
-from zope.server.serverbase import ServerBase
-from zope.server.taskthreads import ThreadedTaskDispatcher
-
-from lp.poppy.filesystem import UploadFileSystem
-
-
-class Channel(FTPServerChannel):
-
- def __init__(self, server, conn, addr, adj=None):
- # Work around a zope3 bug where the status messages dict is copied by
- # reference, not by value.
- self.status_messages = dict(self.status_messages)
- self.status_messages['SERVER_READY'] = (
- '220 %s Canonical FTP server ready.')
-
- FTPServerChannel.__init__(self, server, conn, addr, adj=None)
- self.peername = self.socket.getpeername()
- self.uploadfilesystem, self.fsroot = server.newClient(self)
- self.hook = server.auth_verify_hook
-
- def close(self):
- FTPServerChannel.close(self)
- self.server.clientFinished(self)
-
- def _getFileSystem(self):
- return self.uploadfilesystem
-
- def received(self, data):
- # XXX Steve Alexander 2005-01-18
- # This is a work-around for a bug in Zope 3's ServerChannelBase
- # that it doesn't update self.last_activity.
- # This method can be removed once Zope3 is fixed, and we're using
- # that code.
- # http://collector.zope.org/Zope3-dev/350
- self.record_activity()
- FTPServerChannel.received(self, data)
-
- def record_activity(self):
- self.last_activity = time()
-
- def cmd_pass(self, args):
- 'See IFTPCommandHandler'
- self.authenticated = 0
- password = args
- credentials = (self.username, password)
- okay = True
- if self.hook:
- try:
- if not self.hook(self.fsroot, self.username, password):
- okay = False
- except:
- okay = False
- if not okay:
- self.reply('LOGIN_MISMATCH')
- self.close_when_done()
- else:
- self.credentials = credentials
- self.authenticated = 1
- self.reply('LOGIN_SUCCESS')
-
- def cmd_stor(self, args, write_mode='w'):
- 'See IFTPCommandHandler'
- if not args:
- self.reply('ERR_ARGS')
- return
- path = self._generatePath(args)
-
- start = 0
- if self.restart_position:
- self.start = self.restart_position
- mode = write_mode + self.type_mode_map[self.transfer_mode]
-
- if not self._getFileSystem().writable(path):
- self.reply('ERR_OPEN_WRITE', "Can't write file")
- return
-
- cdc = STORChannel(self, (path, mode, start))
- self.syncConnectData(cdc)
- self.reply('OPEN_CONN', (self.type_map[self.transfer_mode], path))
-
- def cmd_cwd(self, args):
- """Permissive 'cwd', creates any target directories requested.
-
- It relies on the filesystem layer to create directories recursivelly.
- """
- path = self._generatePath(args)
- if not self._getFileSystem().type(path) == 'd':
- self._getFileSystem().mkdir(path)
- self.cwd = path
- self.reply('SUCCESS_250', 'CWD')
-
-
-class STORChannel(OriginalSTORChannel):
-
- def received (self, data):
- if data:
- self.inbuf.append(data)
- self.control_channel.record_activity()
- # This is the point at which some data for an upload has been
- # received by the server from a client.
-
-
-class Server(ServerBase):
-
- channel_class = Channel
-
- def __init__(self, ip, port,
- new_client_hook, client_done_hook, auth_verify_hook,
- *args, **kw):
- ServerBase.__init__(self, ip, port, *args, **kw)
- self.new_client_hook = new_client_hook
- self.client_done_hook = client_done_hook
- self.auth_verify_hook = auth_verify_hook
-
- def newClient(self, channel):
- fsroot = tempfile.mkdtemp("-poppy")
- uploadfilesystem = UploadFileSystem(fsroot)
- clienthost, clientport = channel.peername
- try:
- self.new_client_hook(fsroot, clienthost, clientport)
- except Exception:
- # Almost bare except, result logged, to keep server running.
- self.logger.exception("Exception during new client hook")
- return uploadfilesystem, fsroot
-
- def clientFinished(self, channel):
- clienthost, clientport = channel.peername
- try:
- self.client_done_hook(channel.fsroot, clienthost, clientport)
- except Exception:
- # Almost bare except, result logged, to keep server running.
- self.logger.exception("Exception during client done hook")
-
-
-def run_server(host, port, ident, numthreads,
- new_client_hook, client_done_hook, auth_verify_hook=None):
- task_dispatcher = ThreadedTaskDispatcher()
- task_dispatcher.setThreadCount(numthreads)
- server = Server(host, port,
- new_client_hook, client_done_hook, auth_verify_hook,
- task_dispatcher=task_dispatcher)
- server.SERVER_IDENT = ident
- try:
- asyncore.loop()
- except KeyboardInterrupt:
- # Exit without spewing an exception.
- pass
=== removed file 'lib/lp/poppy/tests/helpers.py'
--- lib/lp/poppy/tests/helpers.py 2010-03-18 11:16:55 +0000
+++ lib/lp/poppy/tests/helpers.py 1970-01-01 00:00:00 +0000
@@ -1,105 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-__metaclass__ = type
-
-import os
-import select
-import subprocess
-import sys
-import time
-
-from canonical.config import config
-from lp.services.osutils import two_stage_kill
-
-
-class SoyuzUploadError(Exception):
- """Used in the soyuz-upload test."""
-
-
-class PoppyTestSetup:
- """Provides a setup and teardown mechanism for a poppy subprocess instance.
-
- Use this like you would LibrarianTestSetup or similar.
- """
-
- def __init__(self, fsroot,
- user='anonymous',
- cmd='echo @distro@; ls @fsroot@',
- port=3421):
- self.fsroot = fsroot
- self.user = user
- self.cmd = cmd
- self.port = str(port)
- self.running = False
-
- def startPoppy(self):
- """Start the poppy instance."""
- script = os.path.join(config.root, "daemons/poppy-upload.py")
- self.process = subprocess.Popen(
- [sys.executable, script, "--allow-user", self.user,
- "--cmd", self.cmd, self.fsroot, self.port],
- stdout=subprocess.PIPE)
- self.running = True
-
- def killPoppy(self):
- """Kill the poppy instance dead."""
- if not self.alive:
- return
- two_stage_kill(self.process.pid)
- self.running = False
-
- @property
- def alive(self):
- """Whether or not the poppy instance is still alive."""
- if not self.running:
- return False
- return self.process.poll() is None
-
- def verify_output(self, expected):
- """Verify that poppy writes the expected output."""
- timelimit = time.time() + 60
- buffer = ""
- while True:
- rlist, wlist, xlist = select.select(
- [self.process.stdout.fileno()], [], [], timelimit - time.time())
- if len(rlist) == 0:
- if self.process.poll() is not None:
- raise SoyuzUploadError("Poppy died unexpectedly")
- # Try and kill poppy too?
- raise SoyuzUploadError(
- "FTP server timed out. The following was expected:\n%r\n"
- "But the following was received:\n%r\n"
- % (expected, buffer))
- else:
- # reset the time limit
- timelimit = time.time() + 60
- chunk = os.read(self.process.stdout.fileno(), 4096)
- buffer += chunk
- # XXX: jamesh 2007-02-20:
- # We might have an incomplete line at the end of the
- # buffer. This doesn't seem to be a problem for the
- # amount of data being written in the tests though.
- lines = buffer.splitlines()
- for line in lines:
- if line == self.user:
- continue
- if line not in expected:
- raise SoyuzUploadError("Unexpected line found in "
- "poppy output: %r" % line)
- expected.remove(line)
- buffer = ""
- if not expected:
- break
- # stdout was closed before we read all the expected lines
- if chunk == "":
- raise SoyuzUploadError("FTP server exited before the "
- "expected data was received: %r"
- % expected)
- else:
- raise SoyuzUploadError(
- "FTP server timed out.\n"
- "The following was expected:\n%r\n"
- "But the following was received:\n%r\n"
- % (expected, buffer))
-
=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt'
--- lib/lp/soyuz/doc/soyuz-upload.txt 2011-07-19 07:50:59 +0000
+++ lib/lp/soyuz/doc/soyuz-upload.txt 2011-08-02 08:24:43 +0000
@@ -41,9 +41,9 @@
call the upload processing tool. We'll do that ourselves in our test,
so that we can control what's going on.
- >>> from lp.poppy.tests.helpers import PoppyTestSetup, SoyuzUploadError
- >>> poppy = PoppyTestSetup(incoming_dir)
- >>> poppy.startPoppy()
+ >>> from lp.poppy.tests.test_poppy import PoppyTac
+ >>> poppy = PoppyTac(incoming_dir)
+ >>> poppy.setUp()
Connect to the server and login. We'll keep trying to connect until
the server dies or the connection succeeds.
@@ -52,16 +52,16 @@
>>> ftp = ftplib.FTP()
>>> while True:
... try:
- ... reply = ftp.connect("localhost", 3421)
+ ... reply = ftp.connect("localhost", 2121)
... except socket.error:
... if not poppy.alive:
- ... raise SoyuzUploadError('Server can not start.')
+ ... raise Exception('Server can not start.')
... else:
... break
>>> ftp.login("anonymous", "")
- '230 Login Successful.'
+ '230 Anonymous login ok, access restrictions apply.'
>>> ftp.cwd("/")
- '250 CWD command successful.'
+ '250 Requested File Action Completed OK'
Good.. let's send all packages we have in the test directory to
@@ -88,8 +88,8 @@
>>> for changes_filepath in changes:
...
... if not ftp.sock:
- ... assert ftp.connect("localhost", 3421).startswith("220 ")
- ... assert ftp.login("anonymous", "") == '230 Login Successful.'
+ ... assert ftp.connect("localhost", 2121).startswith("220 ")
+ ... assert ftp.login("anonymous", "").startswith("230 ")
...
... tf = parse_tagfile(changes_filepath)
...
@@ -108,9 +108,16 @@
... ignore = ftp.cwd("ubuntutest")
...
... for filepath in send_filepaths:
- ... reply = ftp.storbinary(
- ... "STOR %s" % os.path.basename(filepath), open(filepath))
- ... assert reply == '226 Transfer successful.'
+ ... try:
+ ... reply = ftp.storbinary(
+ ... "STOR %s" % os.path.basename(filepath),
+ ... open(filepath))
+ ... assert reply == '226 Transfer Complete.'
+ ... except ftplib.error_perm as e:
+ ... # Our changes files aren't signed, so poppy-sftp will
+ ... # complain. But let them through anyway.
+ ... if not e[0].startswith('550 Changes file must be signed'):
+ ... raise
...
... uploads.append(send_filepaths)
...
@@ -122,18 +129,6 @@
>>> package_names
['drdsl', 'etherwake']
-We create a set of the filenames we expect to see in the FTP server
-process output, and wait until all of them have been shown.
-
-This is a little bit tricky because we won't simply try to read
-the process output in a blocking way, since any failure in the
-FTP process would block automated tests. Instead, we define a
-timeout between output data. If the process doesn't provide new
-data in the given number seconds, we report a failure.
-
- >>> poppy.verify_output(
- ... ['ubuntutest', 'ubuntutest', 'ubuntutest'])
-
At that point we must have a bunch of directories in the upload
base directory named <TIMESTAMP>-XXXXXX, each one as a result of
each FTP session. Below we ensure that, and also that the content
@@ -182,7 +177,7 @@
Right, that's all we need from the FTP server. We don't need it anymore,
so we'll just kill the process.
- >>> status = poppy.killPoppy()
+ >>> status = poppy.cleanUp()
Finally, we'll just create an entirely empty upload folder. We rely for
our tests on a poppy-like naming system, ie. that the upload folder
=== modified file 'utilities/start-dev-soyuz.sh'
--- utilities/start-dev-soyuz.sh 2011-02-17 13:55:43 +0000
+++ utilities/start-dev-soyuz.sh 2011-08-02 08:24:43 +0000
@@ -13,9 +13,9 @@
start_twistd testkeyserver lib/lp/testing/keyserver/testkeyserver.tac
start_twistd buildd-manager daemons/buildd-manager.tac
-
-echo "Starting poppy."
mkdir -p /var/tmp/poppy
-bin/py daemons/poppy-upload.py /var/tmp/poppy/incoming 2121 &
+export POPPY_ROOT=/var/tmp/poppy/incoming
+start_twistd poppy-sftp daemons/poppy-sftp.tac
+
echo "Done."
