launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #17696
[Merge] lp:~cjwatson/launchpad/split-txpkgupload into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/split-txpkgupload into lp:launchpad.
Commit message:
Split out lp.poppy to txpkgupload.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/split-txpkgupload/+merge/246320
Split out lp.poppy to txpkgupload.
The main involved bit here is dealing with twistd setup.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/split-txpkgupload into lp:launchpad.
=== modified file 'buildout.cfg'
--- buildout.cfg 2013-06-03 06:30:28 +0000
+++ buildout.cfg 2015-01-13 15:34:27 +0000
@@ -9,6 +9,7 @@
iharness
i18n
txlongpoll
+ txpkgupload
unzip = true
eggs-directory = eggs
download-cache = download-cache
@@ -97,3 +98,13 @@
initialization = ${scripts:initialization}
entry-points = twistd-for-txlongpoll=twisted.scripts.twistd:run
scripts = twistd-for-txlongpoll
+
+[txpkgupload]
+recipe = z3c.recipe.scripts
+eggs = ${scripts:eggs}
+ txpkgupload
+include-site-packages = false
+allowed-eggs-from-site-packages =
+initialization = ${scripts:initialization}
+entry-points = twistd-for-txpkgupload=twisted.scripts.twistd:run
+scripts = twistd-for-txpkgupload
=== renamed file 'lib/lp/poppy/tests/poppy-sftp' => 'configs/development/txpkgupload-sftp'
=== renamed file 'lib/lp/poppy/tests/poppy-sftp.pub' => 'configs/development/txpkgupload-sftp.pub'
=== removed file 'daemons/poppy-sftp.tac'
--- daemons/poppy-sftp.tac 2015-01-12 18:53:31 +0000
+++ daemons/poppy-sftp.tac 1970-01-01 00:00:00 +0000
@@ -1,121 +0,0 @@
-# Copyright 2010 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# This is a Twisted application config file. To run, use:
-# twistd -noy sftp.tac
-# or similar. Refer to the twistd(1) man page for details.
-
-import logging
-
-from lazr.sshserver.auth import (
- LaunchpadAvatar,
- PublicKeyFromLaunchpadChecker,
- )
-from lazr.sshserver.service import SSHService
-from lazr.sshserver.session import DoNothingSession
-
-from twisted.application import service
-from twisted.conch.interfaces import ISession
-from twisted.conch.ssh import filetransfer
-from twisted.cred.portal import IRealm, Portal
-from twisted.protocols.policies import TimeoutFactory
-from twisted.python import components
-from twisted.scripts.twistd import ServerOptions
-from twisted.web.xmlrpc import Proxy
-
-from zope.interface import implements
-
-from lp.services.config import config
-from lp.services.daemons import readyservice
-
-from lp.poppy import get_poppy_root
-from lp.poppy.twistedftp import (
- FTPServiceFactory,
- )
-from lp.poppy.twistedsftp import SFTPServer
-from lp.services.twistedsupport.loggingsupport import set_up_oops_reporting
-
-
-def make_portal():
- """Create and return a `Portal` for the SSH service.
-
- This portal accepts SSH credentials and returns our customized SSH
- avatars (see `LaunchpadAvatar`).
- """
- authentication_proxy = Proxy(
- config.poppy.authentication_endpoint)
- portal = Portal(Realm(authentication_proxy))
- portal.registerChecker(
- PublicKeyFromLaunchpadChecker(authentication_proxy))
- return portal
-
-
-class Realm:
- implements(IRealm)
-
- def __init__(self, authentication_proxy):
- self.authentication_proxy = authentication_proxy
-
- def requestAvatar(self, avatar_id, mind, *interfaces):
- # Fetch the user's details from the authserver
- deferred = mind.lookupUserDetails(
- self.authentication_proxy, avatar_id)
-
- # Once all those details are retrieved, we can construct the avatar.
- def got_user_dict(user_dict):
- avatar = LaunchpadAvatar(user_dict)
- return interfaces[0], avatar, avatar.logout
-
- return deferred.addCallback(got_user_dict)
-
-
-def poppy_sftp_adapter(avatar):
- return SFTPServer(avatar, get_poppy_root())
-
-
-# Force python logging to all go to the Twisted log.msg interface. The default
-# - output on stderr - will not be watched by anyone.
-from twisted.python import log
-stream = log.StdioOnnaStick()
-logging.basicConfig(stream=stream, level=logging.INFO)
-
-
-components.registerAdapter(
- poppy_sftp_adapter, LaunchpadAvatar, filetransfer.ISFTPServer)
-
-components.registerAdapter(DoNothingSession, LaunchpadAvatar, ISession)
-
-
-# ftpport defaults to 2121 in schema-lazr.conf
-ftpservice = FTPServiceFactory.makeFTPService(port=config.poppy.ftp_port)
-
-# Construct an Application that has the Poppy SSH server,
-# and the Poppy FTP server.
-options = ServerOptions()
-options.parseOptions()
-application = service.Application('poppy-sftp')
-observer = set_up_oops_reporting(
- 'poppy-sftp', 'poppy', options.get('logfile'))
-application.addComponent(observer, ignoreClass=1)
-
-ftpservice.setServiceParent(application)
-
-
-def timeout_decorator(factory):
- """Add idle timeouts to a factory."""
- return TimeoutFactory(factory, timeoutPeriod=config.poppy.idle_timeout)
-
-svc = SSHService(
- portal=make_portal(),
- private_key_path=config.poppy.host_key_private,
- public_key_path=config.poppy.host_key_public,
- main_log='poppy',
- access_log='poppy.access',
- access_log_path=config.poppy.access_log,
- strport=config.poppy.port,
- factory_decorator=timeout_decorator,
- banner=config.poppy.banner)
-svc.setServiceParent(application)
-
-# Service that announces when the daemon is ready
-readyservice.ReadyService().setServiceParent(application)
=== removed directory 'lib/lp/poppy'
=== removed file 'lib/lp/poppy/__init__.py'
--- lib/lp/poppy/__init__.py 2011-12-29 05:29:36 +0000
+++ lib/lp/poppy/__init__.py 1970-01-01 00:00:00 +0000
@@ -1,20 +0,0 @@
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# Make this directory into a Python package.
-
-import os
-
-from lp.services.config import config
-
-
-def get_poppy_root():
- """Return the poppy root to use for this server.
-
- If the POPPY_ROOT environment variable is set, use that. If not, use
- config.poppy.fsroot.
- """
- poppy_root = os.environ.get('POPPY_ROOT', None)
- if poppy_root:
- return poppy_root
- return config.poppy.fsroot
=== removed file 'lib/lp/poppy/filesystem.py'
--- lib/lp/poppy/filesystem.py 2010-08-20 20:31:18 +0000
+++ lib/lp/poppy/filesystem.py 1970-01-01 00:00:00 +0000
@@ -1,248 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-__metaclass__ = type
-__all__ = [
- 'UploadFileSystem',
- ]
-
-import datetime
-import os
-import stat
-
-from zope.interface import implements
-from zope.security.interfaces import Unauthorized
-from zope.server.interfaces.ftp import IFileSystem
-
-
-class UploadFileSystem:
-
- implements(IFileSystem)
-
- def __init__(self, rootpath):
- self.rootpath = rootpath
-
- def _full(self, path):
- """Returns the full path name (i.e. rootpath + path)"""
- full_path = os.path.join(self.rootpath, path)
- if not os.path.realpath(full_path).startswith(self.rootpath):
- raise OSError("Path not allowed:", path)
- return full_path
-
- def _sanitize(self, path):
- if path.startswith('/'):
- path = path[1:]
- path = os.path.normpath(path)
- return path
-
- def type(self, path):
- """Return the file type at path
-
- The 'type' command returns 'f' for a file, 'd' for a directory and
- None if there is no file.
- """
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- if os.path.isdir(full_path):
- return 'd'
- elif os.path.isfile(full_path):
- return 'f'
-
- def names(self, path, filter=None):
- """Return a sequence of the names in a directory
-
- If the filter is not None, include only those names for which
- the filter returns a true value.
- """
- path = self._sanitize(path)
- full_path = self._full(path)
- if not os.path.exists(full_path):
- raise OSError("Not exists:", path)
- filenames = os.listdir(os.path.join(self.rootpath, path))
- files = []
- for filename in filenames:
- if not filter or filter(filename):
- files.append(filename)
- return files
-
- def ls(self, path, filter=None):
- """Return a sequence of information objects.
-
- It considers the names in the given path (returned self.name())
- and builds file information using self.lsinfo().
- """
- return [self.lsinfo(name) for name in self.names(path, filter)]
-
- def readfile(self, path, outstream, start=0, end=None):
- """Outputs the file at path to a stream.
-
- Not allowed - see filesystem.txt.
- """
- raise Unauthorized
-
- def lsinfo(self, path):
- """Return information for a unix-style ls listing for the path
-
- See zope3's interfaces/ftp.py:IFileSystem for details of the
- dictionary's content.
- """
- path = self._sanitize(path)
- full_path = self._full(path)
- if not os.path.exists(full_path):
- raise OSError("Not exists:", path)
-
- info = {"owner_name": "upload",
- "group_name": "upload",
- "name": path.split("/")[-1]}
-
- s = os.stat(full_path)
-
- info["owner_readable"] = bool(s.st_mode & stat.S_IRUSR)
- info["owner_writable"] = bool(s.st_mode & stat.S_IWUSR)
- info["owner_executable"] = bool(s.st_mode & stat.S_IXUSR)
- info["group_readable"] = bool(s.st_mode & stat.S_IRGRP)
- info["group_writable"] = bool(s.st_mode & stat.S_IWGRP)
- info["group_executable"] = bool(s.st_mode & stat.S_IXGRP)
- info["other_readable"] = bool(s.st_mode & stat.S_IROTH)
- info["other_writable"] = bool(s.st_mode & stat.S_IWOTH)
- info["other_executable"] = bool(s.st_mode & stat.S_IXOTH)
- info["mtime"] = datetime.datetime.fromtimestamp(self.mtime(path))
- info["size"] = self.size(path)
- info["type"] = self.type(path)
- info["nlinks"] = s.st_nlink
- return info
-
- def mtime(self, path):
- """Return the modification time for the file"""
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- return os.path.getmtime(full_path)
-
- def size(self, path):
- """Return the size of the file at path"""
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- return os.path.getsize(full_path)
-
- def mkdir(self, path):
- """Create a directory."""
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- if os.path.isfile(full_path):
- raise OSError("File already exists:", path)
- elif os.path.isdir(full_path):
- raise OSError("Directory already exists:", path)
- raise OSError("OOPS, can't create:", path)
- else:
- old_mask = os.umask(0)
- try:
- os.makedirs(full_path, 0775)
- finally:
- os.umask(old_mask)
-
- def remove(self, path):
- """Remove a file."""
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- if os.path.isfile(full_path):
- os.unlink(full_path)
- elif os.path.isdir(full_path):
- raise OSError("Is a directory:", path)
- else:
- raise OSError("Not exists:", path)
-
- def rmdir(self, path):
- """Remove a directory.
-
- Remove a target path recursively.
- """
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- os.rmdir(full_path)
- else:
- raise OSError("Not exists:", path)
-
- def rename(self, old, new):
- """Rename a file."""
- old = self._sanitize(old)
- new = self._sanitize(new)
- full_old = self._full(old)
- full_new = self._full(new)
-
- if os.path.isdir(full_new):
- raise OSError("Is a directory:", new)
-
- if os.path.exists(full_old):
- if os.path.isfile(full_old):
- os.rename(full_old, full_new)
- elif os.path.isdir(full_old):
- raise OSError("Is a directory:", old)
- else:
- raise OSError("Not exists:", old)
-
- def writefile(self, path, instream, start=None, end=None, append=False):
- """Write data to a file.
-
- See zope3's interfaces/ftp.py:IFileSystem for details of the
- handling of the various arguments.
- """
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- if os.path.isdir(full_path):
- raise OSError("Is a directory:", path)
- else:
- dirname = os.path.dirname(full_path)
- if dirname:
- if not os.path.exists(dirname):
- old_mask = os.umask(0)
- try:
- os.makedirs(dirname, 0775)
- finally:
- os.umask(old_mask)
-
- if start and start < 0:
- raise ValueError("Negative start argument:", start)
- if end and end < 0:
- raise ValueError("Negative end argument:", end)
- if start and end and end <= start:
- return
- if append:
- open_flag = 'a'
- elif start or end:
- open_flag = "r+"
- if not os.path.exists(full_path):
- open(full_path, 'w')
-
- else:
- open_flag = 'w'
- outstream = open(full_path, open_flag)
- if start:
- outstream.seek(start)
- chunk = instream.read()
- while chunk:
- outstream.write(chunk)
- chunk = instream.read()
- if not end:
- outstream.truncate()
- instream.close()
- outstream.close()
-
- def writable(self, path):
- """Return boolean indicating whether a file at path is writable."""
- path = self._sanitize(path)
- full_path = self._full(path)
- if os.path.exists(full_path):
- if os.path.isfile(full_path):
- return True
- elif os.path.isdir(full_path):
- return False
- else:
- return True
-
=== removed file 'lib/lp/poppy/hooks.py'
--- lib/lp/poppy/hooks.py 2012-06-29 08:40:05 +0000
+++ lib/lp/poppy/hooks.py 1970-01-01 00:00:00 +0000
@@ -1,164 +0,0 @@
-# Copyright 2009-2010 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-__metaclass__ = type
-
-__all__ = [
- 'Hooks',
- 'PoppyInterfaceFailure',
- ]
-
-
-import logging
-import os
-import shutil
-import stat
-import time
-
-from contrib.glock import GlobalLock
-
-
-class PoppyInterfaceFailure(Exception):
- pass
-
-
-class Hooks:
-
- clients = {}
- LOG_MAGIC = "Post-processing finished"
- _targetcount = 0
-
- def __init__(self, targetpath, logger, allow_user, cmd=None,
- targetstart=0, perms=None, prefix=''):
- self.targetpath = targetpath
- self.logger = logging.getLogger("%s.Hooks" % logger.name)
- self.cmd = cmd
- self.allow_user = allow_user
- self.perms = perms
- self.prefix = prefix
-
- @property
- def targetcount(self):
- """A guaranteed unique integer for ensuring unique upload dirs."""
- Hooks._targetcount += 1
- return Hooks._targetcount
-
- def new_client_hook(self, fsroot, host, port):
- """Prepare a new client record indexed by fsroot..."""
- self.clients[fsroot] = {
- "host": host,
- "port": port
- }
- self.logger.debug("Accepting new session in fsroot: %s" % fsroot)
- self.logger.debug("Session from %s:%s" % (host, port))
-
- def client_done_hook(self, fsroot, host, port):
- """A client has completed. If it authenticated then it stands a chance
- of having uploaded a file to the set. If not; then it is simply an
- aborted transaction and we remove the fsroot."""
-
- if fsroot not in self.clients:
- raise PoppyInterfaceFailure("Unable to find fsroot in client set")
-
- self.logger.debug("Processing session complete in %s" % fsroot)
-
- client = self.clients[fsroot]
- if "distro" not in client:
- # Login username defines the distribution context of the upload.
- # So abort unauthenticated sessions by removing its contents
- shutil.rmtree(fsroot)
- return
-
- # Protect from race condition between creating the directory
- # and creating the distro file, and also in cases where the
- # temporary directory and the upload directory are not in the
- # same filesystem (non-atomic "rename").
- lockfile_path = os.path.join(self.targetpath, ".lock")
- self.lock = GlobalLock(lockfile_path)
-
- # XXX cprov 20071024 bug=156795: We try to acquire the lock as soon
- # as possible after creating the lockfile but are still open to
- # a race.
- self.lock.acquire(blocking=True)
- mode = stat.S_IMODE(os.stat(lockfile_path).st_mode)
-
- # XXX cprov 20081024 bug=185731: The lockfile permission can only be
- # changed by its owner. Since we can't predict which process will
- # create it in production systems we simply ignore errors when trying
- # to grant the right permission. At least, one of the process will
- # be able to do so.
- try:
- os.chmod(lockfile_path, mode | stat.S_IWGRP)
- except OSError:
- pass
-
- try:
- timestamp = time.strftime("%Y%m%d-%H%M%S")
- path = "upload%s-%s-%06d" % (
- self.prefix, timestamp, self.targetcount)
- target_fsroot = os.path.join(self.targetpath, path)
-
- # Create file to store the distro used.
- self.logger.debug("Upload was targetted at %s" % client["distro"])
- distro_filename = target_fsroot + ".distro"
- distro_file = open(distro_filename, "w")
- distro_file.write(client["distro"])
- distro_file.close()
-
- # Move the session directory to the target directory.
- if os.path.exists(target_fsroot):
- self.logger.warn("Targeted upload already present: %s" % path)
- self.logger.warn("System clock skewed ?")
- else:
- try:
- shutil.move(fsroot, target_fsroot)
- except (OSError, IOError):
- if not os.path.exists(target_fsroot):
- raise
-
- # XXX cprov 20071024: We should replace os.system call by os.chmod
- # and fix the default permission value accordingly in poppy-upload
- if self.perms is not None:
- os.system("chmod %s -R %s" % (self.perms, target_fsroot))
-
- # Invoke processing script, if provided.
- if self.cmd:
- cmd = self.cmd
- cmd = cmd.replace("@fsroot@", target_fsroot)
- cmd = cmd.replace("@distro@", client["distro"])
- self.logger.debug("Running upload handler: %s" % cmd)
- os.system(cmd)
- finally:
- # We never delete the lockfile, this way the inode will be
- # constant while the machine is up. See comment on 'acquire'
- self.lock.release(skip_delete=True)
-
- self.clients.pop(fsroot)
- # This is mainly done so that tests know when the
- # post-processing hook has finished.
- self.logger.info(self.LOG_MAGIC)
-
- def auth_verify_hook(self, fsroot, user, password):
- """Verify that the username matches a distribution we care about.
-
- The password is irrelevant to auth, as is the fsroot"""
- if fsroot not in self.clients:
- raise PoppyInterfaceFailure("Unable to find fsroot in client set")
-
- # local authentication
- self.clients[fsroot]["distro"] = self.allow_user
- return True
-
- # When we get on with the poppy path stuff, the below may be useful
- # and is thus left in rather than being removed.
-
- #try:
- # d = Distribution.byName(user)
- # if d:
- # self.logger.debug("Accepting login for %s" % user)
- # self.clients[fsroot]["distro"] = user
- # return True
- #except object as e:
- # print e
- #return False
-
=== removed directory 'lib/lp/poppy/tests'
=== removed file 'lib/lp/poppy/tests/__init__.py'
--- lib/lp/poppy/tests/__init__.py 2010-03-18 10:15:34 +0000
+++ lib/lp/poppy/tests/__init__.py 1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-# Copyright 2010 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for Poppy."""
-
=== removed file 'lib/lp/poppy/tests/filesystem.txt'
--- lib/lp/poppy/tests/filesystem.txt 2010-03-17 11:04:20 +0000
+++ lib/lp/poppy/tests/filesystem.txt 1970-01-01 00:00:00 +0000
@@ -1,607 +0,0 @@
-
-This is an implementation of IFileSystem which the FTP Server in Zope3
-uses to know what to do when people make FTP commands.
-
- >>> from lp.poppy.filesystem import UploadFileSystem
-
-The UploadFileSystem class implements the interface IFileSystem.
-
- >>> from zope.server.interfaces.ftp import IFileSystem
- >>> IFileSystem.implementedBy(UploadFileSystem)
- True
-
-First we need to setup our test environment.
-
- >>> import os
- >>> import shutil
- >>> import tempfile
- >>> rootpath = tempfile.mkdtemp()
-
- >>> testfile = "testfile"
- >>> full_testfile = os.path.join(rootpath, testfile)
- >>> testfile_contents = "contents of the file"
- >>> open(full_testfile, 'w').write(testfile_contents)
-
- >>> testdir = "testdir"
- >>> full_testdir = os.path.join(rootpath, testdir)
- >>> os.mkdir(full_testdir)
- >>> propaganda = """
- ... GNU is aimed initially at machines in the 68000/16000 class with
- ... virtual memory, because they are the easiest machines to make it run
- ... on. The extra effort to make it run on smaller machines will be left
- ... to someone who wants to use it on them.
- ... """
-
-When you create an UploadFileSystem you pass it a directory location
-to use.
-
- >>> ufs = UploadFileSystem(rootpath)
-
-An UploadFileSystem object provides the interface IFileSystem.
-
- >>> from zope.interface.verify import verifyObject
- >>> verifyObject(IFileSystem, ufs)
- True
-
-mkdir
-=====
-
-"mkdir" should work as expected, directory will be created as
-requested by the clients:
-
- >>> ufs.mkdir("anything")
- >>> os.path.exists(os.path.join(rootpath, "anything"))
- True
-
- >>> os.rmdir(os.path.join(rootpath, "anything"))
-
-It recursively creates directories:
-
- >>> ufs.mkdir("anything/something/whatever")
-
- >>> wanted_path = os.path.join(rootpath, "anything/something/whatever")
-
- >>> os.path.exists(wanted_path)
- True
-
- >>> oct(os.stat(wanted_path).st_mode)
- '040775'
-
- >>> shutil.rmtree(os.path.join(rootpath, "anything"))
-
-rmdir
-=====
-
-Check if it complains on removal request of an existent dir
-
- >>> ufs.rmdir("does-not-exist")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] does-not-exist
-
-Check if it works as expected after the directory creation:
-
- >>> ufs.mkdir("new-dir")
- >>> ufs.rmdir("new-dir")
-
- >>> os.path.exists(os.path.join(rootpath, "new-dir"))
- False
-
-
-lsinfo
-======
-
-Return information for a unix-style ls listing for the path.
-
-See zope3's interfaces/ftp.py:IFileSystem for details of the
-dictionary's content.
-
-Setup a default dictionary used for generating the dictionaries we
-expect lsinfo to return.
-
- >>> def clean_mtime(stat_info):
- ... """Return a datetime from an mtime, sans microseconds."""
- ... mtime = stat_info.st_mtime
- ... datestamp = datetime.datetime.fromtimestamp(mtime)
- ... datestamp.replace(microsecond=0)
- ... return datestamp
-
- >>> import copy
- >>> import datetime
- >>> import stat
- >>> def_exp = {"type": 'f',
- ... "owner_name": "upload",
- ... "owner_readable": True,
- ... "owner_writable": True,
- ... "owner_executable": False,
- ... "group_name": "upload",
- ... "group_readable": True,
- ... "group_writable": False,
- ... "group_executable": False,
- ... "other_readable": True,
- ... "other_writable": False,
- ... "other_executable": False,
- ... "nlinks": 1}
- ...
-
- >>> os.chmod(full_testfile, stat.S_IRUSR | stat.S_IWUSR | \
- ... stat.S_IRGRP | stat.S_IROTH)
- >>> exp = copy.copy(def_exp)
- >>> s = os.stat(full_testfile)
- >>> exp["name"] = testfile
- >>> exp["mtime"] = clean_mtime(s)
- >>> exp["size"] = s[stat.ST_SIZE]
- >>> info = ufs.lsinfo(testfile)
- >>> info == exp
- True
-
-ls
-==
-
-`ls` a sequence of item info objects (see ls_info) for the files in a
-directory.
-
- >>> expected = [exp]
- >>> for i in [ "foo", "bar" ]:
- ... filename = os.path.join(rootpath, i)
- ... x = open(filename, 'w')
- ... os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | \
- ... stat.S_IRGRP | stat.S_IROTH)
- ... exp = copy.copy(def_exp)
- ... s = os.stat(filename)
- ... exp["name"] = i
- ... exp["mtime"] = clean_mtime(s)
- ... exp["size"] = s[stat.ST_SIZE]
- ... expected.append(exp)
- ...
-
- >>> dir_exp = copy.copy(def_exp)
- >>> s = os.stat(full_testdir)
- >>> dir_exp["type"] = "d"
- >>> dir_exp["name"] = testdir
- >>> dir_exp["mtime"] = clean_mtime(s)
- >>> dir_exp["size"] = s[stat.ST_SIZE]
- >>> dir_exp["nlinks"] = s[stat.ST_NLINK]
- >>> dir_exp["owner_executable"] = True
- >>> dir_exp["other_executable"] = True
- >>> dir_exp["group_executable"] = True
- >>> expected.append(dir_exp)
-
-We need a helper function to turn the returned and expected data into reliably
-sorted orders for comparison.
-
- >>> from operator import itemgetter
- >>> def sorted_listings(ls_infos):
- ... # ls_infos will be a sequence of dictionaries. They need to be
- ... # sorted for the sequences to compare equal, so do that on the
- ... # dictionary's 'name' key. The equality test used here
- ... # doesn't care about the sort order of the dictionaries.
- ... return sorted(ls_infos, key=itemgetter('name'))
-
- >>> expected.sort()
- >>> returned = ufs.ls(".")
- >>> returned.sort()
- >>> sorted_listings(expected) == sorted_listings(returned)
- True
-
-If `filter` is not None, include only those names for which `filter`
-returns a true value.
-
- >>> def always_false_filter(name):
- ... return False
- >>> def always_true_filter(name):
- ... return True
- >>> def arbitrary_filter(name):
- ... if name == "foo" or name == "baz":
- ... return True
- ... else:
- ... return False
- ...
- >>> for i in expected:
- ... if i["name"] == "foo":
- ... filtered_expected = [i];
- >>> returned = ufs.ls(".", always_true_filter)
- >>> returned.sort()
- >>> sorted_listings(expected) == sorted_listings(returned)
- True
- >>> returned = ufs.ls(".", always_false_filter)
- >>> returned.sort()
- >>> returned == []
- True
- >>> returned = ufs.ls(".", arbitrary_filter)
- >>> returned.sort()
- >>> sorted_listings(filtered_expected) == sorted_listings(returned)
- True
- >>> for i in [ "foo", "bar" ]:
- ... ufs.remove(i)
- ...
-
-readfile
-========
-
-We are not implementing `readfile` as a precautionary measure, i.e. in
-case anyone bypasses the per-session separate directories they still
-aren't able to read any other files and therefore can't abuse the
-server for warez/child porn etc.
-
-Unlike `mkdir` and `rmdir` we will raise an exception so that the
-server returns an error to the client and the client does not receive
-bogus or empty data.
-
- >>> ufs.readfile(testfile, None)
- Traceback (most recent call last):
- ...
- Unauthorized
-
-The 'type' command returns 'f' for a file, 'd' for a directory and
-None if there is no file.
-
- >>> ufs.type(testfile)
- 'f'
-
- >>> ufs.type(testdir)
- 'd'
-
- >>> ufs.type("does-not-exist") is None
- True
-
-size
-====
-
-The 'size' command returns the size of the file. If the file does not
-exist None is returned.
-
- >>> ufs.size("does-not-exist") is None
- True
-
- >>> ufs.size(testfile) == os.path.getsize(full_testfile)
- True
-
- >>> ufs.size(testdir) == os.path.getsize(full_testdir)
- True
-
-mtime
-=====
-
-The 'mtime' command returns the mtime of the file. If the file does not
-exist None is returned.
-
- >>> ufs.size("does-not-exist") is None
- True
-
- >>> ufs.mtime(testfile) == os.path.getmtime(full_testfile)
- True
-
- >>> ufs.mtime(testdir) == os.path.getmtime(full_testdir)
- True
-
-remove
-======
-
-The 'remove' command removes a file. An exception is raised if the
-file does not exist or is a directory.
-
- >>> ufs.remove("does-not-exist")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] does-not-exist
-
- >>> ufs.remove(testfile)
- >>> os.path.exists(full_testfile)
- False
- >>> open(full_testfile, 'w').write("contents of the file")
-
- >>> ufs.remove(testdir)
- Traceback (most recent call last):
- ...
- OSError: [Errno Is a directory:] testdir
-
-rename
-======
-
-The 'rename' command renames a file. An exception is raised if the
-old filename doesn't exist or if the old or new filename is a
-directory.
-
- >>> new_testfile = "baz"
- >>> new_full_testfile = os.path.join(rootpath, new_testfile)
-
- >>> ufs.rename("does-not-exist", new_testfile)
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] does-not-exist
-
- >>> new_testfile = "baz"
- >>> new_full_testfile = os.path.join(rootpath, new_testfile)
- >>> ufs.rename(testfile, new_testfile)
- >>> os.path.exists(full_testfile)
- False
- >>> os.path.exists(new_full_testfile)
- True
- >>> open(new_full_testfile).read() == testfile_contents
- True
- >>> ufs.rename(new_testfile, testfile)
-
- >>> ufs.rename(testdir, new_testfile)
- Traceback (most recent call last):
- ...
- OSError: [Errno Is a directory:] testdir
-
- >>> ufs.rename(testfile, testdir)
- Traceback (most recent call last):
- ...
- OSError: [Errno Is a directory:] testdir
-
-names
-=====
-
-The `names` command returns a sequence of the names in the `path`.
-
- >>> sorted(ufs.names("."))
- ['testdir', 'testfile']
-
-`path` is normalized before used.
-
- >>> sorted(ufs.names("some-directory/.."))
- ['testdir', 'testfile']
-
-'path' under the server root is not allowed:
-
- >>> ufs.names("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
-
-
-If the `filter` argument is provided, each name is only returned if
-the given `filter` function returns True for it.
-
- >>> ufs.names(".", always_false_filter)
- []
-
- >>> sorted(ufs.names(".", always_true_filter))
- ['testdir', 'testfile']
-
- >>> for i in [ "foo", "bar", "baz", "bat" ]:
- ... x = open(os.path.join(rootpath, i), 'w')
- >>> names = ufs.names(".", arbitrary_filter)
- >>> names.sort()
- >>> names == ['baz', 'foo']
- True
- >>> for i in [ "foo", "bar", "baz", "bat" ]:
- ... os.unlink(os.path.join(rootpath, i))
-
-writefile
-=========
-
-`writefile` writes data to a file.
-
- >>> from StringIO import StringIO
- >>> ufs.writefile("upload", StringIO(propaganda))
- >>> open(os.path.join(rootpath, "upload")).read() == propaganda
- True
- >>> ufs.remove("upload")
-
-If neither `start` nor `end` are specified, then the file contents
-are overwritten.
-
- >>> ufs.writefile(testfile, StringIO("MOO"))
- >>> open(full_testfile).read() == "MOO"
- True
- >>> ufs.writefile(testfile, StringIO(testfile_contents))
-
-If `start` or `end` are specified, they must be non-negative.
-
- >>> ufs.writefile("upload", StringIO(propaganda), -37)
- Traceback (most recent call last):
- ...
- ValueError: ('Negative start argument:', -37)
-
- >>> ufs.writefile("upload", StringIO(propaganda), 1, -43)
- Traceback (most recent call last):
- ...
- ValueError: ('Negative end argument:', -43)
-
-If `start` or `end` is not None, then only part of the file is
-written. The remainder of the file is unchanged.
-
- >>> ufs.writefile(testfile, StringIO("MOO"), 9, 12)
- >>> open(full_testfile).read() == "contents MOOthe file"
- True
- >>> ufs.writefile(testfile, StringIO(testfile_contents))
-
-If `end` is None, then the file is truncated after the data are
-written.
-
- >>> ufs.writefile(testfile, StringIO("MOO"), 9)
- >>> open(full_testfile).read() == "contents MOO"
- True
- >>> ufs.writefile(testfile, StringIO(testfile_contents))
-
-If `start` is specified and the file doesn't exist or is shorter
-than start, the file will contain undefined data before start.
-
- >>> ufs.writefile("didnt-exist", StringIO("MOO"), 9)
- >>> open(os.path.join(rootpath, "didnt-exist")).read() == "\x00\x00\x00\x00\x00\x00\x00\x00\x00MOO"
- True
- >>> ufs.remove("didnt-exist")
-
-If `end` is not None and there isn't enough data in `instream` to fill
-out the file, then the missing data is undefined.
-
- >>> ufs.writefile(testfile, StringIO("MOO"), 9, 15)
- >>> open(full_testfile).read() == "contents MOOthe file"
- True
- >>> ufs.writefile(testfile, StringIO(testfile_contents))
-
-If `end` is less than or the same as `start no data is writen to the file.
-
- >>> ufs.writefile(testfile, StringIO("MOO"), 9, 4)
- >>> open(full_testfile).read() == "contents of the file"
- True
-
- >>> ufs.writefile(testfile, StringIO("MOO"), 9, 9)
- >>> open(full_testfile).read() == "contents of the file"
- True
-
-If `append` is true the file is appended to rather than being
-overwritten.
-
- >>> ufs.writefile(testfile, StringIO("MOO"), append=True)
- >>> open(full_testfile).read() == "contents of the fileMOO"
- True
- >>> ufs.writefile(testfile, StringIO(testfile_contents))
-
-Additionally, if `append` is true, `start` and `end` are ignored.
-
- >>> ufs.writefile(testfile, StringIO("MOO"), 10, 13, append=True)
- >>> open(full_testfile).read() == "contents of the fileMOO"
- True
- >>> ufs.writefile(testfile, StringIO(testfile_contents))
-
-'writefile' is able to create inexistent directories in a requested
-path:
-
- >>> os.path.exists(os.path.join(rootpath, "foo"))
- False
- >>> ufs.writefile("foo/bar", StringIO("fake")) is None
- True
- >>> os.path.exists(os.path.join(rootpath, "foo/bar"))
- True
- >>> open(os.path.join(rootpath, "foo/bar")).read()
- 'fake'
-
-
-writable
-========
-
-`writable` returns a boolean indicating whether `path` is writable or
-not.
-
- >>> ufs.writable(testfile)
- True
-
-`writable` returns True if `path` is a non-existent file.
-
- >>> ufs.writable("does-not-exist")
- True
-
-`writable` returns False if `path` is a directory as we don't allow
-the creation of sub-directories.
-
- >>> ufs.writable(testdir)
- False
-
-path checking
-=============
-
-`path` arguments must be normalized.
-
- >>> ufs.type(os.path.join("non-existent-dir", "..", testfile))
- 'f'
-
-
-Cleanup the server root:
-
- >>> for leaf in os.listdir(rootpath):
- ... full_path = os.path.join(rootpath, leaf)
- ... if os.path.isdir(full_path):
- ... shutil.rmtree(full_path)
- ... else:
- ... os.remove(full_path)
-
-
-Dealing with inexistent path:
-
- >>> ufs.type("foo/bar") is None
- True
- >>> ufs.mtime("foo/bar") is None
- True
- >>> ufs.size("foo/bar") is None
- True
- >>> ufs.writable("foo/bar")
- True
- >>> ufs.names("foo/bar")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] foo/bar
- >>> ufs.ls("foo/bar")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] foo/bar
- >>> ufs.lsinfo("foo/bar")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] foo/bar
- >>> ufs.remove("foo/bar")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] foo/bar
- >>> ufs.rename("foo/bar", "baz")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] foo/bar
- >>> ufs.rename("baz", "foo/bar")
- Traceback (most recent call last):
- ...
- OSError: [Errno Not exists:] baz
-
-
-Dealing with paths outside the server root directory:
-
- >>> ufs.type("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.mtime("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.size("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.writable("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.names("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.ls("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.lsinfo("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.remove("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.rename("..", "baz")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.rename("baz", "..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.mkdir("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
- >>> ufs.rmdir("..")
- Traceback (most recent call last):
- ...
- OSError: [Errno Path not allowed:] ..
-
-
-------------------------------------------------------------------------
-
-Finally, cleanup after ourselves.
-
- >>> shutil.rmtree(rootpath)
-
=== removed file 'lib/lp/poppy/tests/test_filesystem.py'
--- lib/lp/poppy/tests/test_filesystem.py 2011-12-22 05:09:10 +0000
+++ lib/lp/poppy/tests/test_filesystem.py 1970-01-01 00:00:00 +0000
@@ -1,23 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-__metaclass__ = type
-
-import os
-
-from lp.testing.systemdocs import LayeredDocFileSuite
-
-# The setUp() and tearDown() functions ensure that this doctest is not umask
-# dependent.
-def setUp(testobj):
- testobj._old_umask = os.umask(022)
-
-
-def tearDown(testobj):
- os.umask(testobj._old_umask)
-
-
-def test_suite():
- return LayeredDocFileSuite(
- "filesystem.txt",
- setUp=setUp, tearDown=tearDown, stdout_logging=False)
=== removed file 'lib/lp/poppy/tests/test_poppy.py'
--- lib/lp/poppy/tests/test_poppy.py 2012-03-26 05:50:20 +0000
+++ lib/lp/poppy/tests/test_poppy.py 1970-01-01 00:00:00 +0000
@@ -1,388 +0,0 @@
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Functional tests for poppy FTP daemon."""
-
-__metaclass__ = type
-
-import os
-import shutil
-import stat
-import StringIO
-import tempfile
-import time
-import unittest
-
-from bzrlib.tests import (
- condition_id_re,
- exclude_tests_by_condition,
- multiply_tests,
- )
-from bzrlib.transport import get_transport
-from fixtures import (
- EnvironmentVariableFixture,
- Fixture,
- )
-import transaction
-from zope.component import getUtility
-
-from lp.poppy.hooks import Hooks
-from lp.registry.interfaces.ssh import ISSHKeySet
-from lp.services.config import config
-from lp.services.daemons.tachandler import TacTestSetup
-from lp.testing import TestCaseWithFactory
-from lp.testing.layers import (
- ZopelessAppServerLayer,
- ZopelessDatabaseLayer,
- )
-
-
-class FTPServer(Fixture):
- """This is an abstraction of connecting to an FTP server."""
-
- def __init__(self, root_dir, factory):
- self.root_dir = root_dir
- self.port = 2121
-
- def setUp(self):
- super(FTPServer, self).setUp()
- self.poppytac = self.useFixture(PoppyTac(self.root_dir))
-
- def getAnonTransport(self):
- return get_transport(
- 'ftp://anonymous:me@xxxxxxxxxxx@localhost:%s/' % (self.port,))
-
- def getTransport(self):
- return get_transport('ftp://ubuntu:@localhost:%s/' % (self.port,))
-
- def disconnect(self, transport):
- transport._get_connection().close()
-
- def waitForStartUp(self):
- """Wait for the FTP server to start up."""
- pass
-
- def waitForClose(self, number=1):
- """Wait for an FTP connection to close.
-
- Poppy is configured to echo 'Post-processing finished' to stdout
- when a connection closes, so we wait for that to appear in its
- output as a way to tell that the server has finished with the
- connection.
- """
- self.poppytac.waitForPostProcessing(number)
-
-
-class SFTPServer(Fixture):
- """This is an abstraction of connecting to an SFTP server."""
-
- def __init__(self, root_dir, factory):
- self.root_dir = root_dir
- self._factory = factory
- self.port = int(config.poppy.port.partition(':')[2])
-
- def addSSHKey(self, person, public_key_path):
- f = open(public_key_path, 'r')
- try:
- public_key = f.read()
- finally:
- f.close()
- sshkeyset = getUtility(ISSHKeySet)
- key = sshkeyset.new(person, public_key)
- transaction.commit()
- return key
-
- def setUpUser(self, name):
- user = self._factory.makePerson(name=name)
- self.addSSHKey(
- user, os.path.join(os.path.dirname(__file__), 'poppy-sftp.pub'))
- # Set up a temporary home directory for Paramiko's sake
- self._home_dir = tempfile.mkdtemp()
- self.addCleanup(shutil.rmtree, self._home_dir)
- os.mkdir(os.path.join(self._home_dir, '.ssh'))
- os.symlink(
- os.path.join(os.path.dirname(__file__), 'poppy-sftp'),
- os.path.join(self._home_dir, '.ssh', 'id_rsa'))
- self.useFixture(EnvironmentVariableFixture('HOME', self._home_dir))
- self.useFixture(EnvironmentVariableFixture('SSH_AUTH_SOCK', None))
- self.useFixture(EnvironmentVariableFixture('BZR_SSH', 'paramiko'))
-
- def setUp(self):
- super(SFTPServer, self).setUp()
- self.setUpUser('joe')
- self.poppytac = self.useFixture(PoppyTac(self.root_dir))
-
- def disconnect(self, transport):
- transport._get_connection().close()
-
- def waitForStartUp(self):
- pass
-
- def waitForClose(self, number=1):
- self.poppytac.waitForPostProcessing(number)
-
- def getTransport(self):
- return get_transport('sftp://joe@localhost:%s/' % (self.port,))
-
-
-class PoppyTac(TacTestSetup):
- """A SFTP Poppy server fixture.
-
- This class has two distinct roots:
- - the POPPY_ROOT where the test looks for uploaded output.
- - the server root where ssh keys etc go.
- """
-
- def __init__(self, fs_root):
- self.fs_root = fs_root
- # The setUp check for stale pids races with self._root being assigned,
- # so store a plausible path temporarily. Once all fixtures use unique
- # environments this can go.
- self._root = '/var/does/not/exist'
-
- def setUp(self):
- os.environ['POPPY_ROOT'] = self.fs_root
- super(PoppyTac, self).setUp(umask='0')
-
- def setUpRoot(self):
- self._root = tempfile.mkdtemp()
- self.addCleanup(shutil.rmtree, self.root)
-
- @property
- def root(self):
- return self._root
-
- @property
- def tacfile(self):
- return os.path.abspath(
- os.path.join(config.root, 'daemons', 'poppy-sftp.tac'))
-
- @property
- def logfile(self):
- return os.path.join(self.root, 'poppy-sftp.log')
-
- @property
- def pidfile(self):
- return os.path.join(self.root, 'poppy-sftp.pid')
-
- def waitForPostProcessing(self, number=1):
- now = time.time()
- deadline = now + 20
- while now < deadline and not self._hasPostProcessed(number):
- time.sleep(0.1)
- now = time.time()
-
- if now >= deadline:
- raise Exception("Poppy post-processing did not complete")
-
- def _hasPostProcessed(self, number):
- if os.path.exists(self.logfile):
- with open(self.logfile, "r") as logfile:
- occurrences = logfile.read().count(Hooks.LOG_MAGIC)
- return occurrences >= number
- else:
- return False
-
-
-class TestPoppy(TestCaseWithFactory):
- """Test if poppy.py daemon works properly."""
-
- def setUp(self):
- """Set up poppy in a temp dir."""
- super(TestPoppy, self).setUp()
- self.root_dir = self.makeTemporaryDirectory()
- self.server = self.server_factory(self.root_dir, self.factory)
- self.useFixture(self.server)
-
- def _uploadPath(self, path):
- """Return system path of specified path inside an upload.
-
- Only works for a single upload (poppy transaction).
- """
- contents = sorted(os.listdir(self.root_dir))
- upload_dir = contents[1]
- return os.path.join(self.root_dir, upload_dir, path)
-
- def test_change_directory_anonymous(self):
- # Check that FTP access with an anonymous user works.
- transport = self.server.getAnonTransport()
- self.test_change_directory(transport)
-
- def test_change_directory(self, transport=None):
- """Check automatic creation of directories 'cwd'ed in.
-
- Also ensure they are created with proper permission (g+rwxs)
- """
- self.server.waitForStartUp()
-
- if transport is None:
- transport = self.server.getTransport()
- transport.stat('foo/bar') # .stat will implicity chdir for us
-
- self.server.disconnect(transport)
- self.server.waitForClose()
-
- wanted_path = self._uploadPath('foo/bar')
- self.assertTrue(os.path.exists(wanted_path))
- self.assertEqual(os.stat(wanted_path).st_mode, 042775)
-
- def test_mkdir(self):
- # Creating directories on the server makes actual directories where we
- # expect them, and creates them with g+rwxs
- self.server.waitForStartUp()
-
- transport = self.server.getTransport()
- transport.mkdir('foo/bar', mode=None)
-
- self.server.disconnect(transport)
- self.server.waitForClose()
-
- wanted_path = self._uploadPath('foo/bar')
- self.assertTrue(os.path.exists(wanted_path))
- self.assertEqual(os.stat(wanted_path).st_mode, 042775)
-
- def test_rmdir(self):
- """Check recursive RMD (aka rmdir)"""
- self.server.waitForStartUp()
-
- transport = self.server.getTransport()
- transport.mkdir('foo/bar')
- transport.rmdir('foo/bar')
- transport.rmdir('foo')
-
- self.server.disconnect(transport)
- self.server.waitForClose()
-
- wanted_path = self._uploadPath('foo')
- self.assertFalse(os.path.exists(wanted_path))
-
- def test_single_upload(self):
- """Check if the parent directories are created during file upload.
-
- The uploaded file permissions are also special (g+rwxs).
- """
- self.server.waitForStartUp()
-
- transport = self.server.getTransport()
- fake_file = StringIO.StringIO("fake contents")
-
- transport.put_file('foo/bar/baz', fake_file, mode=None)
-
- self.server.disconnect(transport)
- self.server.waitForClose()
-
- wanted_path = self._uploadPath('foo/bar/baz')
- fs_content = open(os.path.join(wanted_path)).read()
- self.assertEqual(fs_content, "fake contents")
- # Expected mode is -rw-rwSr--.
- self.assertEqual(
- os.stat(wanted_path).st_mode,
- stat.S_IROTH | stat.S_ISGID | stat.S_IRGRP | stat.S_IWGRP
- | stat.S_IWUSR | stat.S_IRUSR | stat.S_IFREG)
-
- def test_full_source_upload(self):
- """Check that the connection will deal with multiple files being
- uploaded.
- """
- self.server.waitForStartUp()
-
- transport = self.server.getTransport()
-
- files = ['test-source_0.1.dsc',
- 'test-source_0.1.orig.tar.gz',
- 'test-source_0.1.diff.gz',
- 'test-source_0.1_source.changes']
-
- for upload in files:
- fake_file = StringIO.StringIO(upload)
- file_to_upload = "~ppa-user/ppa/ubuntu/%s" % upload
- transport.put_file(file_to_upload, fake_file, mode=None)
-
- self.server.disconnect(transport)
- self.server.waitForClose()
-
- upload_path = self._uploadPath('')
- self.assertEqual(os.stat(upload_path).st_mode, 042770)
- dir_name = upload_path.split('/')[-2]
- if transport._user == 'joe':
- self.assertEqual(dir_name.startswith('upload-sftp-2'), True)
- elif transport._user == 'ubuntu':
- self.assertEqual(dir_name.startswith('upload-ftp-2'), True)
- for upload in files:
- wanted_path = self._uploadPath(
- "~ppa-user/ppa/ubuntu/%s" % upload)
- fs_content = open(os.path.join(wanted_path)).read()
- self.assertEqual(fs_content, upload)
- # Expected mode is -rw-rwSr--.
- self.assertEqual(
- os.stat(wanted_path).st_mode,
- stat.S_IROTH | stat.S_ISGID | stat.S_IRGRP | stat.S_IWGRP
- | stat.S_IWUSR | stat.S_IRUSR | stat.S_IFREG)
-
- def test_upload_isolation(self):
- """Check if poppy isolates the uploads properly.
-
- Upload should be done atomically, i.e., poppy should isolate the
- context according each connection/session.
- """
- # Perform a pair of sessions with distinct connections in time.
- self.server.waitForStartUp()
-
- conn_one = self.server.getTransport()
- fake_file = StringIO.StringIO("ONE")
- conn_one.put_file('test', fake_file, mode=None)
- self.server.disconnect(conn_one)
- self.server.waitForClose(1)
-
- conn_two = self.server.getTransport()
- fake_file = StringIO.StringIO("TWO")
- conn_two.put_file('test', fake_file, mode=None)
- self.server.disconnect(conn_two)
- self.server.waitForClose(2)
-
- # Perform a pair of sessions with simultaneous connections.
- conn_three = self.server.getTransport()
- conn_four = self.server.getTransport()
-
- fake_file = StringIO.StringIO("THREE")
- conn_three.put_file('test', fake_file, mode=None)
-
- fake_file = StringIO.StringIO("FOUR")
- conn_four.put_file('test', fake_file, mode=None)
-
- self.server.disconnect(conn_three)
- self.server.waitForClose(3)
-
- self.server.disconnect(conn_four)
- self.server.waitForClose(4)
-
- # Build a list of directories representing the 4 sessions.
- upload_dirs = [leaf for leaf in sorted(os.listdir(self.root_dir))
- if not leaf.startswith(".") and
- not leaf.endswith(".distro")]
- self.assertEqual(len(upload_dirs), 4)
-
- # Check the contents of files on each session.
- expected_contents = ['ONE', 'TWO', 'THREE', 'FOUR']
- for index in range(4):
- content = open(os.path.join(
- self.root_dir, upload_dirs[index], "test")).read()
- self.assertEqual(content, expected_contents[index])
-
-
-def test_suite():
- tests = unittest.TestLoader().loadTestsFromName(__name__)
- scenarios = [
- ('ftp', {'server_factory': FTPServer,
- # XXX: In an ideal world, this would be in the UnitTests
- # layer. Let's get one step closer to that ideal world.
- 'layer': ZopelessDatabaseLayer}),
- ('sftp', {'server_factory': SFTPServer,
- 'layer': ZopelessAppServerLayer}),
- ]
- suite = unittest.TestSuite()
- multiply_tests(tests, scenarios, suite)
- # SFTP doesn't have the concept of the server changing directories, since
- # clients will only send absolute paths, so drop that test.
- return exclude_tests_by_condition(
- suite, condition_id_re(r'test_change_directory.*\(sftp\)$'))
=== removed file 'lib/lp/poppy/tests/test_twistedsftp.py'
--- lib/lp/poppy/tests/test_twistedsftp.py 2015-01-06 12:47:59 +0000
+++ lib/lp/poppy/tests/test_twistedsftp.py 1970-01-01 00:00:00 +0000
@@ -1,70 +0,0 @@
-# Copyright 2010-2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Tests for twistedsftp."""
-
-__metaclass__ = type
-
-import os
-
-from fixtures import TempDir
-from lazr.sshserver.sftp import FileIsADirectory
-
-from lp.poppy.twistedsftp import SFTPServer
-from lp.testing import (
- NestedTempfile,
- TestCase,
- )
-
-
-class TestSFTPServer(TestCase):
-
- def setUp(self):
- self.useFixture(NestedTempfile())
- self.fs_root = self.useFixture(TempDir()).path
- self.sftp_server = SFTPServer(None, self.fs_root)
- super(TestSFTPServer, self).setUp()
-
- def assertPermissions(self, expected, file_name):
- observed = os.stat(file_name).st_mode
- self.assertEqual(
- expected, observed, "Expected %07o, got %07o, for %s" % (
- expected, observed, file_name))
-
- def test_gotVersion(self):
- # gotVersion always returns an empty dict, since the server does not
- # support any extended features. See ISFTPServer.
- extras = self.sftp_server.gotVersion(None, None)
- self.assertEquals(extras, {})
-
- def test_mkdir_and_rmdir(self):
- self.sftp_server.makeDirectory('foo/bar', None)
- self.assertEqual(
- os.listdir(os.path.join(self.sftp_server._current_upload))[0],
- 'foo')
- dir_name = os.path.join(self.sftp_server._current_upload, 'foo')
- self.assertEqual(os.listdir(dir_name)[0], 'bar')
- self.assertPermissions(040775, dir_name)
- self.sftp_server.removeDirectory('foo/bar')
- self.assertEqual(
- os.listdir(os.path.join(self.sftp_server._current_upload,
- 'foo')), [])
- self.sftp_server.removeDirectory('foo')
- self.assertEqual(
- os.listdir(os.path.join(self.sftp_server._current_upload)), [])
-
- def test_file_creation(self):
- upload_file = self.sftp_server.openFile('foo/bar', None, None)
- upload_file.writeChunk(0, "This is a test")
- file_name = os.path.join(self.sftp_server._current_upload, 'foo/bar')
- test_file = open(file_name, 'r')
- self.assertEqual(test_file.read(), "This is a test")
- test_file.close()
- self.assertPermissions(0100644, file_name)
- dir_name = os.path.join(self.sftp_server._current_upload, 'bar/foo')
- os.makedirs(dir_name)
- upload_file = self.sftp_server.openFile('bar/foo', None, None)
- self.assertRaisesWithContent(
- FileIsADirectory,
- "File is a directory: '%s'" % dir_name,
- upload_file.writeChunk, 0, "This is a test")
=== removed file 'lib/lp/poppy/twistedftp.py'
--- lib/lp/poppy/twistedftp.py 2012-03-26 07:14:35 +0000
+++ lib/lp/poppy/twistedftp.py 1970-01-01 00:00:00 +0000
@@ -1,160 +0,0 @@
-# Copyright 2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Twisted FTP implementation of the Poppy upload server."""
-
-__metaclass__ = type
-__all__ = [
- 'FTPRealm',
- 'PoppyAnonymousShell',
- ]
-
-import logging
-import os
-import tempfile
-
-from twisted.application import (
- service,
- strports,
- )
-from twisted.cred import (
- checkers,
- credentials,
- )
-from twisted.cred.portal import (
- IRealm,
- Portal,
- )
-from twisted.internet import defer
-from twisted.protocols import ftp
-from twisted.python import filepath
-from zope.interface import implements
-
-from lp.poppy import get_poppy_root
-from lp.poppy.filesystem import UploadFileSystem
-from lp.poppy.hooks import Hooks
-from lp.services.config import config
-
-
-class PoppyAccessCheck:
- """An `ICredentialsChecker` for Poppy FTP sessions."""
- implements(checkers.ICredentialsChecker)
- credentialInterfaces = (
- credentials.IUsernamePassword, credentials.IAnonymous)
-
- def requestAvatarId(self, credentials):
- # Poppy allows any credentials. People can use "anonymous" if
- # they want but anything goes. Thus, we don't actually *check* the
- # credentials, and we return the standard avatarId for 'anonymous'.
- return checkers.ANONYMOUS
-
-
-class PoppyAnonymousShell(ftp.FTPShell):
- """The 'command' interface for sessions.
-
- Roughly equivalent to the SFTPServer in the sftp side of things.
- """
-
- def __init__(self, fsroot):
- self._fs_root = fsroot
- self.uploadfilesystem = UploadFileSystem(tempfile.mkdtemp())
- self._current_upload = self.uploadfilesystem.rootpath
- os.chmod(self._current_upload, 0770)
- self._log = logging.getLogger("poppy-sftp")
- self.hook = Hooks(
- self._fs_root, self._log, "ubuntu", perms='g+rws',
- prefix='-ftp')
- self.hook.new_client_hook(self._current_upload, 0, 0)
- self.hook.auth_verify_hook(self._current_upload, None, None)
- super(PoppyAnonymousShell, self).__init__(
- filepath.FilePath(self._current_upload))
-
- def openForWriting(self, file_segments):
- """Write the uploaded file to disk, safely.
-
- :param file_segments: A list containing string items, one for each
- path component of the file being uploaded. The file referenced
- is relative to the temporary root for this session.
-
- If the file path contains directories, we create them.
- """
- filename = os.sep.join(file_segments)
- self._create_missing_directories(filename)
- return super(PoppyAnonymousShell, self).openForWriting(file_segments)
-
- def makeDirectory(self, path):
- """Make a directory using the secure `UploadFileSystem`."""
- path = os.sep.join(path)
- return defer.maybeDeferred(self.uploadfilesystem.mkdir, path)
-
- def access(self, segments):
- """Permissive CWD that auto-creates target directories."""
- if segments:
- path = self._path(segments)
- path.makedirs()
- return super(PoppyAnonymousShell, self).access(segments)
-
- def logout(self):
- """Called when the client disconnects.
-
- We need to post-process the upload.
- """
- self.hook.client_done_hook(self._current_upload, 0, 0)
-
- def _create_missing_directories(self, filename):
- # Same as SFTPServer
- new_dir, new_file = os.path.split(
- self.uploadfilesystem._sanitize(filename))
- if new_dir != '':
- if not os.path.exists(
- os.path.join(self._current_upload, new_dir)):
- self.uploadfilesystem.mkdir(new_dir)
-
- def list(self, path_segments, attrs):
- return defer.fail(ftp.CmdNotImplementedError("LIST"))
-
-
-class FTPRealm:
- """FTP Realm that lets anyone in."""
- implements(IRealm)
-
- def __init__(self, root):
- self.root = root
-
- def requestAvatar(self, avatarId, mind, *interfaces):
- """Return a Poppy avatar - that is, an "authorisation".
-
- Poppy FTP avatars are totally fake, we don't care about credentials.
- See `PoppyAccessCheck` above.
- """
- for iface in interfaces:
- if iface is ftp.IFTPShell:
- avatar = PoppyAnonymousShell(self.root)
- return ftp.IFTPShell, avatar, getattr(
- avatar, 'logout', lambda: None)
- raise NotImplementedError(
- "Only IFTPShell interface is supported by this realm")
-
-
-class FTPServiceFactory(service.Service):
- """A factory that makes an `FTPService`"""
-
- def __init__(self, port):
- realm = FTPRealm(get_poppy_root())
- portal = Portal(realm)
- portal.registerChecker(PoppyAccessCheck())
- factory = ftp.FTPFactory(portal)
-
- factory.tld = get_poppy_root()
- factory.protocol = ftp.FTP
- factory.welcomeMessage = "Launchpad upload server"
- factory.timeOut = config.poppy.idle_timeout
-
- self.ftpfactory = factory
- self.portno = port
-
- @staticmethod
- def makeFTPService(port=2121):
- strport = "tcp:%s" % port
- factory = FTPServiceFactory(port)
- return strports.service(strport, factory.ftpfactory)
=== removed file 'lib/lp/poppy/twistedsftp.py'
--- lib/lp/poppy/twistedsftp.py 2015-01-06 12:47:59 +0000
+++ lib/lp/poppy/twistedsftp.py 1970-01-01 00:00:00 +0000
@@ -1,144 +0,0 @@
-# Copyright 2010-2011 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Twisted SFTP implementation of the Poppy upload server."""
-
-__metaclass__ = type
-__all__ = [
- 'SFTPFile',
- 'SFTPServer',
- ]
-
-import errno
-import logging
-import os
-import tempfile
-
-from lazr.sshserver.events import SFTPClosed
-from lazr.sshserver.sftp import FileIsADirectory
-from twisted.conch.interfaces import (
- ISFTPFile,
- ISFTPServer,
- )
-from zope.component import (
- adapter,
- provideHandler,
- )
-from zope.interface import implements
-
-from lp.poppy.filesystem import UploadFileSystem
-from lp.poppy.hooks import Hooks
-
-
-class SFTPServer:
- """An implementation of `ISFTPServer` that backs onto a Poppy filesystem.
- """
-
- implements(ISFTPServer)
-
- def __init__(self, avatar, fsroot):
- provideHandler(self.connectionClosed)
- self._avatar = avatar
- self._fs_root = fsroot
- self.uploadfilesystem = UploadFileSystem(tempfile.mkdtemp())
- self._current_upload = self.uploadfilesystem.rootpath
- os.chmod(self._current_upload, 0770)
- self._log = logging.getLogger("poppy-sftp")
- self.hook = Hooks(
- self._fs_root, self._log, "ubuntu", perms='g+rws', prefix='-sftp')
- self.hook.new_client_hook(self._current_upload, 0, 0)
- self.hook.auth_verify_hook(self._current_upload, None, None)
-
- def gotVersion(self, other_version, ext_data):
- return {}
-
- def openFile(self, filename, flags, attrs):
- self._create_missing_directories(filename)
- absfile = self._translate_path(filename)
- return SFTPFile(absfile)
-
- def removeFile(self, filename):
- pass
-
- def renameFile(self, old_path, new_path):
- abs_old = self._translate_path(old_path)
- abs_new = self._translate_path(new_path)
- os.rename(abs_old, abs_new)
-
- def makeDirectory(self, path, attrs):
- # XXX: We ignore attrs here
- self.uploadfilesystem.mkdir(path)
-
- def removeDirectory(self, path):
- self.uploadfilesystem.rmdir(path)
-
- def openDirectory(self, path):
- pass
-
- def getAttrs(self, path, follow_links):
- pass
-
- def setAttrs(self, path, attrs):
- pass
-
- def readLink(self, path):
- pass
-
- def makeLink(self, link_path, target_path):
- pass
-
- def realPath(self, path):
- return path
-
- def extendedRequest(self, extended_name, extended_data):
- pass
-
- def _create_missing_directories(self, filename):
- new_dir, new_file = os.path.split(
- self.uploadfilesystem._sanitize(filename))
- if new_dir != '':
- if not os.path.exists(
- os.path.join(self._current_upload, new_dir)):
- self.uploadfilesystem.mkdir(new_dir)
-
- def _translate_path(self, filename):
- return self.uploadfilesystem._full(
- self.uploadfilesystem._sanitize(filename))
-
- @adapter(SFTPClosed)
- def connectionClosed(self, event):
- if event.avatar is not self._avatar:
- return
- self.hook.client_done_hook(self._current_upload, 0, 0)
-
-
-class SFTPFile:
-
- implements(ISFTPFile)
-
- def __init__(self, filename):
- self.filename = filename
-
- def close(self):
- pass
-
- def readChunk(self, offset, length):
- pass
-
- def writeChunk(self, offset, data):
- try:
- chunk_file = os.open(
- self.filename, os.O_CREAT | os.O_WRONLY, 0644)
- except OSError as e:
- if e.errno != errno.EISDIR:
- raise
- raise FileIsADirectory(self.filename)
- os.lseek(chunk_file, offset, 0)
- os.write(chunk_file, data)
- os.close(chunk_file)
-
- def getAttrs(self):
- pass
-
- def setAttrs(self, attr):
- pass
=== modified file 'lib/lp/testing/__init__.py'
--- lib/lp/testing/__init__.py 2013-05-14 05:29:03 +0000
+++ lib/lp/testing/__init__.py 2015-01-13 15:34:27 +0000
@@ -1398,17 +1398,6 @@
return launchpad.load(canonical_url(obj, request=api_request))
-class NestedTempfile(fixtures.Fixture):
- """Nest all temporary files and directories inside a top-level one."""
-
- def setUp(self):
- super(NestedTempfile, self).setUp()
- tempdir = fixtures.TempDir()
- self.useFixture(tempdir)
- patch = fixtures.MonkeyPatch("tempfile.tempdir", tempdir.path)
- self.useFixture(patch)
-
-
@contextmanager
def monkey_patch(context, **kwargs):
"""In the ContextManager scope, monkey-patch values.
=== modified file 'lib/lp/testing/tests/test_testing.py'
--- lib/lp/testing/tests/test_testing.py 2012-06-08 08:54:37 +0000
+++ lib/lp/testing/tests/test_testing.py 2015-01-13 15:34:27 +0000
@@ -6,7 +6,6 @@
__metaclass__ = type
import os
-import tempfile
from lp.services.config import config
from lp.services.features import (
@@ -15,7 +14,6 @@
)
from lp.testing import (
feature_flags,
- NestedTempfile,
set_feature_flag,
TestCase,
YUIUnitTestCase,
@@ -62,31 +60,3 @@
test_path = os.path.join(config.root, "../bar/baz/../bob.html")
test.initialize(test_path)
self.assertEqual("../bar/bob.html", test.id())
-
-
-class NestedTempfileTest(TestCase):
- """Tests for `NestedTempfile`."""
-
- def test_normal(self):
- # The temp directory is removed when the context is exited.
- starting_tempdir = tempfile.gettempdir()
- with NestedTempfile():
- self.assertEqual(tempfile.tempdir, tempfile.gettempdir())
- self.assertNotEqual(tempfile.tempdir, starting_tempdir)
- self.assertTrue(os.path.isdir(tempfile.tempdir))
- nested_tempdir = tempfile.tempdir
- self.assertEqual(tempfile.tempdir, tempfile.gettempdir())
- self.assertEqual(starting_tempdir, tempfile.tempdir)
- self.assertFalse(os.path.isdir(nested_tempdir))
-
- def test_exception(self):
- # The temp directory is removed when the context is exited, even if
- # the code running in context raises an exception.
- class ContrivedException(Exception):
- pass
- try:
- with NestedTempfile():
- nested_tempdir = tempfile.tempdir
- raise ContrivedException
- except ContrivedException:
- self.assertFalse(os.path.isdir(nested_tempdir))
=== modified file 'utilities/snakefood/lp-sfood-packages'
--- utilities/snakefood/lp-sfood-packages 2011-12-30 06:47:17 +0000
+++ utilities/snakefood/lp-sfood-packages 2015-01-13 15:34:27 +0000
@@ -5,7 +5,6 @@
lp/services
lp/scripts
lp/registry
-lp/poppy
lp/hardwaredb
lp/coop/answersbugs
lp/codehosting
=== modified file 'utilities/start-dev-soyuz.sh'
--- utilities/start-dev-soyuz.sh 2011-12-08 01:38:24 +0000
+++ utilities/start-dev-soyuz.sh 2015-01-13 15:34:27 +0000
@@ -14,11 +14,24 @@
-y "$tac" $@
}
+start_twistd_plugin() {
+ # Start twistd for plugin service $1.
+ name=$1
+ plugin=$2
+ shift 2
+ echo "Starting $name."
+ "bin/twistd-for-$name" \
+ --logfile "/var/tmp/development-$name.log" \
+ --pidfile "/var/tmp/development-$name.pid" \
+ "$plugin" "$@"
+}
+
start_twistd testkeyserver lib/lp/testing/keyserver/testkeyserver.tac
start_twistd buildd-manager daemons/buildd-manager.tac
-mkdir -p /var/tmp/poppy/incoming
-export POPPY_ROOT=/var/tmp/poppy/incoming
-start_twistd poppy-sftp daemons/poppy-sftp.tac
+mkdir -p /var/tmp/txpkgupload/incoming
+export TXPKGUPLOAD_ROOT=/var/tmp/txpkgupload/incoming
+start_twistd_plugin txpkgupload pkgupload \
+ -c configs/development/txpkgupload.yaml
echo "Done."
=== modified file 'versions.cfg'
--- versions.cfg 2015-01-06 12:47:59 +0000
+++ versions.cfg 2015-01-13 15:34:27 +0000
@@ -32,6 +32,7 @@
FeedParser = 4.1
feedvalidator = 0.0.0DEV-r1049
fixtures = 0.3.9
+FormEncode = 1.2.4
funkload = 1.16.1
grokcore.component = 1.6
html5browser = 0.0.9
@@ -104,6 +105,7 @@
python-openid = 2.2.5-fix1034376
python-subunit = 0.0.8beta
python-swiftclient = 1.5.0
+PyYAML = 3.10
rabbitfixture = 0.3.5
requests = 1.2.3
s4 = 0.1.2
@@ -125,6 +127,7 @@
txfixtures = 0.1.4
txlongpoll = 0.2.12
txlongpollfixture = 0.1.3
+txpkgupload = 0.1.1
unittest2 = 0.5.1
van.testing = 3.0.0
wadllib = 1.3.2
Follow ups
