launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #22455
[Merge] lp:~cjwatson/launchpad/scandir into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/scandir into lp:launchpad with lp:~cjwatson/launchpad/importlib-resources as a prerequisite.
Commit message:
Convert all uses of os.walk and many uses of os.listdir to scandir.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/scandir/+merge/345139
When switching to importlib-resources, I noticed scandir among the new dependencies, and it seems like a good idea. Replacing os.walk is straightforward; replacing os.listdir isn't always worth it, so I left many instances of that alone, but in some cases it's a clear improvement.
We lose the randomisation of os.listdir performed by the test harness when switching to scandir.scandir, which is a bit of a shame, but there's no good way to do that randomisation while preserving the iteration property, so it would potentially leave us open to problems caused by unlinking files while iterating over their containing directory. This seems a reasonable trade-off.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/scandir into lp:launchpad.
=== modified file 'lib/devscripts/sourcecode.py'
--- lib/devscripts/sourcecode.py 2012-06-25 12:21:10 +0000
+++ lib/devscripts/sourcecode.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Tools for maintaining the Launchpad source code."""
@@ -32,6 +32,7 @@
)
from bzrlib.upgrade import upgrade
from bzrlib.workingtree import WorkingTree
+import scandir
from devscripts import get_launchpad_root
@@ -136,12 +137,10 @@
def find_branches(directory):
"""List the directory names in 'directory' that are branches."""
branches = []
- for name in os.listdir(directory):
- if name in ('.', '..'):
- continue
+ for entry in scandir.scandir(directory):
try:
- Branch.open(os.path.join(directory, name))
- branches.append(name)
+ Branch.open(entry.path)
+ branches.append(entry.name)
except NotBranchError:
pass
return branches
=== modified file 'lib/lp/archivepublisher/customupload.py'
--- lib/lp/archivepublisher/customupload.py 2018-01-19 16:29:38 +0000
+++ lib/lp/archivepublisher/customupload.py 2018-05-06 14:00:16 +0000
@@ -20,6 +20,7 @@
import tarfile
import tempfile
+import scandir
from zope.interface import implementer
from lp.archivepublisher.debversion import (
@@ -287,7 +288,7 @@
"""Install the files from the custom upload to the archive."""
assert self.tmpdir is not None, "Must extract tarfile first"
extracted = False
- for dirpath, dirnames, filenames in os.walk(self.tmpdir):
+ for dirpath, dirnames, filenames in scandir.walk(self.tmpdir):
# Create symbolic links to directories.
for dirname in dirnames:
@@ -356,17 +357,17 @@
# now present in the target. Deliberately skip 'broken' versions
# because they can't be sorted anyway.
versions = []
- for inst in os.listdir(self.targetdir):
+ for entry in scandir.scandir(self.targetdir):
# Skip the symlink.
- if inst == 'current':
+ if entry.name == 'current':
continue
# Skip broken versions.
try:
- make_version(inst)
+ make_version(entry.name)
except VersionError:
continue
# Append the valid versions to the list.
- versions.append(inst)
+ versions.append(entry.name)
versions.sort(key=make_version, reverse=True)
# Make sure the 'current' symlink points to the most recent version
=== modified file 'lib/lp/archivepublisher/model/ftparchive.py'
--- lib/lp/archivepublisher/model/ftparchive.py 2016-09-24 04:24:30 +0000
+++ lib/lp/archivepublisher/model/ftparchive.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
from collections import defaultdict
@@ -7,6 +7,7 @@
from StringIO import StringIO
import time
+import scandir
from storm.expr import (
Desc,
Join,
@@ -60,16 +61,16 @@
If omitted, all files are removed.
"""
if os.path.isdir(path):
- for name in os.listdir(path):
- if name == "by-hash" or not re.match(clean_pattern, name):
+ for entry in list(scandir.scandir(path)):
+ if (entry.name == "by-hash" or
+ not re.match(clean_pattern, entry.name)):
# Ignore existing by-hash directories; they will be cleaned
# up to match the rest of the directory tree later.
continue
- child_path = os.path.join(path, name)
# Directories containing index files should never have
# subdirectories other than by-hash. Guard against expensive
# mistakes by not recursing here.
- os.unlink(child_path)
+ os.unlink(entry.path)
else:
os.makedirs(path, 0o755)
=== modified file 'lib/lp/archivepublisher/publishing.py'
--- lib/lp/archivepublisher/publishing.py 2018-03-27 23:26:12 +0000
+++ lib/lp/archivepublisher/publishing.py 2018-05-06 14:00:16 +0000
@@ -34,6 +34,7 @@
_multivalued,
Release,
)
+import scandir
from storm.expr import Desc
from zope.component import getUtility
from zope.interface import (
@@ -377,12 +378,12 @@
hash_path = os.path.join(self.path, archive_hash.apt_name)
if os.path.exists(hash_path):
prune_hash_directory = True
- for digest in list(os.listdir(hash_path)):
- if digest not in self.known_digests[archive_hash.apt_name]:
- digest_path = os.path.join(hash_path, digest)
+ for entry in list(scandir.scandir(hash_path)):
+ if entry.name not in self.known_digests[
+ archive_hash.apt_name]:
self.log.debug(
- "by-hash: Deleting unreferenced %s" % digest_path)
- os.unlink(digest_path)
+ "by-hash: Deleting unreferenced %s" % entry.path)
+ os.unlink(entry.path)
else:
prune_hash_directory = False
if prune_hash_directory:
@@ -1202,11 +1203,11 @@
distroseries, pocket, component, core_files)
dep11_dir = os.path.join(suite_dir, component, "dep11")
try:
- for dep11_file in os.listdir(dep11_dir):
- if (dep11_file.startswith("Components-") or
- dep11_file.startswith("icons-")):
+ for entry in scandir.scandir(dep11_dir):
+ if (entry.name.startswith("Components-") or
+ entry.name.startswith("icons-")):
dep11_path = os.path.join(
- component, "dep11", dep11_file)
+ component, "dep11", entry.name)
extra_files.add(remove_suffix(dep11_path))
extra_files.add(dep11_path)
except OSError as e:
@@ -1359,11 +1360,11 @@
i18n_dir = os.path.join(self._config.distsroot, suite, i18n_subpath)
i18n_files = set()
try:
- for i18n_file in os.listdir(i18n_dir):
- if not i18n_file.startswith('Translation-'):
+ for entry in scandir.scandir(i18n_dir):
+ if not entry.name.startswith('Translation-'):
continue
- i18n_files.add(remove_suffix(i18n_file))
- i18n_files.add(i18n_file)
+ i18n_files.add(remove_suffix(entry.name))
+ i18n_files.add(entry.name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
@@ -1556,7 +1557,7 @@
def add_dir(self, path):
"""Recursively add a directory path to be checksummed."""
- for dirpath, dirnames, filenames in os.walk(path):
+ for dirpath, dirnames, filenames in scandir.walk(path):
for filename in filenames:
self.add(os.path.join(dirpath, filename))
=== modified file 'lib/lp/archivepublisher/scripts/publish_ftpmaster.py'
--- lib/lp/archivepublisher/scripts/publish_ftpmaster.py 2018-01-18 15:31:02 +0000
+++ lib/lp/archivepublisher/scripts/publish_ftpmaster.py 2018-05-06 14:00:16 +0000
@@ -14,6 +14,7 @@
import shutil
from pytz import utc
+import scandir
from zope.component import getUtility
from lp.archivepublisher.config import getPubConfig
@@ -472,7 +473,7 @@
backup_top = os.path.join(get_backup_dists(archive_config), suite)
staging_top = os.path.join(archive_config.stagingroot, suite)
updated = False
- for staging_dir, _, filenames in os.walk(staging_top):
+ for staging_dir, _, filenames in scandir.walk(staging_top):
rel_dir = os.path.relpath(staging_dir, staging_top)
backup_dir = os.path.join(backup_top, rel_dir)
for filename in filenames:
=== modified file 'lib/lp/archivepublisher/signing.py'
--- lib/lp/archivepublisher/signing.py 2018-01-19 16:29:38 +0000
+++ lib/lp/archivepublisher/signing.py 2018-05-06 14:00:16 +0000
@@ -26,6 +26,8 @@
import tempfile
import textwrap
+import scandir
+
from lp.archivepublisher.config import getPubConfig
from lp.archivepublisher.customupload import CustomUpload
from lp.services.osutils import remove_if_exists
@@ -166,7 +168,7 @@
def findSigningHandlers(self):
"""Find all the signable files in an extracted tarball."""
- for dirpath, dirnames, filenames in os.walk(self.tmpdir):
+ for dirpath, dirnames, filenames in scandir.walk(self.tmpdir):
for filename in filenames:
if filename.endswith(".efi"):
yield (os.path.join(dirpath, filename), self.signUefi)
=== modified file 'lib/lp/archivepublisher/tests/test_publisher.py'
--- lib/lp/archivepublisher/tests/test_publisher.py 2018-04-05 11:32:50 +0000
+++ lib/lp/archivepublisher/tests/test_publisher.py 2018-05-06 14:00:16 +0000
@@ -38,6 +38,7 @@
from backports import lzma
import mock
import pytz
+import scandir
from testscenarios import (
load_tests_apply_scenarios,
WithScenarios,
@@ -513,7 +514,7 @@
def match(self, root):
children = set()
- for dirpath, dirnames, _ in os.walk(root):
+ for dirpath, dirnames, _ in scandir.walk(root):
if "by-hash" in dirnames:
children.add(os.path.relpath(dirpath, root))
mismatch = MatchesSetwise(
=== modified file 'lib/lp/archivepublisher/tests/test_signing.py'
--- lib/lp/archivepublisher/tests/test_signing.py 2018-03-28 09:30:48 +0000
+++ lib/lp/archivepublisher/tests/test_signing.py 2018-05-06 14:00:16 +0000
@@ -12,6 +12,7 @@
import tarfile
from fixtures import MonkeyPatch
+import scandir
from testtools.matchers import (
Contains,
Equals,
@@ -59,7 +60,7 @@
def match(self, base):
content = []
- for root, dirs, files in os.walk(base):
+ for root, dirs, files in scandir.walk(base):
content.extend(
[os.path.relpath(os.path.join(root, f), base) for f in files])
=== modified file 'lib/lp/archiveuploader/dscfile.py'
--- lib/lp/archiveuploader/dscfile.py 2018-03-02 16:17:35 +0000
+++ lib/lp/archiveuploader/dscfile.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
""" DSCFile and related.
@@ -30,6 +30,7 @@
Deb822Dict,
PkgRelation,
)
+import scandir
from zope.component import getUtility
from lp.app.errors import NotFoundError
@@ -613,8 +614,8 @@
# Check if 'dpkg-source' created only one directory.
temp_directories = [
- dirname for dirname in os.listdir(unpacked_dir)
- if os.path.isdir(dirname)]
+ entry.name for entry in scandir.scandir(unpacked_dir)
+ if entry.is_dir()]
if len(temp_directories) > 1:
yield UploadError(
'Unpacked source contains more than one directory: %r'
=== modified file 'lib/lp/archiveuploader/livefsupload.py'
--- lib/lp/archiveuploader/livefsupload.py 2015-01-23 17:57:59 +0000
+++ lib/lp/archiveuploader/livefsupload.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2014-2015 Canonical Ltd. This software is licensed under the
+# Copyright 2014-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Process a live filesystem upload."""
@@ -7,6 +7,7 @@
import os
+import scandir
from zope.component import getUtility
from lp.buildmaster.enums import BuildStatus
@@ -36,7 +37,7 @@
"""Process this upload, loading it into the database."""
self.logger.debug("Beginning processing.")
- for dirpath, _, filenames in os.walk(self.upload_path):
+ for dirpath, _, filenames in scandir.walk(self.upload_path):
if dirpath == self.upload_path:
# All relevant files will be in a subdirectory.
continue
=== modified file 'lib/lp/archiveuploader/snapupload.py'
--- lib/lp/archiveuploader/snapupload.py 2016-08-30 12:34:51 +0000
+++ lib/lp/archiveuploader/snapupload.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2015-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Process a snap package upload."""
@@ -7,6 +7,7 @@
import os
+import scandir
from zope.component import getUtility
from lp.archiveuploader.utils import UploadError
@@ -39,7 +40,7 @@
found_snap = False
snap_paths = []
- for dirpath, _, filenames in os.walk(self.upload_path):
+ for dirpath, _, filenames in scandir.walk(self.upload_path):
if dirpath == self.upload_path:
# All relevant files will be in a subdirectory.
continue
=== modified file 'lib/lp/archiveuploader/uploadprocessor.py'
--- lib/lp/archiveuploader/uploadprocessor.py 2018-03-02 16:17:35 +0000
+++ lib/lp/archiveuploader/uploadprocessor.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Code for 'processing' 'uploads'. Also see nascentupload.py.
@@ -51,6 +51,7 @@
import shutil
import sys
+import scandir
from sqlobject import SQLObjectNotFound
from zope.component import getUtility
@@ -210,8 +211,7 @@
alphabetically sorted.
"""
return sorted(
- dir_name for dir_name in os.listdir(fsroot)
- if os.path.isdir(os.path.join(fsroot, dir_name)))
+ entry.name for entry in scandir.scandir(fsroot) if entry.is_dir())
class UploadHandler:
@@ -258,7 +258,7 @@
"""
changes_files = []
- for dirpath, dirnames, filenames in os.walk(self.upload_path):
+ for dirpath, dirnames, filenames in scandir.walk(self.upload_path):
relative_path = dirpath[len(self.upload_path) + 1:]
for filename in filenames:
if filename.endswith(".changes"):
=== modified file 'lib/lp/bugs/tests/test_doc.py'
--- lib/lp/bugs/tests/test_doc.py 2016-09-07 22:41:08 +0000
+++ lib/lp/bugs/tests/test_doc.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""
@@ -9,6 +9,8 @@
import os
import unittest
+import scandir
+
from lp.code.tests.test_doc import branchscannerSetUp
from lp.services.config import config
from lp.services.features.testing import FeatureFixture
@@ -482,11 +484,10 @@
stories_dir = os.path.join(os.path.pardir, 'stories')
suite.addTest(PageTestSuite(stories_dir))
stories_path = os.path.join(here, stories_dir)
- for story_dir in os.listdir(stories_path):
- full_story_dir = os.path.join(stories_path, story_dir)
- if not os.path.isdir(full_story_dir):
+ for story_entry in scandir.scandir(stories_path):
+ if not story_entry.is_dir():
continue
- story_path = os.path.join(stories_dir, story_dir)
+ story_path = os.path.join(stories_dir, story_entry.name)
suite.addTest(PageTestSuite(story_path))
testsdir = os.path.abspath(
=== modified file 'lib/lp/codehosting/codeimport/tests/test_worker.py'
--- lib/lp/codehosting/codeimport/tests/test_worker.py 2018-03-15 20:44:04 +0000
+++ lib/lp/codehosting/codeimport/tests/test_worker.py 2018-05-06 14:00:16 +0000
@@ -45,6 +45,7 @@
)
from dulwich.repo import Repo as GitRepo
from fixtures import FakeLogger
+import scandir
import subvertpy
import subvertpy.client
import subvertpy.ra
@@ -136,7 +137,7 @@
`directory1` are laid out in the same way as `directory2`.
"""
def list_files(directory):
- for path, ignored, ignored in os.walk(directory):
+ for path, ignored, ignored in scandir.walk(directory):
yield path[len(directory):]
self.assertEqual(
sorted(list_files(directory1)), sorted(list_files(directory2)))
=== modified file 'lib/lp/registry/model/codeofconduct.py'
--- lib/lp/registry/model/codeofconduct.py 2018-03-02 16:17:35 +0000
+++ lib/lp/registry/model/codeofconduct.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""A module for CodeOfConduct (CoC) related classes.
@@ -14,6 +14,7 @@
import os
import pytz
+import scandir
from sqlobject import (
BoolCol,
ForeignKey,
@@ -151,11 +152,11 @@
cocs_path = getUtility(ICodeOfConductConf).path
# iter through files and store the CoC Object
- for filename in os.listdir(cocs_path):
+ for entry in scandir.scandir(cocs_path):
# Select the correct filenames
- if filename.endswith('.txt'):
+ if entry.name.endswith('.txt'):
# Extract the version from filename
- version = filename.replace('.txt', '')
+ version = entry.name.replace('.txt', '')
releases.append(CodeOfConduct(version))
# Return the available list of CoCs objects
=== modified file 'lib/lp/registry/scripts/productreleasefinder/walker.py'
--- lib/lp/registry/scripts/productreleasefinder/walker.py 2017-10-21 18:14:14 +0000
+++ lib/lp/registry/scripts/productreleasefinder/walker.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""HTTP and FTP walker.
@@ -30,6 +30,7 @@
InvalidURIError,
URI,
)
+import scandir
from lp.registry.scripts.productreleasefinder import log
from lp.services.beautifulsoup import BeautifulSoup
@@ -63,7 +64,7 @@
"""Base class for URL walkers.
This class is a base class for those wishing to implement protocol
- specific walkers. Walkers behave much like the os.walk() function,
+ specific walkers. Walkers behave much like the scandir.walk() function,
but taking a URL and working remotely.
A typical usage would be:
@@ -116,7 +117,7 @@
"""Walk through the URL.
Yields (dirpath, dirnames, filenames) for each path under the base;
- dirnames can be modified as with os.walk.
+ dirnames can be modified as with scandir.walk.
"""
try:
self.open()
@@ -421,7 +422,7 @@
elif scheme in ["http", "https"]:
return HTTPWalker(url, log_parent)
elif scheme in ["file"]:
- return os.walk(path)
+ return scandir.walk(path)
else:
raise WalkerError("Unknown scheme: %s" % scheme)
=== modified file 'lib/lp/scripts/utilities/js/combinecss.py'
--- lib/lp/scripts/utilities/js/combinecss.py 2017-10-20 12:01:51 +0000
+++ lib/lp/scripts/utilities/js/combinecss.py 2018-05-06 14:00:16 +0000
@@ -1,13 +1,14 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
import os
+import scandir
+
+from lp.scripts.utilities.js.combo import combine_files
from lp.scripts.utilities.js.jsbuild import ComboFile
-from lp.scripts.utilities.js.combo import combine_files
from lp.services.config import config
-
# It'd probably be nice to have this script find all the CSS files we might
# need and combine them together, but if we do that we'd certainly end up
# including lots of styles that we don't need/want, so keeping this hard-coded
@@ -60,7 +61,7 @@
# every time a new component is added.
component_dir = 'css/components'
component_path = os.path.abspath(os.path.join(icing, component_dir))
- for root, dirs, files in os.walk(component_path):
+ for root, dirs, files in scandir.walk(component_path):
for file in files:
if file.endswith('.css'):
names.append('%s/%s' % (component_dir, file))
=== modified file 'lib/lp/scripts/utilities/js/jsbuild.py'
--- lib/lp/scripts/utilities/js/jsbuild.py 2015-12-01 12:27:09 +0000
+++ lib/lp/scripts/utilities/js/jsbuild.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
+# Copyright 2011-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""build.py - Minifies and creates the JS build directory."""
@@ -17,6 +17,7 @@
import cssutils
from cssutils import settings
+import scandir
HERE = os.path.dirname(__file__)
@@ -276,8 +277,8 @@
return
# Process sub-skins.
- for skin in os.listdir(src_skins_dir):
- self.build_skin(component_name, skin)
+ for entry in scandir.scandir(src_skins_dir):
+ self.build_skin(component_name, entry.name)
def link_directory_content(self, src_dir, target_dir, link_filter=None):
"""Link all the files in src_dir into target_dir.
@@ -289,14 +290,13 @@
If the filter returns False, no symlink will be created. By
default a symlink is created for everything.
"""
- for name in os.listdir(src_dir):
- if name.endswith('~'):
- continue
- src = os.path.join(src_dir, name)
- if link_filter and not link_filter(src):
- continue
- target = os.path.join(target_dir, name)
- self.ensure_link(relative_path(target, src), target)
+ for entry in scandir.scandir(src_dir):
+ if entry.name.endswith('~'):
+ continue
+ if link_filter and not link_filter(entry.path):
+ continue
+ target = os.path.join(target_dir, entry.name)
+ self.ensure_link(relative_path(target, entry.path), target)
def build_skin(self, component_name, skin_name):
"""Build a skin for a particular component."""
@@ -350,11 +350,10 @@
combined_css.update()
def do_build(self):
- for name in os.listdir(self.src_dir):
- path = os.path.join(self.src_dir, name)
- if not os.path.isdir(path):
+ for entry in scandir.scandir(self.src_dir):
+ if not entry.is_dir():
continue
- self.build_assets(name)
+ self.build_assets(entry.name)
self.update_combined_css_skins()
=== modified file 'lib/lp/services/config/fixture.py'
--- lib/lp/services/config/fixture.py 2017-12-19 17:16:38 +0000
+++ lib/lp/services/config/fixture.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Fixtures related to configs.
@@ -18,6 +18,7 @@
from textwrap import dedent
from fixtures import Fixture
+import scandir
from lp.services.config import config
@@ -54,12 +55,12 @@
self.absroot = os.path.abspath(root)
self.addCleanup(shutil.rmtree, self.absroot)
source = os.path.join(config.root, 'configs', self.copy_from_instance)
- for basename in os.listdir(source):
- if basename == 'launchpad-lazr.conf':
+ for entry in scandir.scandir(source):
+ if entry.name == 'launchpad-lazr.conf':
self.add_section(self._extend_str % self.copy_from_instance)
continue
- with open(source + '/' + basename, 'rb') as input:
- with open(root + '/' + basename, 'wb') as out:
+ with open(entry.path, 'rb') as input:
+ with open(os.path.join(root, entry.name), 'wb') as out:
out.write(input.read())
=== modified file 'lib/lp/services/config/tests/test_config.py'
--- lib/lp/services/config/tests/test_config.py 2018-05-06 14:00:16 +0000
+++ lib/lp/services/config/tests/test_config.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
# We know we are not using root and handlers.
@@ -19,6 +19,7 @@
import importlib_resources
from lazr.config import ConfigSchema
from lazr.config.interfaces import ConfigErrors
+import scandir
import testtools
import ZConfig
@@ -150,7 +151,7 @@
load_testcase = unittest.defaultTestLoader.loadTestsFromTestCase
# Add a test for every launchpad[.lazr].conf file in our tree.
for config_dir in lp.services.config.CONFIG_ROOT_DIRS:
- for dirpath, dirnames, filenames in os.walk(config_dir):
+ for dirpath, dirnames, filenames in scandir.walk(config_dir):
if os.path.basename(dirpath) in EXCLUDED_CONFIGS:
del dirnames[:] # Don't look in subdirectories.
continue
=== modified file 'lib/lp/services/librarianserver/librariangc.py'
--- lib/lp/services/librarianserver/librariangc.py 2018-04-19 00:03:57 +0000
+++ lib/lp/services/librarianserver/librariangc.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Librarian garbage collection routines"""
@@ -18,6 +18,7 @@
import iso8601
import pytz
+import scandir
from swiftclient import client as swiftclient
from zope.interface import implementer
@@ -633,7 +634,7 @@
hex_content_id_re = re.compile('^([0-9a-f]{8})(\.migrated)?$')
ONE_DAY = 24 * 60 * 60
- for dirpath, dirnames, filenames in os.walk(
+ for dirpath, dirnames, filenames in scandir.walk(
get_storage_root(), followlinks=True):
# Ignore known and harmless noise in the Librarian storage area.
=== modified file 'lib/lp/services/librarianserver/swift.py'
--- lib/lp/services/librarianserver/swift.py 2018-01-03 17:17:12 +0000
+++ lib/lp/services/librarianserver/swift.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2013-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2013-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Move files from Librarian disk storage into Swift."""
@@ -21,6 +21,7 @@
import time
import urllib
+import scandir
from swiftclient import client as swiftclient
from lp.services.config import config
@@ -79,7 +80,8 @@
# Walk the Librarian on disk file store, searching for matching
# files that may need to be copied into Swift. We need to follow
# symlinks as they are being used span disk partitions.
- for dirpath, dirnames, filenames in os.walk(fs_root, followlinks=True):
+ for dirpath, dirnames, filenames in scandir.walk(
+ fs_root, followlinks=True):
# Don't recurse if we know this directory contains no matching
# files.
=== modified file 'lib/lp/services/mail/mailbox.py'
--- lib/lp/services/mail/mailbox.py 2015-09-21 06:04:04 +0000
+++ lib/lp/services/mail/mailbox.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -16,6 +16,7 @@
import socket
import threading
+import scandir
from zope.interface import (
implementer,
Interface,
@@ -168,10 +169,10 @@
def items(self):
"""See IMailBox."""
- for name in os.listdir(self.mail_dir):
- filename = os.path.join(self.mail_dir, name)
- if os.path.isfile(filename):
- yield (filename, open(filename).read())
+ for entry in scandir.scandir(self.mail_dir):
+ if entry.is_file():
+ with open(entry.path) as mail_file:
+ yield (entry.path, mail_file.read())
def delete(self, id):
"""See IMailBox."""
=== modified file 'lib/lp/services/mailman/runmailman.py'
--- lib/lp/services/mailman/runmailman.py 2015-10-14 15:22:01 +0000
+++ lib/lp/services/mailman/runmailman.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Start and stop the Mailman processes."""
@@ -16,6 +16,8 @@
import subprocess
import sys
+import scandir
+
import lp.services.config
from lp.services.mailman.config import configure_prefix
from lp.services.mailman.monkeypatches import monkey_patch
@@ -96,8 +98,8 @@
if error.errno != errno.ENOENT:
raise
lock_dir = os.path.join(mailman_path, 'locks')
- for filename in os.listdir(lock_dir):
- os.remove(os.path.join(lock_dir, filename))
+ for entry in scandir.scandir(lock_dir):
+ os.remove(entry.path)
def start_mailman(quiet=False, config=None):
=== modified file 'lib/lp/services/scripts/tests/__init__.py'
--- lib/lp/services/scripts/tests/__init__.py 2012-06-27 13:57:04 +0000
+++ lib/lp/services/scripts/tests/__init__.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -10,6 +10,8 @@
import os
import subprocess
+import scandir
+
import lp
from lp.services.config import config
@@ -32,7 +34,7 @@
scripts = []
for script_location in SCRIPT_LOCATIONS:
location = os.path.join(LP_TREE, script_location)
- for path, dirs, filenames in os.walk(location):
+ for path, dirs, filenames in scandir.walk(location):
for filename in filenames:
script_path = os.path.join(path, filename)
if (filename.startswith('_') or
=== modified file 'lib/lp/services/testing/__init__.py'
--- lib/lp/services/testing/__init__.py 2018-04-13 20:47:03 +0000
+++ lib/lp/services/testing/__init__.py 2018-05-06 14:00:16 +0000
@@ -21,6 +21,8 @@
import os
import unittest
+import scandir
+
# This import registers the 'doctest' Unicode codec.
import lp.services.testing.doctestcodec
from lp.testing.systemdocs import (
@@ -93,11 +95,10 @@
stories_path = os.path.join(base_dir, stories_dir)
if os.path.exists(stories_path):
suite.addTest(PageTestSuite(stories_dir, package))
- for story_dir in os.listdir(stories_path):
- full_story_dir = os.path.join(stories_path, story_dir)
- if not os.path.isdir(full_story_dir):
+ for story_entry in scandir.scandir(stories_path):
+ if not story_entry.is_dir():
continue
- story_path = os.path.join(stories_dir, story_dir)
+ story_path = os.path.join(stories_dir, story_entry.name)
if story_path in special_tests:
continue
suite.addTest(PageTestSuite(story_path, package))
=== modified file 'lib/lp/services/webservice/tests/test_wadllib.py'
--- lib/lp/services/webservice/tests/test_wadllib.py 2015-10-14 15:22:01 +0000
+++ lib/lp/services/webservice/tests/test_wadllib.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Run the standalone wadllib tests."""
@@ -10,6 +10,7 @@
import os
import unittest
+import scandir
import wadllib
from lp.testing.systemdocs import LayeredDocFileSuite
@@ -23,7 +24,7 @@
# Find all the doctests in wadllib.
packages = []
- for dirpath, dirnames, filenames in os.walk(topdir):
+ for dirpath, dirnames, filenames in scandir.walk(topdir):
if 'docs' in dirnames:
docsdir = os.path.join(dirpath, 'docs')[len(topdir) + 1:]
packages.append(docsdir)
=== modified file 'lib/lp/soyuz/doc/soyuz-set-of-uploads.txt'
--- lib/lp/soyuz/doc/soyuz-set-of-uploads.txt 2016-01-26 15:47:37 +0000
+++ lib/lp/soyuz/doc/soyuz-set-of-uploads.txt 2018-05-06 14:00:16 +0000
@@ -107,13 +107,14 @@
Firstly, we need a way to copy a test upload into the queue (but skip
lock files, which have names starting with a dot).
+ >>> import shutil
+ >>> import scandir
>>> from lp.archiveuploader.tests import datadir
>>> def punt_upload_into_queue(leaf, distro):
... inc_dir = os.path.join(incoming_dir, leaf, distro)
... os.makedirs(inc_dir)
- ... for file_leaf in os.listdir(datadir(os.path.join("suite", leaf))):
- ... os.system("cp %s %s" % (
- ... datadir(os.path.join("suite", leaf, file_leaf)), inc_dir))
+ ... for entry in scandir.scandir(datadir(os.path.join("suite", leaf))):
+ ... shutil.copy(entry.path, inc_dir)
We need a way to count the items in a queue directory
@@ -227,7 +228,6 @@
We'll be doing a lot of uploads with sanity checks, and expect them to
succeed. A helper function, simulate_upload does that with all the checking.
- >>> import shutil
>>> from lp.services.mail import stub
>>> def simulate_upload(
=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt'
--- lib/lp/soyuz/doc/soyuz-upload.txt 2017-08-03 14:26:40 +0000
+++ lib/lp/soyuz/doc/soyuz-upload.txt 2018-05-06 14:00:16 +0000
@@ -99,11 +99,12 @@
>>> def get_md5(filename):
... return hashlib.md5(open(filename).read()).digest()
+ >>> import scandir
>>> def get_upload_dir(num, dir=incoming_dir):
... """Return the path to the upload, if found in the dir."""
- ... for upload_dir in os.listdir(dir):
- ... if upload_dir.endswith("%06d" % num):
- ... return os.path.join(dir, upload_dir)
+ ... for upload_entry in scandir.scandir(dir):
+ ... if upload_entry.name.endswith("%06d" % num):
+ ... return upload_entry.path
... return None
>>> def find_upload_dir(num):
@@ -183,7 +184,6 @@
... IPersonSet,
... PersonCreationRationale,
... )
- >>> from lp.services.gpg.interfaces import GPGKeyAlgorithm
>>> name, address = "Katie", "katie@xxxxxxxxxxxxxxxxxxxxx"
>>> user = getUtility(IPersonSet).ensurePerson(
... address, name, PersonCreationRationale.OWNER_CREATED_LAUNCHPAD)
=== modified file 'lib/lp/soyuz/scripts/gina/packages.py'
--- lib/lp/soyuz/scripts/gina/packages.py 2016-03-18 14:12:07 +0000
+++ lib/lp/soyuz/scripts/gina/packages.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Package information classes.
@@ -28,6 +28,8 @@
import shutil
import tempfile
+import scandir
+
from lp.app.validators.version import valid_debian_version
from lp.archivepublisher.diskpool import poolify
from lp.archiveuploader.changesfile import ChangesFile
@@ -88,13 +90,13 @@
return filename, fullpath, component
# Do a second pass, scrubbing through all components in the pool.
- for alt_component in os.listdir(pool_root):
- if not os.path.isdir(os.path.join(pool_root, alt_component)):
+ for alt_component_entry in scandir.scandir(pool_root):
+ if not alt_component_entry.is_dir():
continue
- pool_dir = poolify(name, alt_component)
+ pool_dir = poolify(name, alt_component_entry.name)
fullpath = os.path.join(pool_root, pool_dir, filename)
if os.path.exists(fullpath):
- return filename, fullpath, alt_component
+ return filename, fullpath, alt_component_entry.name
# Couldn't find the file anywhere -- too bad.
raise PoolFileNotFound("File %s not in archive" % filename)
=== modified file 'lib/lp/soyuz/tests/fakepackager.py'
--- lib/lp/soyuz/tests/fakepackager.py 2018-02-02 03:14:35 +0000
+++ lib/lp/soyuz/tests/fakepackager.py 2018-05-06 14:00:16 +0000
@@ -20,6 +20,7 @@
import tempfile
import time
+import scandir
from zope.component import getUtility
from lp.archiveuploader.nascentupload import NascentUpload
@@ -378,9 +379,8 @@
def listAvailableUploads(self):
"""Return the path for all available changesfiles."""
- changes = [os.path.join(self.sandbox_path, filename)
- for filename in os.listdir(self.sandbox_path)
- if filename.endswith('.changes')]
+ changes = [entry.path for entry in scandir.scandir(self.sandbox_path)
+ if entry.name.endswith('.changes')]
return sorted(changes)
=== modified file 'lib/lp/soyuz/tests/test_doc.py'
--- lib/lp/soyuz/tests/test_doc.py 2018-02-02 03:14:35 +0000
+++ lib/lp/soyuz/tests/test_doc.py 2018-05-06 14:00:16 +0000
@@ -11,6 +11,7 @@
import os
import unittest
+import scandir
import transaction
from lp.services.config import config
@@ -165,11 +166,10 @@
stories_dir = os.path.join(os.path.pardir, 'stories')
suite.addTest(PageTestSuite(stories_dir))
stories_path = os.path.join(here, stories_dir)
- for story_dir in os.listdir(stories_path):
- full_story_dir = os.path.join(stories_path, story_dir)
- if not os.path.isdir(full_story_dir):
+ for story_entry in scandir.scandir(stories_path):
+ if not story_entry.is_dir():
continue
- story_path = os.path.join(stories_dir, story_dir)
+ story_path = os.path.join(stories_dir, story_entry.name)
suite.addTest(PageTestSuite(story_path))
# Add special needs tests
=== modified file 'lib/lp/soyuz/tests/test_publishing.py'
--- lib/lp/soyuz/tests/test_publishing.py 2018-02-02 03:14:35 +0000
+++ lib/lp/soyuz/tests/test_publishing.py 2018-05-06 14:00:16 +0000
@@ -13,6 +13,7 @@
import tempfile
import pytz
+import scandir
from storm.store import Store
from testtools.matchers import Equals
import transaction
@@ -494,7 +495,7 @@
def _findChangesFile(self, top, name_fragment):
"""File with given name fragment in directory tree starting at top."""
- for root, dirs, files in os.walk(top, topdown=False):
+ for root, dirs, files in scandir.walk(top, topdown=False):
for name in files:
if (name.endswith('.changes') and
name.find(name_fragment) > -1):
=== modified file 'lib/lp/testing/__init__.py'
--- lib/lp/testing/__init__.py 2017-11-10 02:13:28 +0000
+++ lib/lp/testing/__init__.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
from __future__ import absolute_import
@@ -93,6 +93,7 @@
import lp_sitecustomize
import oops_datedir_repo.serializer_rfc822
import pytz
+import scandir
import simplejson
from storm.store import Store
import subunit
@@ -1172,7 +1173,7 @@
def _harvest_yui_test_files(file_path):
- for dirpath, dirnames, filenames in os.walk(file_path):
+ for dirpath, dirnames, filenames in scandir.walk(file_path):
for filename in filenames:
if fnmatchcase(filename, "test_*.html"):
yield os.path.join(dirpath, filename)
=== modified file 'lib/lp/testing/gpgkeys/__init__.py'
--- lib/lp/testing/gpgkeys/__init__.py 2017-07-31 11:19:23 +0000
+++ lib/lp/testing/gpgkeys/__init__.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""OpenPGP keys used for testing.
@@ -23,14 +23,12 @@
import os
import gpgme
+import scandir
from zope.component import getUtility
from lp.registry.interfaces.gpg import IGPGKeySet
from lp.registry.interfaces.person import IPersonSet
-from lp.services.gpg.interfaces import (
- GPGKeyAlgorithm,
- IGPGHandler,
- )
+from lp.services.gpg.interfaces import IGPGHandler
gpgkeysdir = os.path.join(os.path.dirname(__file__), 'data')
@@ -109,9 +107,9 @@
def test_keyrings():
"""Iterate over the filenames for test keyrings."""
- for name in os.listdir(gpgkeysdir):
- if name.endswith('.gpg'):
- yield os.path.join(gpgkeysdir, name)
+ for entry in scandir.scandir(gpgkeysdir):
+ if entry.name.endswith('.gpg'):
+ yield entry.path
def decrypt_content(content, password):
=== modified file 'lib/lp/testing/yuixhr.py'
--- lib/lp/testing/yuixhr.py 2017-07-20 17:32:10 +0000
+++ lib/lp/testing/yuixhr.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2011-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2011-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Fixture code for YUITest + XHR integration testing."""
@@ -20,6 +20,7 @@
from lazr.restful import ResourceJSONEncoder
from lazr.restful.utils import get_current_browser_request
+import scandir
import simplejson
from zope.component import getUtility
from zope.exceptions.exceptionformatter import format_exception
@@ -451,7 +452,7 @@
def find_tests(root):
- for dirpath, dirnames, filenames in os.walk(root):
+ for dirpath, dirnames, filenames in scandir.walk(root):
dirpath = os.path.relpath(dirpath, root)
for filename in filenames:
if fnmatchcase(filename, 'test_*.js'):
=== modified file 'lib/lp/testopenid/stories/tests.py'
--- lib/lp/testopenid/stories/tests.py 2013-03-20 03:41:40 +0000
+++ lib/lp/testopenid/stories/tests.py 2018-05-06 14:00:16 +0000
@@ -1,9 +1,11 @@
-# Copyright 2004-2008 Canonical Ltd. This software is licensed under the
+# Copyright 2004-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
import os
import unittest
+import scandir
+
from lp.testing.pages import PageTestSuite
@@ -12,8 +14,8 @@
def test_suite():
stories = sorted(
- dir for dir in os.listdir(here)
- if not dir.startswith('.') and os.path.isdir(os.path.join(here, dir)))
+ entry.name for entry in scandir.scandir(here)
+ if not entry.name.startswith('.') and entry.is_dir())
suite = unittest.TestSuite()
suite.addTest(PageTestSuite('.'))
=== modified file 'lib/lp/tests/test_opensource.py'
--- lib/lp/tests/test_opensource.py 2015-10-26 14:54:43 +0000
+++ lib/lp/tests/test_opensource.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Run the standalone tests in an opensource package.
@@ -21,6 +21,7 @@
import unittest
import launchpadlib
+import scandir
import wadllib
from lp.testing.layers import AppServerLayer
@@ -32,7 +33,7 @@
topdir = os.path.dirname(package.__file__)
packages = []
- for dirpath, dirnames, filenames in os.walk(topdir):
+ for dirpath, dirnames, filenames in scandir.walk(topdir):
if 'docs' in dirnames:
docsdir = os.path.join(dirpath, 'docs')[len(topdir) + 1:]
packages.append(docsdir)
=== modified file 'lib/lp/translations/tests/test_doc.py'
--- lib/lp/translations/tests/test_doc.py 2011-12-28 17:03:06 +0000
+++ lib/lp/translations/tests/test_doc.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""
@@ -9,6 +9,8 @@
import os
import unittest
+import scandir
+
from lp.testing.layers import (
LaunchpadFunctionalLayer,
LaunchpadZopelessLayer,
@@ -54,11 +56,10 @@
stories_dir = os.path.join(os.path.pardir, 'stories')
suite.addTest(PageTestSuite(stories_dir))
stories_path = os.path.join(here, stories_dir)
- for story_dir in os.listdir(stories_path):
- full_story_dir = os.path.join(stories_path, story_dir)
- if not os.path.isdir(full_story_dir):
+ for story_entry in scandir.scandir(stories_path):
+ if not story_entry.is_dir():
continue
- story_path = os.path.join(stories_dir, story_dir)
+ story_path = os.path.join(stories_dir, story_entry.name)
suite.addTest(PageTestSuite(story_path))
testsdir = os.path.abspath(
=== modified file 'lib/lp/translations/utilities/tests/xpi_helpers.py'
--- lib/lp/translations/utilities/tests/xpi_helpers.py 2010-08-20 20:31:18 +0000
+++ lib/lp/translations/utilities/tests/xpi_helpers.py 2018-05-06 14:00:16 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Helper methods for XPI testing"""
@@ -15,6 +15,8 @@
from textwrap import dedent
import zipfile
+import scandir
+
import lp.translations
@@ -55,7 +57,7 @@
jar = zipfile.ZipFile(jarfile, 'w')
jarlist = []
data_dir = os.path.join(test_root, 'en-US-jar/')
- for root, dirs, files in os.walk(data_dir):
+ for root, dirs, files in scandir.walk(data_dir):
for name in files:
relative_dir = root[len(data_dir):].strip('/')
jarlist.append(os.path.join(relative_dir, name))
@@ -69,11 +71,10 @@
xpifile = tempfile.TemporaryFile()
xpi = zipfile.ZipFile(xpifile, 'w')
- xpilist = os.listdir(test_root)
- xpilist.remove('en-US-jar')
- for file_name in xpilist:
- f = open(os.path.join(test_root, file_name), 'r')
- xpi.writestr(file_name, f.read())
+ for xpi_entry in scandir.scandir(test_root):
+ if xpi_entry.name != 'en-US-jar':
+ with open(xpi_entry.path) as f:
+ xpi.writestr(xpi_entry.name, f.read())
xpi.writestr('chrome/en-US.jar', jarfile.read())
xpi.close()
xpifile.seek(0)
=== modified file 'scripts/migrate-librarian-content-md5.py'
--- scripts/migrate-librarian-content-md5.py 2012-01-01 03:13:08 +0000
+++ scripts/migrate-librarian-content-md5.py 2018-05-06 14:00:16 +0000
@@ -1,15 +1,20 @@
-#!/usr/bin/env python
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+#!/usr/bin/python -S
+#
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Script to generate SQL to add MD5 sums for existing librarian files."""
__metaclass__ = type
+import _pythonpath
+
import commands
import os
import sys
+import scandir
+
SQL = "UPDATE LibraryFileContent SET md5 = '%s' WHERE id = %d;"
@@ -18,10 +23,10 @@
if not path.endswith('/'):
path += '/'
- for dirpath, dirname, filenames in os.walk(path):
+ for dirpath, dirname, filenames in scandir.walk(path):
dirname.sort()
databaseID = dirpath[len(path):]
- if not len(databaseID) == 8: # "xx/xx/xx"
+ if not len(databaseID) == 8: # "xx/xx/xx"
continue
for filename in filenames:
databaseID = int(databaseID.replace('/', '') + filename, 16)
@@ -31,6 +36,7 @@
md5sum = commands.getoutput('md5sum ' + filename).split(' ', 1)[0]
yield databaseID, md5sum
+
if __name__ == '__main__':
if len(sys.argv) > 2:
minimumID = int(sys.argv[2])
=== modified file 'setup.py'
--- setup.py 2018-05-06 14:00:16 +0000
+++ setup.py 2018-05-06 14:00:16 +0000
@@ -209,6 +209,7 @@
'rabbitfixture',
'requests',
'requests-toolbelt',
+ 'scandir',
'setproctitle',
'setuptools',
'six',
=== modified file 'utilities/findimports.py'
--- utilities/findimports.py 2012-06-29 08:40:05 +0000
+++ utilities/findimports.py 2018-05-06 14:00:16 +0000
@@ -1,6 +1,6 @@
#!/usr/bin/python
#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""
@@ -44,6 +44,8 @@
import os
import sys
+import scandir
+
class ImportFinder(ASTVisitor):
"""AST visitor that collects all imported names in its imports attribute.
@@ -167,7 +169,7 @@
def parsePathname(self, pathname):
if os.path.isdir(pathname):
- for root, dirs, files in os.walk(pathname):
+ for root, dirs, files in scandir.walk(pathname):
for fn in files:
# ignore emacsish junk
if fn.endswith('.py') and not fn.startswith('.#'):
=== modified file 'utilities/format-imports'
--- utilities/format-imports 2012-09-28 06:15:58 +0000
+++ utilities/format-imports 2018-05-06 14:00:16 +0000
@@ -1,6 +1,6 @@
#!/usr/bin/python
#
-# Copyright 2010-2012 Canonical Ltd. This software is licensed under the
+# Copyright 2010-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
""" Format import sections in python files
@@ -133,6 +133,8 @@
import sys
from textwrap import dedent
+import scandir
+
sys.path[0:0] = [os.path.dirname(__file__)]
from python_standard_libs import python_standard_libs
@@ -385,7 +387,7 @@
def process_tree(dpath):
"""Walk a directory tree and process all *.py files."""
- for dirpath, dirnames, filenames in os.walk(dpath):
+ for dirpath, dirnames, filenames in scandir.walk(dpath):
for filename in filenames:
if filename.endswith('.py'):
process_file(os.path.join(dirpath, filename))
Follow ups
