duplicity-team team mailing list archive
-
duplicity-team team
-
Mailing list archive
-
Message #04107
[Merge] lp:~aaron-whitehouse/duplicity/08-refactor-unit-test-globmatch into lp:duplicity
Aaron Whitehouse has proposed merging lp:~aaron-whitehouse/duplicity/08-refactor-unit-test-globmatch into lp:duplicity.
Requested reviews:
duplicity-team (duplicity-team)
For more details, see:
https://code.launchpad.net/~aaron-whitehouse/duplicity/08-refactor-unit-test-globmatch/+merge/316397
* Rename path_matches_glob_fn to select_fn_from_glob, as this more accurately reflects the return value.
* Significantly refactored unit/test_globmatch.py to make this cleaner and clearer.
--
Your team duplicity-team is requested to review the proposed merge of lp:~aaron-whitehouse/duplicity/08-refactor-unit-test-globmatch into lp:duplicity.
=== modified file 'duplicity/globmatch.py'
--- duplicity/globmatch.py 2017-01-29 18:11:52 +0000
+++ duplicity/globmatch.py 2017-02-05 01:16:14 +0000
@@ -49,7 +49,7 @@
return list(map(glob_to_regex, prefixes))
-def path_matches_glob_fn(glob_str, include, ignore_case=False):
+def select_fn_from_glob(glob_str, include, ignore_case=False):
"""Return a function test_fn(path) which
tests whether path matches glob, as per the Unix shell rules, taking as
arguments a path, a glob string and include (0 indicating that the glob
@@ -143,7 +143,7 @@
"""Returned regular expression equivalent to shell glob pat
Currently only the ?, *, [], and ** expressions are supported.
- Ranges like [a-z] are also currently unsupported. There is no
+ Ranges like [a-z] are currently unsupported. There is no
way to quote these special characters.
This function taken with minor modifications from efnmatch.py
=== modified file 'duplicity/selection.py'
--- duplicity/selection.py 2017-01-29 19:03:45 +0000
+++ duplicity/selection.py 2017-02-05 01:16:14 +0000
@@ -33,7 +33,7 @@
from duplicity import diffdir
from duplicity import util # @Reimport
from duplicity.globmatch import GlobbingError, FilePrefixError, \
- path_matches_glob_fn
+ select_fn_from_glob
"""Iterate exactly the requested files in a directory
@@ -501,13 +501,13 @@
# Check to make sure prefix is ok, i.e. the glob string is within
# the root folder being backed up
- file_prefix_selection = path_matches_glob_fn(glob_str, include=1)(self.rootpath)
+ file_prefix_selection = select_fn_from_glob(glob_str, include=1)(self.rootpath)
if not file_prefix_selection:
# file_prefix_selection == 1 (include) or 2 (scan)
raise FilePrefixError(glob_str + " glob with " + self.rootpath.name +
" path gives " + str(file_prefix_selection))
- return path_matches_glob_fn(glob_str, include, ignore_case)
+ return select_fn_from_glob(glob_str, include, ignore_case)
def exclude_older_get_sf(self, date):
"""Return selection function based on files older than modification date """
=== modified file 'testing/unit/test_globmatch.py'
--- testing/unit/test_globmatch.py 2017-01-29 18:11:52 +0000
+++ testing/unit/test_globmatch.py 2017-02-05 01:16:14 +0000
@@ -20,75 +20,93 @@
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-import sys
from duplicity.globmatch import *
from duplicity.path import *
from . import UnitTestCase
from mock import patch
-import unittest
-
-
-class MatchingTest(UnitTestCase):
- """Test matching of file names against various selection functions"""
-
- def test_glob_re(self):
+
+
+def sel_file(glob_str, include, file_path):
+ """Returns the selection value for file_path, given the include value,
+ returning:
+ 0 - if the file should be excluded
+ 1 - if the file should be included
+ 2 - if the folder should be scanned for any included/excluded files
+ None - if the selection function has nothing to say about the file
+
+ Note: including a folder implicitly includes everything within it."""
+
+ select_fn = select_fn_from_glob(glob_str, include)
+ selection_value = select_fn(Path(file_path))
+ return selection_value
+
+
+def inc_sel_file(glob_str, file_path):
+ """Returns result of sel_file with include value set to 1"""
+ # Aids readability of the testing code to only have one number (the
+ # result of the select function)
+ return sel_file(glob_str, 1, file_path)
+
+
+def exc_sel_file(glob_str, file_path):
+ """Returns result of sel_file with include value set to 0"""
+ return sel_file(glob_str, 0, file_path)
+
+
+def sel_dir(glob_str, include, file_path):
+ """As per sel_file, but mocks file_path to be a directory"""
+ with patch('duplicity.path.Path.isdir') as mock_isdir:
+ mock_isdir.return_value = True
+ return sel_file(glob_str, include, file_path)
+
+
+def inc_sel_dir(glob_str, file_path):
+ """Returns result of sel_dir with include value set to 1"""
+ return sel_dir(glob_str, 1, file_path)
+
+
+def exc_sel_dir(glob_str, file_path):
+ """Returns result of sel_dir with include value set to 0"""
+ return sel_dir(glob_str, 0, file_path)
+
+
+class TestGlobToRegex(UnitTestCase):
+ """Test translation of glob strings into regular expressions"""
+
+ def test_glob_to_regex(self):
"""test_glob_re - test translation of shell pattern to regular exp"""
- assert glob_to_regex("hello") == "hello"
- assert glob_to_regex(".e?ll**o") == "\\.e[^/]ll.*o"
- r = glob_to_regex("[abc]el[^de][!fg]h")
- assert r == "[abc]el[^de][^fg]h", r
- r = glob_to_regex("/usr/*/bin/")
- assert r == "\\/usr\\/[^/]*\\/bin\\/", r
- assert glob_to_regex("[a.b/c]") == "[a.b/c]"
- r = glob_to_regex("[a*b-c]e[!]]")
- assert r == "[a*b-c]e[^]]", r
-
-
-class TestMiscellaneousGlobMatching(UnitTestCase):
- """Test various glob matching"""
+ self.assertEqual(glob_to_regex("hello"), "hello")
+ self.assertEqual(glob_to_regex(".e?ll**o"), "\\.e[^/]ll.*o")
+ self.assertEqual(glob_to_regex("[abc]el[^de][!fg]h"),
+ "[abc]el[^de][^fg]h")
+ self.assertEqual(glob_to_regex("/usr/*/bin/"),
+ "\\/usr\\/[^/]*\\/bin\\/")
+ self.assertEqual(glob_to_regex("[a.b/c]"), "[a.b/c]")
+ self.assertEqual(glob_to_regex("[a*b-c]e[!]]"), "[a*b-c]e[^]]")
+
+
+class TestSelectValuesFromGlobs(UnitTestCase):
+ """Test the select values returned from various globs"""
def test_glob_scans_parent_directories(self):
"""Test glob scans parent"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- # Note: paths ending in '/' do not work!
- # self.assertEqual(
- # path_matches_glob_fn("testfiles/parent/sub/", 1)(Path(
- # "testfiles/parent/")), 2)
- self.assertEqual(
- path_matches_glob_fn("testfiles/parent/sub", 1)(Path(
- "testfiles/parent")), 2)
- self.assertEqual(
- path_matches_glob_fn("testfiles/select2/3/3sub2", 1)(Path(
- "testfiles/select2/3")), 2)
-
-
-class TestDoubleAsteriskOnIncludesExcludes(UnitTestCase):
- """Test ** on includes and exclude patterns"""
+ self.assertEqual(
+ inc_sel_dir("testfiles/parent/sub", "testfiles/parent"), 2)
+ self.assertEqual(
+ inc_sel_dir("testfiles/select2/3/3sub2", "testfiles/select2/3"), 2)
def test_double_asterisk_include(self):
"""Test a few globbing patterns, including **"""
- self.assertEqual(
- path_matches_glob_fn("**", 1)(Path("foo.txt")), 1)
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("**", 1)(Path("folder")), 1)
+ self.assertEqual(inc_sel_file("**", "foo.txt"), 1)
+ self.assertEqual(inc_sel_dir("**", "folder"), 1)
def test_double_asterisk_extension_include(self):
"""Test **.py"""
- self.assertEqual(
- path_matches_glob_fn("**.py", 1)(Path("what/ever.py")), 1)
- self.assertEqual(
- path_matches_glob_fn("**.py", 1)(Path("what/ever.py/foo")), 1)
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("**.py", 1)(Path("foo")), 2)
- self.assertEqual(
- path_matches_glob_fn("**.py", 1)(Path("usr/local/bin")), 2)
- self.assertEqual(
- path_matches_glob_fn("**.py", 1)(Path("/usr/local/bin")), 2)
+ self.assertEqual(inc_sel_file("**.py", "what/ever.py"), 1)
+ self.assertEqual(inc_sel_file("**.py", "what/ever.py/foo"), 1)
+ self.assertEqual(inc_sel_dir("**.py", "foo"), 2)
+ self.assertEqual(inc_sel_dir("**.py", "usr/local/bin"), 2)
+ self.assertEqual(inc_sel_dir("**.py", "/usr/local/bin"), 2)
class TestTrailingSlash(UnitTestCase):
@@ -96,175 +114,116 @@
def test_trailing_slash_matches_only_dirs(self):
"""Test matching where glob includes a trailing slash"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("fold*/", 1)(Path("folder")), 1)
-
- # Test the file named "folder" is not included if it is not a dir
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("fold*/", 1)(Path("folder")), None)
-
- # Test the file named "folder" is not included if it is not a dir
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("folder/", 1)(Path("folder")), None)
-
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("fo*/", 1)(Path("foo.txt")), None)
+ # Test the folder named "folder" is included
+ self.assertEqual(inc_sel_dir("fold*/", "folder"), 1)
+
+ # Test the file (not folder) named "folder" is not included
+ self.assertEqual(inc_sel_file("fold*/", "folder"), None)
+ self.assertEqual(inc_sel_file("folder/", "folder"), None)
+
+ # Test miscellaneous file/folder
+ self.assertEqual(inc_sel_file("fo*/", "foo.txt"), None)
def test_included_files_are_matched_no_slash(self):
"""Test that files within an included folder are matched"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("fold*", 1)(Path("folder/file.txt")), 1)
-
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("fold*", 1)(Path("folder/2/file.txt")), 1)
+ self.assertEqual(inc_sel_file("fold*", "folder/file.txt"), 1)
+ self.assertEqual(inc_sel_file("fold*", "folder/file.txt"), 1)
+ self.assertEqual(inc_sel_file("fold*", "folder/2/file.txt"), 1)
def test_included_files_are_matched_no_slash_2(self):
"""Test that files within an included folder are matched"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("folder", 1)(Path("folder/file.txt")), 1)
-
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("folder/2", 1)(Path("folder/2/file.txt")), 1)
+ self.assertEqual(inc_sel_file("folder", "folder/file.txt"), 1)
+ self.assertEqual(inc_sel_file("folder/2", "folder/2/file.txt"), 1)
def test_included_files_are_matched_slash(self):
"""Test that files within an included folder are matched with /"""
# Bug #1624725
# https://bugs.launchpad.net/duplicity/+bug/1624725
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("folder/", 1)(Path("folder/file.txt")), 1)
+ self.assertEqual(inc_sel_file("folder/", "folder/file.txt"), 1)
def test_included_files_are_matched_slash_2(self):
"""Test that files within an included folder are matched with /"""
# Bug #1624725
# https://bugs.launchpad.net/duplicity/+bug/1624725
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("testfiles/select2/1/1sub1/1sub1sub1/", 1)
- (Path("testfiles/select2/1/1sub1/1sub1sub1/1sub1sub1_file.txt")
- ), 1)
+ self.assertEqual(inc_sel_file(
+ "testfiles/select2/1/1sub1/1sub1sub1/",
+ "testfiles/select2/1/1sub1/1sub1sub1/1sub1sub1_file.txt"), 1)
def test_included_files_are_matched_slash_2_parents(self):
"""Test that duplicity will scan parent of glob/"""
# Bug #1624725
# https://bugs.launchpad.net/duplicity/+bug/1624725
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("testfiles/select2/1/1sub1/1sub1sub1/", 1)
- (Path("testfiles/select2/1/1sub1/1sub1sub1")
- ), 1)
- self.assertEqual(
- path_matches_glob_fn("testfiles/select2/1/1sub1/1sub1sub1/", 1)
- (Path("testfiles/select2/1/1sub1")
- ), 2)
+ self.assertEqual(inc_sel_dir(
+ "testfiles/select2/1/1sub1/1sub1sub1/",
+ "testfiles/select2/1/1sub1/1sub1sub1"), 1)
+ self.assertEqual(inc_sel_dir(
+ "testfiles/select2/1/1sub1/1sub1sub1/",
+ "testfiles/select2/1/1sub1"), 2)
def test_included_files_are_matched_slash_wildcard(self):
"""Test that files within an included folder are matched with /"""
# Bug #1624725
# https://bugs.launchpad.net/duplicity/+bug/1624725
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = False
- self.assertEqual(
- path_matches_glob_fn("fold*/", 1)(Path("folder/file.txt")), 1)
+ self.assertEqual(inc_sel_file("fold*/", "folder/file.txt"), 1)
def test_slash_matches_everything(self):
"""Test / matches everything"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("/",
- 1)(Path("/tmp/testfiles/select/1/2")), 1)
- self.assertEqual(
- path_matches_glob_fn("/",
- 1)(Path("/test/random/path")), 1)
- self.assertEqual(
- path_matches_glob_fn("/",
- 1)(Path("/")), 1)
- self.assertEqual(
- path_matches_glob_fn("/",
- 1)(Path("/var/log")), 1)
- self.assertEqual(
- path_matches_glob_fn("/",
- 1)(Path("/var/log/log.txt")), 1)
+ self.assertEqual(inc_sel_dir("/", "/tmp/testfiles/select/1/2"), 1)
+ self.assertEqual(inc_sel_dir("/", "/test/random/path"), 1)
+ self.assertEqual(exc_sel_dir("/", "/test/random/path"), 0)
+ self.assertEqual(inc_sel_dir("/", "/"), 1)
+ self.assertEqual(inc_sel_dir("/", "/var/log"), 1)
+ self.assertEqual(inc_sel_file("/", "/var/log/log.txt"), 1)
def test_slash_star_scans_folder(self):
"""Test that folder/* scans folder/"""
# This behaviour is a bit ambiguous - either include or scan could be
# argued as most appropriate here, but only an empty folder is at stake
# so long as test_slash_star_includes_folder_contents passes.
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("folder/*", 1)(Path("folder")), 2)
+ self.assertEqual(inc_sel_dir("folder/*", "folder"), 2)
def test_slash_star_includes_folder_contents(self):
"""Test that folder/* includes folder contents"""
- self.assertEqual(path_matches_glob_fn("folder/*", 1)
- (Path("folder/file.txt")), 1)
- self.assertEqual(path_matches_glob_fn("folder/*", 1)
- (Path("folder/other_file.log")), 1)
+ self.assertEqual(inc_sel_file("folder/*", "folder/file.txt"), 1)
+ self.assertEqual(inc_sel_file("folder/*", "folder/other_file.log"), 1)
- def test_slash_star_star_includes_folder(self):
- """Test that folder/** includes folder/"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
+ def test_slash_star_star_scans_folder(self):
+ """Test that folder/** scans folder/"""
+ self.assertEqual(inc_sel_dir("folder/**", "folder"), 2)
def test_simple_trailing_slash_match(self):
"""Test that a normal folder string ending in / matches that path"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- test_path = "testfiles/select/1/2/1"
- self.assertEqual(
- path_matches_glob_fn(test_path, 1)(Path(test_path)), 1)
+ self.assertEqual(inc_sel_dir("testfiles/select/1/2/1/",
+ "testfiles/select/1/2/1"), 1)
def test_double_asterisk_string_slash(self):
"""Test string starting with ** and ending in /"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("**/1/2/",
- 1)(Path("testfiles/select/1/2")), 1)
+ self.assertEqual(inc_sel_dir("**/1/2/", "testfiles/select/1/2"), 1)
def test_string_double_asterisk_string_slash(self):
"""Test string ** string /"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("testfiles**/2/",
- 1)(Path("testfiles/select/1/2")), 1)
+ self.assertEqual(inc_sel_dir("testfiles**/2/",
+ "testfiles/select/1/2"), 1)
class TestDoubleAsterisk(UnitTestCase):
"""Test glob matching where the glob finishes with a **"""
- def test_simple_trailing_slash_match(self):
+ def test_double_asterisk_no_match(self):
+ """Test that a folder string ending /** does not match other paths"""
+ self.assertEqual(inc_sel_dir("/test/folder/**", "/test/foo"), None)
+
+ def test_double_asterisk_match(self):
"""Test that a folder string ending in /** matches that path"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("/test/folder/**", 1)(
- Path("/test/foo")), None)
+ self.assertEqual(inc_sel_dir("/test/folder/**",
+ "/test/folder/foo"), 1)
+ self.assertEqual(inc_sel_file("/test/folder/**",
+ "/test/folder/foo.txt"), 1)
+ self.assertEqual(inc_sel_dir("/test/folder/**",
+ "/test/folder/2/foo"), 1)
+ self.assertEqual(inc_sel_file("/test/folder/**",
+ "/test/folder/2/foo.txt"), 1)
- def test_simple_trailing_slash_match_2(self):
+ def test_asterisk_slash_double_asterisk(self):
"""Test folder string ending in */**"""
- with patch('duplicity.path.Path.isdir') as mock_isdir:
- mock_isdir.return_value = True
- self.assertEqual(
- path_matches_glob_fn("fold*/**", 1)(
- Path("folder")), 2)
\ No newline at end of file
+ self.assertEqual(inc_sel_dir("fold*/**", "folder"), 2)
Follow ups
