arsenal-devel team mailing list archive

[Kamran Riaz Khan <krkhan@xxxxxxxxxxxxxx>]

Kamran Riaz Khan has proposed merging lp:~inspirated/arsenal/send-attachments-enforce-mimetype into lp:arsenal.

Requested reviews:
  arsenal-devel (arsenal-devel)


The following options are implemented in this branch:

  -x, --extract         Extract tar archives
  -g, --gzip            Gzip files which exceed maximum attachment size
  -f, --force-mime      Extract tar archives
  -e EXCLUDE, --exclude=EXCLUDE
                        Wildcard pattern for attachment exclusion
  -l XLIMIT, --extract-limit=XLIMIT
                        Limit on number of archive members for extraction
  -m MAXSIZE, --max-size=MAXSIZE
                        Maximum filesize for processing attachments

-- 
https://code.launchpad.net/~inspirated/arsenal/send-attachments-enforce-mimetype/+merge/30329
Your team arsenal-devel is requested to review the proposed merge of lp:~inspirated/arsenal/send-attachments-enforce-mimetype into lp:arsenal.

=== modified file 'scripts/send-attachments-upstream.py'
--- scripts/send-attachments-upstream.py	2010-07-13 20:41:41 +0000
+++ scripts/send-attachments-upstream.py	2010-07-19 19:33:46 +0000
@@ -1,19 +1,23 @@
 #!/usr/bin/env python
 
 import cgi
+import gzip
 import os
 import pdb
 import re
 import shutil
 import sys
+import tarfile
 import tempfile
 
 from ConfigParser import ConfigParser
 from StringIO import StringIO
+from fnmatch import fnmatch
 from getpass import getpass
 from locale import getpreferredencoding
 from optparse import OptionParser
 from urlparse import urlparse
+from zipfile import ZipFile
 
 import pycurl
 import gnomekeyring as gkeyring
@@ -61,36 +65,58 @@
 class BugzillaLoginError(Exception):
     pass
 
-class Flags(type):
+class FlagsAndModifiers(type):
     def __init__(cls, name, bases, dct):
-        super(Flags, cls).__init__(name, bases, dict)
+        super(FlagsAndModifiers, cls).__init__(name, bases, dict)
 
         for index, flagname in enumerate(cls.flagnames):
             setattr(cls, flagname, 1 << index)
 
+        for modifier in cls.modifiers:
+            setattr(cls, modifier, None)
+
         def parse_flags(self, flags):
             for flagname in self.flagnames:
                 flag = getattr(self, flagname)
                 if not flag & flags:
                     setattr(self, flagname, 0)
 
+        def parse_modifiers(self, modifiers):
+            for modifier, value in modifiers.iteritems():
+                setattr(self, modifier, value)
+
         setattr(cls, 'parse_flags', parse_flags)
-
+        setattr(cls, 'parse_modifiers', parse_modifiers)
 
 class SendAttachments(object):
-    __metaclass__ = Flags
+    __metaclass__ = FlagsAndModifiers
 
     flagnames = ('DEBUG',
-                 'OWNERONLY')
-
-    def __init__(self, lpurl, bzurl, bzuser, bzpassword, flags):
+                 'OWNERONLY',
+                 'GZIP',
+                 'EXTRACT',
+                 'FORCEMIME')
+
+    modifiers = ('bzuser',
+                 'bzpassword',
+                 'exclude',
+                 'xlimit',
+                 'maxsize')
+
+    mimetypes = {
+            '*.txt' : 'text/plain',
+            '*.log' : 'text/plain',
+            }
+
+    def __init__(self, lpurl, bzurl, flags, modifiers):
+        self.tmpdir = None
         self.parse_flags(flags)
-        self.tmpdir = None
+        self.parse_modifiers(modifiers)
 
         self.launchpad = LaunchpadInfo(url=lpurl)
         self.bugzilla = BugzillaInfo(url=bzurl,
-                                     user=bzuser,
-                                     password=bzpassword)
+                                     user=self.bzuser,
+                                     password=self.bzpassword)
         self.parse_bugids()
         self.parse_urls()
 
@@ -209,6 +235,75 @@
 
         return secret
 
+    def check_tar_limit(self, fd):
+        if not self.xlimit:
+            return True
+
+        tar = tarfile.open(fileobj=fd)
+        result = len(tar.getnames()) <= self.xlimit
+        fd.seek(0)
+
+        return result
+
+    def extract_tar_files(self, fd):
+        tar = tarfile.open(fileobj=fd)
+
+        infos = []
+        for member in tar:
+            if member.isfile():
+                info = AttachmentInfo()
+                info.title = os.path.basename(
+                             member.name.decode(getpreferredencoding()))
+                info.comment = info.title
+                info.ispatch = (fnmatch(info.title, '*.diff') or
+                                fnmatch(info.title, '*.patch'))
+
+                tar.extract(member, self.tmpdir)
+                oldpath = os.path.join(self.tmpdir,
+                                       member.name)
+                newpath = os.path.join(self.tmpdir,
+                                       os.path.basename(member.name))
+                shutil.move(oldpath, newpath)
+
+                infos.append(info)
+
+        return infos
+
+    def check_zip_limit(self, fd):
+        if not self.xlimit:
+            return True
+
+        zip = ZipFile(file=fd)
+        result = len(zip.namelist()) <= self.xlimit
+        fd.seek(0)
+
+        return result
+
+    def extract_zip_files(self, fd):
+        zip = ZipFile(file=fd)
+
+        infos = []
+        for member in [zip.open(name) for name in zip.namelist()]:
+            if member.name[-1] == '/':
+                continue
+
+            info = AttachmentInfo()
+            info.title = os.path.basename(member.name)
+            info.comment = info.title
+            info.ispatch = (fnmatch(info.title, '*.diff') or
+                            fnmatch(info.title, '*.patch'))
+
+            zip.extract(member.name, self.tmpdir)
+            oldpath = os.path.join(self.tmpdir,
+                                   member.name)
+            newpath = os.path.join(self.tmpdir,
+                                   os.path.basename(member.name))
+            shutil.move(oldpath, newpath)
+
+            infos.append(info)
+
+        return infos
+
     def login_launchpad(self):
         print "Logging into Launchpad"
         self.launchpad.service = LaunchpadService(config={
@@ -252,12 +347,6 @@
     def download_launchpad_attachments(self):
         bug = self.launchpad.service.get_launchpad_bug(self.launchpad.bugid)
         for attachment in bug.attachments:
-            if self.OWNERONLY:
-                if not attachment.message.owner == bug.owner:
-                    print 'Skipping attachment <%s>' % attachment.title,
-                    print '[Not uploaded by bug owner]'
-                    continue
-
             info = AttachmentInfo()
             info.title = attachment.title
             info.comment = attachment.message.subject
@@ -265,6 +354,64 @@
 
             remotefd = attachment.data.open()
             info.contenttype = remotefd.content_type
+
+            if self.OWNERONLY:
+                if not attachment.message.owner == bug.owner:
+                    print 'Skipping attachment <%s>' % attachment.title
+                    print '\t[Not uploaded by bug owner]'
+                    continue
+
+            if (self.EXTRACT and
+                    (fnmatch(attachment.title, '*.tar.gz') or
+                     fnmatch(attachment.title, '*.tar.bz2') or
+                     remotefd.content_type == 'application/x-tar') and
+                    self.check_tar_limit(remotefd)):
+                infos = self.extract_tar_files(remotefd)
+                self.attachments.extend(infos)
+                continue
+
+            if (self.EXTRACT and
+                    (fnmatch(attachment.title, '*.zip') or
+                     remotefd.content_type == 'application/zip') and
+                    self.check_zip_limit(remotefd)):
+                infos = self.extract_zip_files(remotefd)
+                self.attachments.extend(infos)
+                continue
+
+            if self.FORCEMIME:
+                for pattern, mimetype in self.mimetypes.iteritems():
+                    if fnmatch(attachment.title, pattern):
+                        info.contenttype = mimetype
+                        break
+
+            if self.exclude and fnmatch(attachment.title, self.exclude):
+                print "Skipping attachment <%s>" % attachment.title
+                print "\t[Matches filter: %s]" % self.exclude
+                continue
+
+            if self.maxsize and remotefd.len > self.maxsize:
+                if self.GZIP:
+                    print "Gzipping attachment <%s>" % attachment.title
+
+                    info.title = info.title + '.gz'
+                    info.contenttype = 'application/gzip'
+                    tmpfile = os.path.join(self.tmpdir, info.title)
+                    gzipfd = gzip.open(tmpfile, 'w+b')
+                    gzipfd.write(remotefd.read())
+                    gzipfd.close()
+                    stat = os.stat(tmpfile)
+
+                    print "\t[Old size: %d]" % remotefd.len
+                    print "\t[New size: %d]" % stat.st_size
+
+                    if stat.st_size <= self.maxsize:
+                        self.attachments.append(info)
+                        continue
+
+                print "Skipping attachment <%s>" % attachment.title
+                print "\t[Filesize exceeds limit: %d]" % self.maxsize
+                continue
+
             tmpfile = os.path.join(self.tmpdir, attachment.title)
             with open(tmpfile, 'w+b') as localfd:
                 localfd.write(remotefd.read())
@@ -287,9 +434,15 @@
                           tmpfile.encode(getpreferredencoding()))),
                 ('description',
                           attachment.title.encode(self.bugzilla.encoding)),
-                ('ispatch', attachment.ispatch and '1' or '0'),
-                ('contenttypemethod', 'manual'),
-                ('contenttypeentry', attachment.contenttype)])
+                ('ispatch', attachment.ispatch and '1' or '0')])
+
+            if attachment.contenttype:
+                formdata.extend([
+                    ('contenttypemethod', 'manual'),
+                    ('contenttypeentry', attachment.contenttype)])
+            else:
+                formdata.extend([
+                    ('contenttypemethod', 'autodetect')])
 
             headers, response = self.get_bugzilla_response(
                                     opts=[(pycurl.HTTPPOST, formdata),])
@@ -302,7 +455,10 @@
             if pattern.search(response):
                 print "\t[Success]"
             else:
-                print "\t[Error]"
+                pattern = re.compile(r'<title>(.*?)</title>')
+                matches = pattern.search(response).groups()
+                reason = matches[0]
+                print "\t[Error: %s]" % reason
 
     def run(self):
         self.tmpdir = tempfile.mkdtemp()
@@ -336,21 +492,44 @@
     %prog [OPTIONS] LP_URL BUGZILLA_URL
     '''
     parser = OptionParser(usage=usage)
-    parser.add_option('-u', '--user',
-        action='store', type='string', dest='user',
-        help='Bugzilla username')
-    parser.add_option('-p', '--password',
-        action='store', type='string', dest='password',
-        help='Bugzilla password')
     parser.add_option('-o', '--owner-only',
         action='store_true', dest='OWNERONLY',
         default=False,
         help='Process only the attachments uploaded by '
              'original bug reporter')
+    parser.add_option('-x', '--extract',
+        action='store_true', dest='EXTRACT',
+        default=False,
+        help='Extract tar archives')
+    parser.add_option('-g', '--gzip',
+        action='store_true', dest='GZIP',
+        default=False,
+        help='Gzip files which exceed maximum attachment '
+             'size')
+    parser.add_option('-f', '--force-mime',
+        action='store_true', dest='FORCEMIME',
+        default=False,
+        help='Extract tar archives')
     parser.add_option('-d', '--debug',
         action='store_true', dest='DEBUG',
         default=False,
         help='Enable debugging output')
+    parser.add_option('-u', '--user',
+        action='store', type='string', dest='user',
+        help='Bugzilla username')
+    parser.add_option('-p', '--password',
+        action='store', type='string', dest='password',
+        help='Bugzilla password')
+    parser.add_option('-e', '--exclude',
+        action='store', type='string', dest='exclude',
+        help='Wildcard pattern for attachment exclusion')
+    parser.add_option('-l', '--extract-limit',
+        action='store', dest='xlimit',
+        help='Limit on number of archive members for '
+             'extraction')
+    parser.add_option('-m', '--max-size',
+        action='store', dest='maxsize',
+        help='Maximum filesize for processing attachments')
     (options, args) = parser.parse_args()
 
     if len(args) < 2:
@@ -358,13 +537,42 @@
         sys.exit(1)
 
     flags = 0
-    flags |= options.DEBUG and SendAttachments.DEBUG
-    flags |= options.OWNERONLY and SendAttachments.OWNERONLY
+    for flagname in SendAttachments.flagnames:
+        flags |= getattr(options, flagname) and \
+                 getattr(SendAttachments, flagname)
+
+    if options.xlimit:
+        try:
+            options.xlimit = int(options.xlimit)
+        except ValueError:
+            options.xlimit = None
+
+    if options.maxsize:
+        try:
+            options.maxsize = int(options.maxsize)
+        except ValueError:
+            try:
+                prefixes = {'k': 1024, 'm': 1048576, 'g': 1073741824}
+                prefix = options.maxsize[-1:].lower()
+                options.maxsize = int(options.maxsize[:-1])
+                if prefixes.has_key(prefix):
+                    options.maxsize = options.maxsize * prefixes[prefix]
+                else:
+                    raise ValueError
+            except ValueError:
+                options.maxsize = None
+
+    modifiers = {
+        'bzuser' : options.user,
+        'bzpassword' : options.password,
+        'exclude' : options.exclude,
+        'xlimit' : options.xlimit,
+        'maxsize' : options.maxsize,
+        }
 
     app = SendAttachments(lpurl=args[0],
         bzurl=args[1],
-        bzuser=options.user,
-        bzpassword=options.password,
-        flags=flags)
+        flags=flags,
+        modifiers=modifiers)
     app.run()