keryx team mailing list archive

[mac9416 <mac9416@xxxxxxxxxxxxxxxx>]

mac9416 has proposed merging lp:~mac9416/keryx/trunk into lp:keryx.

Requested reviews:
    Chris Oliver (excid3)
-- 
https://code.launchpad.net/~mac9416/keryx/trunk/+merge/9720
Your team Keryx Development Team is subscribed to branch lp:keryx.

=== added file 'doc/LocalRepo-Add.py'
--- doc/LocalRepo-Add.py	1970-01-01 00:00:00 +0000
+++ doc/LocalRepo-Add.py	2009-08-05 20:27:15 +0000
@@ -0,0 +1,89 @@
+import os, os.path, shutil
+from gzip import GzipFile
+
+REPO_NAME = 'quick_repo'
+repodir = os.path.join(os.getcwd(), REPO_NAME)
+
+listdir   = os.path.join(os.getcwd(), 'lists')
+packdir   = os.path.join(os.getcwd(), 'packages')
+if not os.path.exists(listdir):
+    print "There is somethin' wrong with you, son! You don't even have a ./lists directory. Make sure that you are running this script from within an APT-type Keryx project.'"
+if not os.path.exists(packdir):
+    print "What's this?! I can't find a ./packages directory. Make sure that you are running this script from within an APT-type Keryx project."
+listfiles = os.listdir(listdir)
+packfiles = os.listdir(packdir)
+
+def stripDotCom(text):
+    append = False
+    final = ''
+    for item in text.split('_'):
+        if item == 'dists': append = True
+        if append:
+            final = os.path.join(final, item)
+    return final
+
+def convertUnderscores(text, prefix): # Returns a file that has converted underscores ("_") to / or \ in a filename.
+    final = ''
+    for item in text.split('_'):
+        final = os.path.join(final, item)
+    return os.path.join(prefix, final)
+
+def splitPacks(text):
+    filename = ''
+    packs = {}
+    for block in text.split('\n\n'):
+        for line in block.split('\n'):
+            if line.startswith('Filename: '):
+                filename = line[10:]
+        if filename != '':
+            packs.update({filename:block})
+    return packs
+
+count = 0
+lists = {}
+for filename in listfiles:
+    try:
+        listfile = open(os.path.join(listdir, filename), 'rb')
+    except:
+        print "Well, that list just wouldn't load: " + filename
+        continue
+    packs = splitPacks(listfile.read()) # splitPacks returns {packfilename:packtext, etc.}
+    listfile.close()
+    if lists.has_key(stripDotCom(filename)):       # If a list located in the same part of the repo has already been scanned...
+        lists[stripDotCom(filename)].update(packs) # Simply add the current data to that file.
+    else:                                          # Else...
+        lists.update({stripDotCom(filename):packs})# Add the new list!
+    countdup = len(lists[stripDotCom(filename)]) - len(packs)
+    count += 1
+    print "Loaded", count, "of", len(listfiles), "lists ->", str(len(packs)), "more packages,", str(countdup), "duplicates."
+
+for packlist in lists.iteritems():
+    packlisttext = ""
+    dirlist = os.path.abspath(os.path.join(repodir, packlist[0]))
+    for pack in packlist[1].iteritems():
+        dirpack = os.path.abspath(os.path.join(repodir, pack[0]))
+        if os.path.split(pack[0])[-1] in packfiles and not os.path.exists(dirpack): # If the file from the index file is in the packages directory but not yet in repo...
+            packlisttext += (pack[1] + '\n\n')                                      # Add this to the new index file,
+            if not os.path.exists(os.path.dirname(dirpack)):                        # Then copy the deb into the repo.
+                try:
+                    os.makedirs(os.path.dirname(dirpack))                           # If the destination dir doesn't exist, create it.
+                    print "Creating dir: " + os.path.dirname(pack[0])
+                except:
+                    print "Failed creating dir: " + os.path.dirname(pack[0])
+                    pass
+            print "Copying: " + os.path.split(pack[0])[-1] + "..."
+            shutil.copy(os.path.join(packdir, os.path.split(pack[0])[-1]), dirpack)
+    if packlisttext != '':               # Only bother with the Packages.gz file if there is a reason
+        if not os.path.exists(os.path.dirname(dirlist)): 
+            try:
+                os.makedirs(os.path.dirname(dirlist))
+                print "Creating dir: " + os.path.dirname(packlist[0])
+            except:
+                print "Failed creating dir: " + os.path.dirname(packlist[0])
+                pass
+        print "Writing file: " + packlist[0] + '.gz'
+        packlistfile = file(dirlist + '.gz', 'ab')           # If repo already has this Packages.gz file then add the new files to it.
+        gzfile = GzipFile(dirlist, 'ab', 9, packlistfile)
+        gzfile.write(packlisttext)
+        gzfile.close()
+        packlistfile.close()

=== added file 'doc/VerifyChecksums.py'
--- doc/VerifyChecksums.py	1970-01-01 00:00:00 +0000
+++ doc/VerifyChecksums.py	2009-08-05 20:27:15 +0000
@@ -0,0 +1,86 @@
+# original code from mac9416's script 'QuickRepo.py'
+# modified and repurposed by jaseen
+
+import os, os.path, hashlib
+#from gzip import GzipFile
+
+
+listdir   = os.path.join(os.getcwd(), 'lists')
+packdir   = os.path.join(os.getcwd(), 'packages')
+hashtype = ('MD5sum: ', 'md5')
+#hashtype = ('SHA1: ', 'sha1')  # sha1 checks keep failing completely !?! Use  md5
+
+if not os.path.exists(listdir):
+    print "There is somethin' wrong with you, son! You don't even have a ./lists directory. Make sure that you are running this script from within an APT-type Keryx project.'"
+if not os.path.exists(packdir):
+    print "What's this?! I can't find a ./packages directory. Make sure that you are running this script from within an APT-type Keryx project."
+listfiles = os.listdir(listdir)
+packfiles = os.listdir(packdir)
+
+def calcChecksum(filepath):
+    try:                            # This code originally found in doubledetector.py from http://sebsauvage.net/python/
+        file = open(filepath,'rb')  # some modifications made of course.
+        if hashtype[1] == 'md5':
+            digest = hashlib.md5()
+        elif hashtype[1] == 'sha1':
+            digest = hashlib.sha1()
+        else:
+            return '0'
+        data = file.read(65536)
+        while len(data) != 0:
+            digest.update(data)
+            data = file.read(65536)
+        file.close()
+    except:
+        return '0'
+    else:
+        return digest.hexdigest()
+
+
+def ParseLists(text):
+    filename = ''
+    hashsum = ''
+    packs = {}
+    for block in text.split('\n\n'):
+        for line in block.split('\n'):
+            if line.startswith('Filename: '):
+                filename = os.path.split(line[10:])[-1]
+            if line.startswith(hashtype[0]):
+                hashsum = line[8:]
+        if filename != '':
+            if hashsum == '':
+                print "A file without a '" + hashtype[0] + "', interesting: " + filename
+            else:
+                packs.update({filename:hashsum})
+    return packs
+
+count = 0
+lists = {}
+for filename in listfiles:
+    try:
+        listfile = open(os.path.join(listdir, filename), 'rb')
+    except:
+        print "Well, that list just wouldn't load: " + filename
+        continue
+    packs = ParseLists(listfile.read()) # ParseLists returns {packfilename:packmd5sum, etc.}
+    listfile.close()
+    lists.update(packs)         # Add the new list!
+    count += 1
+    print count, "read of", len(listfiles), "-", len(packs), "more names, ", len(lists), "unique"
+
+packlistsums = 0
+failed = 0
+nocalc = 0
+listsorted = sorted(lists.iterkeys())
+for key in listsorted:
+    if key in packfiles: # If the file from the index file is in the packages directory...
+        sum = calcChecksum(os.path.join(packdir, key))
+        packlistsums += 1
+        if sum != lists[key] and sum != '0':
+            os.remove(os.path.join(packdir, key))
+            print "Failed " + str(key) + ": Removed."
+            failed += 1
+        elif sum == '0':
+            print "Could not calc checksum", key
+            nocalc += 1
+print "Of", packlistsums, "debs processed,", failed, "failed,", nocalc, "could not be checked."