zorba-coders team mailing list archive

[Luis Rodriguez Gonzalez <kuraru@xxxxxxxxx>]

Luis Rodriguez Gonzalez has proposed merging lp:~zorba-coders/zorba/1039576 into lp:zorba/archive-module.

Requested reviews:
  Matthias Brantner (matthias-brantner)
Related bugs:
  Bug #1039576 in Zorba: "archive module doesnt create directory entries"
  https://bugs.launchpad.net/zorba/+bug/1039576

For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/1039576/+merge/123170

- Added ability to create directories in archives
- Added testcases
-- 
https://code.launchpad.net/~zorba-coders/zorba/1039576/+merge/123170
Your team Zorba Coders is subscribed to branch lp:zorba/archive-module.

=== modified file 'src/archive.xsd'
--- src/archive.xsd	2012-08-01 20:47:46 +0000
+++ src/archive.xsd	2012-09-06 20:59:23 +0000
@@ -53,6 +53,13 @@
     <xs:restriction base="xs:string"/>
   </xs:simpleType>
   
+  <xs:simpleType name="entry-type">
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="regular"/>
+      <xs:enumeration value="directory"/>
+    </xs:restriction>
+  </xs:simpleType>
+  
   <!-- elements -->
   <xs:element name="archive-option-element"
     abstract="true" type="xs:string"/>
@@ -88,6 +95,7 @@
  
   <!-- entry attributes -->
   <xs:attributeGroup name="entry-attributes">
+    <xs:attribute name="type" type="entry-type"/>
     <xs:attribute name="size" type="entry-size-type"/>
     <xs:attribute name="last-modified" type="entry-datetime-type"/>
     <xs:attribute name="encoding" type="entry-encoding-type"/>

=== modified file 'src/archive_module.xq.src/archive_module.cpp'
--- src/archive_module.xq.src/archive_module.cpp	2012-08-03 04:53:47 +0000
+++ src/archive_module.xq.src/archive_module.cpp	2012-09-06 20:59:23 +0000
@@ -119,6 +119,8 @@
     gmtime_r(&aTime, &gmtm);
 #endif
 
+    // create a datetime item without timezone because
+    // this is what the entry tells us (at least for zip)
     Item lModifiedItem = getItemFactory()->createDateTime(
         static_cast<short>(gmtm.tm_year + 1900),
         static_cast<short>(gmtm.tm_mon + 1),
@@ -167,7 +169,8 @@
   ******************/
 
   ArchiveFunction::ArchiveEntry::ArchiveEntry()
-    : theEncoding("UTF-8")
+    : theEncoding("UTF-8"),
+      theEntryType(regular)
   {
     // use current time as a default for each entry
 #if defined (WIN32)
@@ -187,7 +190,7 @@
 
     if (archive_entry_size_is_set(aEntry))
     {
-      //add a size variable
+      theSize = (int)archive_entry_size(aEntry);
     }
 
     if (archive_entry_mtime_is_set(aEntry))
@@ -195,12 +198,19 @@
       theLastModified = archive_entry_mtime(aEntry);
     }
     //check if it is encoded
+
+    switch(archive_entry_filetype(aEntry))
+    {
+      case AE_IFDIR: theEntryType = directory; break;
+      default: theEntryType = regular; break;
+    }
   }
 
   void
   ArchiveFunction::ArchiveEntry::setValues(zorba::Item& aEntry)
   {
     theEntryPath = aEntry.getStringValue();
+
     if (aEntry.isNode())
     {
       Item lAttr;
@@ -212,7 +222,15 @@
         Item lAttrName;
         lAttr.getNodeName(lAttrName);
 
-        if (lAttrName.getLocalName() == "last-modified")
+        if(lAttrName.getLocalName() == "type")
+        {
+          String filetype = lAttr.getStringValue();
+          if(filetype == "directory")
+          {
+            theEntryType = directory;
+          }
+        }
+        else if (lAttrName.getLocalName() == "last-modified")
         {
           ArchiveModule::parseDateTimeItem(lAttr, theLastModified);
         }
@@ -548,12 +566,15 @@
 
     for (size_t i = 0; i < aEntries.size(); ++i)
     {
-      if (!aFiles->next(lFile))
+      if(aEntries[i].getEntryType() == ArchiveEntry::regular)
       {
-        std::ostringstream lMsg;
-        lMsg << "number of entries (" << aEntries.size()
-          << ") doesn't match number of content arguments (" << i << ")";
-        throwError("ARCH0001", lMsg.str().c_str());
+        if (!aFiles->next(lFile))
+        {
+          std::ostringstream lMsg;
+          lMsg << "number of entries (" << aEntries.size()
+            << ") doesn't match number of content arguments (" << i << ")";
+          throwError("ARCH0001", lMsg.str().c_str());
+        }
       }
 
       const ArchiveEntry& lEntry = aEntries[i];
@@ -578,14 +599,19 @@
       std::istream* lStream;
       bool lDeleteStream;
       uint64_t lFileSize;
-      
-      lDeleteStream = getStream(
-          aEntry, aFile, lStream, lFileSize);
 
       archive_entry_set_pathname(theEntry, aEntry.getEntryPath().c_str());
       archive_entry_set_mtime(theEntry, aEntry.getLastModified(), 0);
       // TODO: modified to allow the creation of empty directories
-      archive_entry_set_filetype(theEntry, AE_IFREG);
+      if(aEntry.getEntryType() == ArchiveEntry::regular){
+        archive_entry_set_filetype(theEntry, AE_IFREG);
+        lDeleteStream = getStream(
+          aEntry, aFile, lStream, lFileSize);
+      } else {
+        archive_entry_set_filetype(theEntry, AE_IFDIR);
+        lDeleteStream = false;
+        lFileSize = 0;
+      }
       // TODO: specifies the permits of a file
       archive_entry_set_perm(theEntry, 0644);
       archive_entry_set_size(theEntry, lFileSize);
@@ -632,11 +658,14 @@
 
       archive_write_header(theArchive, theEntry);
 
-      char lBuf[ZORBA_ARCHIVE_MAX_READ_BUF];
-      while (lStream->good())
+      if(aEntry.getEntryType() == ArchiveEntry::regular)
       {
-        lStream->read(lBuf, ZORBA_ARCHIVE_MAX_READ_BUF);
-        archive_write_data(theArchive, lBuf, lStream->gcount());
+        char lBuf[ZORBA_ARCHIVE_MAX_READ_BUF];
+        while (lStream->good())
+        {
+          lStream->read(lBuf, ZORBA_ARCHIVE_MAX_READ_BUF);
+          archive_write_data(theArchive, lBuf, lStream->gcount());
+        }
       }
 
       archive_entry_clear(theEntry);
@@ -890,9 +919,15 @@
         base64::attach(*theData.theStream);
       }
 
-	    lErr = archive_read_open(
-          theArchive, &theData, 0, ArchiveItemSequence::readStream, 0);
-
+	    lErr = archive_read_set_read_callback(
+        theArchive, ArchiveItemSequence::readStream);
+      ArchiveFunction::checkForError(lErr, 0, theArchive);
+
+      lErr = archive_read_set_callback_data(
+        theArchive, &theData);
+      ArchiveFunction::checkForError(lErr, 0, theArchive);
+
+      lErr = archive_read_open1(theArchive);
       ArchiveFunction::checkForError(lErr, 0, theArchive);
     }
     else
@@ -1021,6 +1056,7 @@
 
     theLastModifiedName = theFactory->createQName("", "last-modified");
     theUncompressedSizeName = theFactory->createQName("", "size");
+    theEntryType = theFactory->createQName("", "type");
   }
 
   bool
@@ -1071,6 +1107,26 @@
           aRes, theLastModifiedName, lType, lModifiedItem);
     }
 
+    Item lEntryType;
+    if(archive_entry_filetype(lEntry) == AE_IFDIR)
+    {
+      // this entry is a directory
+      lEntryType = theFactory->createString("directory");
+    }
+    else if(archive_entry_filetype(lEntry) == AE_IFREG)
+    {
+      lEntryType = theFactory->createString("regular");
+    }
+    else
+    {
+      // error! type not supported!
+      // for the time being don't do anything
+    }
+
+    lType = theUntypedQName;
+    theFactory->createAttributeNode(
+      aRes, theEntryType, lType, lEntryType);
+
     // skip to the next entry and raise an error if that fails
     lErr = archive_read_data_skip(theArchive);
     ArchiveFunction::checkForError(lErr, 0, theArchive);
@@ -1183,26 +1239,6 @@
     if (!lEntry)
       return false;
 
-    /*while (true)
-    {
-      int lErr = archive_read_next_header(theArchive, &lEntry);
-      
-      if (lErr == ARCHIVE_EOF) return false;
-
-      if (lErr != ARCHIVE_OK)
-      {
-        ArchiveFunction::checkForError(lErr, 0, theArchive);
-      }
-
-      if (theReturnAll) break;
-
-      String lName = archive_entry_pathname(lEntry);
-      if (theEntryNames.find(lName) != theEntryNames.end())
-      {
-        break;
-      }
-    }*/
-
     String lResult;
 
     // reserve some space if we know the decompressed size
@@ -1294,26 +1330,6 @@
     if (!lEntry)
       return false;
     
-    /*while (true)
-    {
-      int lErr = archive_read_next_header(theArchive, &lEntry);
-      
-      if (lErr == ARCHIVE_EOF) return false;
-
-      if (lErr != ARCHIVE_OK)
-      {
-        ArchiveFunction::checkForError(lErr, 0, theArchive);
-      }
-
-      if (theReturnAll) break;
-
-      String lName = archive_entry_pathname(lEntry);
-      if (theEntryNames.find(lName) != theEntryNames.end())
-      {
-        break;
-      }
-    }*/
-
     std::vector<unsigned char> lResult;
 
     // reserve some space if we know the decompressed size
@@ -1429,32 +1445,34 @@
 
     //form an ArchiveEntry with the entry
     theEntry.setValues(lEntry);
-    
-    //read entry content
-    std::vector<unsigned char> lResult;
-
-    if (archive_entry_size_is_set(lEntry))
-    {
-      long long lSize = archive_entry_size(lEntry);
-      lResult.reserve(lSize);
-    }
-
-    std::vector<unsigned char> lBuf;
-    lBuf.resize(ZORBA_ARCHIVE_MAX_READ_BUF);
-
-    //read entry into string
-    while (true)
-    {
-      int s = archive_read_data(
-        theArchive, &lBuf[0], ZORBA_ARCHIVE_MAX_READ_BUF);
+
+    if(archive_entry_filetype(lEntry) == AE_IFREG){
+      //read entry content
+      std::vector<unsigned char> lResult;
+
+      if (archive_entry_size_is_set(lEntry))
+      {
+        long long lSize = archive_entry_size(lEntry);
+        lResult.reserve(lSize);
+      }
+
+      std::vector<unsigned char> lBuf;
+      lBuf.resize(ZORBA_ARCHIVE_MAX_READ_BUF);
+
+      //read entry into string
+      while (true)
+      {
+        int s = archive_read_data(
+          theArchive, &lBuf[0], ZORBA_ARCHIVE_MAX_READ_BUF);
      
-      if (s == 0) break;
-
-      lResult.insert(lResult.end(), lBuf.begin(), lBuf.begin() + s);
+        if (s == 0) break;
+
+        lResult.insert(lResult.end(), lBuf.begin(), lBuf.begin() + s);
+      }
+
+      aRes = theFactory->createBase64Binary(&lResult[0], lResult.size());
     }
 
-    aRes = theFactory->createBase64Binary(&lResult[0], lResult.size());
-
     return true;
   }
 
@@ -1608,31 +1626,36 @@
     
     //form an ArchiveEntry with the entry
     theEntry.setValues(lEntry);
+
+    if(archive_entry_filetype(lEntry) == AE_IFREG){
     
-    //read entry content
-    std::vector<unsigned char> lResult;
-
-    if (archive_entry_size_is_set(lEntry))
-    {
-      long long lSize = archive_entry_size(lEntry);
-      lResult.reserve(lSize);
-    }
-
-    std::vector<unsigned char> lBuf;
-    lBuf.resize(ZORBA_ARCHIVE_MAX_READ_BUF);
-
-    //read entry into string
-    while (true)
-    {
-      int s = archive_read_data(
-        theArchive, &lBuf[0], ZORBA_ARCHIVE_MAX_READ_BUF);
+      //read entry content
+      std::vector<unsigned char> lResult;
+
+      if (archive_entry_size_is_set(lEntry))
+      {
+        long long lSize = archive_entry_size(lEntry);
+        lResult.reserve(lSize);
+      }
+
+      std::vector<unsigned char> lBuf;
+      lBuf.resize(ZORBA_ARCHIVE_MAX_READ_BUF);
+
+      //read entry into string
+      while (true)
+      {
+        int s = archive_read_data(
+          theArchive, &lBuf[0], ZORBA_ARCHIVE_MAX_READ_BUF);
      
-      if (s == 0) break;
-
-      lResult.insert(lResult.end(), lBuf.begin(), lBuf.begin() + s);
+        if (s == 0) break;
+
+        lResult.insert(lResult.end(), lBuf.begin(), lBuf.begin() + s);
+      }
+
+      aRes = theFactory->createBase64Binary(&lResult[0], lResult.size());
     }
-
-    aRes = theFactory->createBase64Binary(&lResult[0], lResult.size());
+    // else? if the entry represents a directory what are we
+    // going to return??
 
     return true;
   }

=== modified file 'src/archive_module.xq.src/archive_module.h'
--- src/archive_module.xq.src/archive_module.h	2012-08-01 20:47:46 +0000
+++ src/archive_module.xq.src/archive_module.h	2012-09-06 20:59:23 +0000
@@ -150,6 +150,14 @@
 #endif    
       readStream(struct archive *a, void *client_data, const void **buff);
 
+      // needed for the "non-linear" zip format
+#ifdef WIN32
+      static __int64 seekStream(struct archive *a, void *data, __int64 request, int whence);
+#else
+      static off_t seekStream(struct archive *a, void *data, off_t request, int whence);
+#endif
+      
+
   };
 
 
@@ -196,12 +204,16 @@
 
       class ArchiveEntry
       {
+      public:
+        enum ArchiveEntryType { regular = 0, directory };
+
       protected:
         String theEntryPath;
         String theEncoding;
         int theSize;
         time_t theLastModified;
         String theCompression;
+        ArchiveEntryType theEntryType;
         bool theSkipExtras;
 
       public:
@@ -217,6 +229,8 @@
         
         const String& getCompression() const { return theCompression; }
 
+        const ArchiveEntryType& getEntryType() const { return theEntryType; }
+
         void setValues(zorba::Item& aEntry);
 
         void setValues(struct archive_entry* aEntry);
@@ -362,6 +376,7 @@
               zorba::Item theEntryName;
               zorba::Item theUncompressedSizeName;
               zorba::Item theLastModifiedName;
+              zorba::Item theEntryType;
 
             public:
               EntriesIterator(zorba::Item& aArchive);

=== added file 'test/Queries/dir_01.xq'
--- test/Queries/dir_01.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/dir_01.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,10 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+
+let $foo-content := "<foo/>"
+let $bar-content := "<bar/>"
+let $archive := a:create(
+  ("foo.xml", "bar.xml", <entry type="directory">dir1</entry>),
+  ($foo-content, $bar-content)
+)
+return
+  for $e in a:entries($archive) return concat($e/text(), ",")

=== added file 'test/Queries/dir_02.xq'
--- test/Queries/dir_02.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/dir_02.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,11 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+
+let $foo-content := "<foo/>"
+let $bar-content := "<bar/>"
+let $archive := a:create(
+  ("foo.xml", "bar.xml", <entry type="directory">dir1</entry>),
+  ($foo-content, $bar-content)
+)
+let $archive2 := a:delete($archive, "dir1/")
+return
+  for $e in a:entries($archive2) return concat($e/text(), ",")

=== added file 'test/Queries/dir_03.xq'
--- test/Queries/dir_03.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/dir_03.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,13 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+
+let $foo-content := "<foo/>"
+let $bar-content := "<bar/>"
+let $archive := a:create(
+  ("foo.xml", "bar.xml", <entry type="directory">dir1</entry>),
+  ($foo-content, $bar-content)
+)
+let $archive2 := a:delete($archive, "nonexistent.xml")
+let $entries := a:entries($archive)
+let $entries2 := a:entries($archive2)
+return $entries=$entries2
+  

=== added file 'test/Queries/dir_04.xq'
--- test/Queries/dir_04.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/dir_04.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,13 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+
+let $foo-content := "<foo/>"
+let $bar-content := "<bar/>"
+let $archive := a:create(
+  ("foo.xml", "bar.xml", <entry type="directory">dir1</entry>),
+  ($foo-content, $bar-content)
+)
+let $archive2 := a:update($archive, <a:entry type="directory">newdir</a:entry>, ())
+let $entries := a:entries($archive)
+let $entries2 := a:entries($archive2)
+return for $e in $entries2 return concat($e/text(), ",")
+  

=== added file 'test/Queries/entries_03.spec'
--- test/Queries/entries_03.spec	1970-01-01 00:00:00 +0000
+++ test/Queries/entries_03.spec	2012-09-06 20:59:23 +0000
@@ -0,0 +1,1 @@
+Error: http://www.zorba-xquery.com/modules/archive:ARCH9999

=== added file 'test/Queries/options_02.spec'
--- test/Queries/options_02.spec	1970-01-01 00:00:00 +0000
+++ test/Queries/options_02.spec	2012-09-06 20:59:23 +0000
@@ -0,0 +1,1 @@
+Error: http://www.zorba-xquery.com/modules/archive:ARCH9999

=== added file 'test/Queries/options_02.xq'
--- test/Queries/options_02.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/options_02.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,5 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+import module namespace f = "http://expath.org/ns/file";;
+
+let $tar-bz2 := f:read-binary(fn:resolve-uri("simple.tar.bz2"))
+return a:options($tar-bz2)

=== added file 'test/Queries/options_03.spec'
--- test/Queries/options_03.spec	1970-01-01 00:00:00 +0000
+++ test/Queries/options_03.spec	2012-09-06 20:59:23 +0000
@@ -0,0 +1,1 @@
+Error: http://www.zorba-xquery.com/modules/archive:ARCH9999

=== added file 'test/Queries/options_03.xq'
--- test/Queries/options_03.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/options_03.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,5 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+
+let $fake_archive := xs:base64Binary("5Pb8")
+return 
+  a:options($fake_archive)

=== added file 'test/Queries/update_02.xq'
--- test/Queries/update_02.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/update_02.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,12 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+import module namespace b = "http://www.zorba-xquery.com/modules/converters/base64";;
+
+let $foo-content := "<foo/>"
+let $bar-content := xs:base64Binary("YWJj")
+let $archive := a:create(
+  ("foo.xml", "bar.txt"),
+  ($foo-content, $bar-content)
+)
+let $new-archive := a:update($archive, "foo2.xml", "<foo2/>")
+return (count(a:entries($new-archive)), a:extract-text($new-archive, "foo2.xml"))
+

=== added file 'test/Queries/update_03.spec'
--- test/Queries/update_03.spec	1970-01-01 00:00:00 +0000
+++ test/Queries/update_03.spec	2012-09-06 20:59:23 +0000
@@ -0,0 +1,1 @@
+Error: http://www.zorba-xquery.com/modules/archive:ARCH0001

=== added file 'test/Queries/update_03.xq'
--- test/Queries/update_03.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/update_03.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,12 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+import module namespace b = "http://www.zorba-xquery.com/modules/converters/base64";;
+
+let $foo-content := "<foo/>"
+let $bar-content := xs:base64Binary("YWJj")
+let $archive := a:create(
+  ("foo.xml", "bar.txt"),
+  ($foo-content, $bar-content)
+)
+let $new-archive := a:update($archive, ("foo2.xml", "bar2.xml"), ("<foo2/>"))
+return (count(a:entries($new-archive)), a:extract-text($new-archive, "foo2.xml"))
+

=== added file 'test/Queries/update_04.spec'
--- test/Queries/update_04.spec	1970-01-01 00:00:00 +0000
+++ test/Queries/update_04.spec	2012-09-06 20:59:23 +0000
@@ -0,0 +1,1 @@
+Error: http://www.zorba-xquery.com/modules/archive:ARCH9999

=== added file 'test/Queries/update_04.xq'
--- test/Queries/update_04.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/update_04.xq	2012-09-06 20:59:23 +0000
@@ -0,0 +1,7 @@
+import module namespace a = "http://www.zorba-xquery.com/modules/archive";;
+import module namespace b = "http://www.zorba-xquery.com/modules/converters/base64";;
+
+let $fake_archive := xs:base64Binary("YWJj")
+let $new-archive := a:update($fake_archive, "foo2.xml", "<foo2/>")
+return (count(a:entries($new-archive)), a:extract-text($new-archive, "foo2.xml"))
+