zorba-coders team mailing list archive

["Paul J. Lucas" <paul@xxxxxxxxxxxxx>]

Paul J. Lucas has proposed merging lp:~zorba-coders/zorba/feature-mem_size into lp:zorba.

Requested reviews:
  Paul J. Lucas (paul-lucas)

For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/feature-mem_size/+merge/114764

Added framework for calculating the total memory used by a data structure and using it with store::Item.
-- 
https://code.launchpad.net/~zorba-coders/zorba/feature-mem_size/+merge/114764
Your team Zorba Coders is subscribed to branch lp:zorba.

=== modified file 'src/store/api/item.h'
--- src/store/api/item.h	2012-07-12 17:29:55 +0000
+++ src/store/api/item.h	2012-07-13 00:33:23 +0000
@@ -104,6 +104,7 @@
 
   void removeReference();
 
+  virtual size_t alloc_size() const;
 
   /* -------------------   General Methods for Items ------------------------- */
 

=== modified file 'src/store/naive/atomic_items.cpp'
--- src/store/naive/atomic_items.cpp	2012-07-12 17:29:55 +0000
+++ src/store/naive/atomic_items.cpp	2012-07-13 00:33:23 +0000
@@ -43,6 +43,7 @@
 #include "tree_id.h"
 
 #include "util/ascii_util.h"
+#include "util/mem_sizeof.h"
 #include "util/string_util.h"
 #include "util/utf8_util.h"
 
@@ -381,6 +382,14 @@
 }
 
 
+size_t UserTypedAtomicItem::alloc_size() const
+{
+  return  AtomicItem::alloc_size()
+        + ztd::alloc_sizeof( theBaseItem )
+        + ztd::alloc_sizeof( theTypeName );
+}
+
+
 /*******************************************************************************
   class UntypedAtomicItem
 ********************************************************************************/
@@ -571,6 +580,12 @@
 }
 
 
+size_t UntypedAtomicItem::alloc_size() const
+{
+  return AtomicItem::alloc_size() + ztd::alloc_sizeof( theValue );
+}
+
+
 bool UntypedAtomicItem::equals(
     const store::Item* other,
     long timezone,
@@ -679,6 +694,15 @@
 }
 
 
+size_t QNameItem::alloc_size() const
+{
+  return  AtomicItem::alloc_size()
+        + ztd::alloc_sizeof( theNamespace )
+        + ztd::alloc_sizeof( thePrefix )
+        + ztd::alloc_sizeof( theLocal );
+}
+
+
 store::Item* QNameItem::getType() const
 {
   return GET_STORE().theSchemaTypeNames[store::XS_QNAME];
@@ -816,6 +840,12 @@
 }
 
 
+size_t NotationItem::alloc_size() const
+{
+  return ztd::mem_sizeof( *theQName );
+}
+
+
 store::Item* NotationItem::getType() const
 {
   return GET_STORE().theSchemaTypeNames[store::XS_NOTATION];
@@ -869,6 +899,12 @@
 }
 
 
+size_t AnyUriItem::alloc_size() const
+{
+  return AtomicItem::alloc_size() + ztd::alloc_sizeof( theValue );
+}
+
+
 bool AnyUriItem::getEBV() const
 {
   return ! (theValue == "");
@@ -1609,10 +1645,21 @@
   }
 }
 
+size_t StructuralAnyUriItem::alloc_size() const
+{
+  return AnyUriItem::alloc_size() + ztd::alloc_sizeof( theOrdPath );
+}
+
 
 /*******************************************************************************
   class StringItem
 ********************************************************************************/
+
+size_t StringItem::alloc_size() const
+{
+  return AtomicItem::alloc_size() + ztd::alloc_sizeof( theValue );
+}
+
 store::Item* StringItem::getType() const
 {
   return GET_STORE().theSchemaTypeNames[store::XS_STRING];
@@ -2402,6 +2449,12 @@
   class DecimalItem
 ********************************************************************************/
 
+size_t DecimalItem::alloc_size() const
+{
+  return AtomicItem::alloc_size() + ztd::alloc_sizeof( theValue );
+}
+
+
 store::Item* DecimalItem::getType() const
 {
   return GET_STORE().theSchemaTypeNames[store::XS_DECIMAL];
@@ -2451,6 +2504,14 @@
   class IntegerItemImpl
 ********************************************************************************/
 
+#ifdef ZORBA_WITH_BIG_INTEGER
+size_t IntegerItemImpl::alloc_size() const
+{
+  return ztd::alloc_size( theValue );
+}
+#endif /* ZORBA_WITH_BIG_INTEGER */
+
+
 long IntegerItemImpl::compare( Item const *other, long,
                                const XQPCollator* ) const {
   try
@@ -2557,6 +2618,15 @@
 /*******************************************************************************
   class NonPositiveIntegerItem
 ********************************************************************************/
+
+#ifdef ZORBA_WITH_BIG_INTEGER
+size_t NonPositiveIntegerItem::alloc_size() const
+{
+  return ztd::alloc_size( theValue );
+}
+#endif /* ZORBA_WITH_BIG_INTEGER */
+
+
 long NonPositiveIntegerItem::compare( Item const *other, long,
                                       const XQPCollator* ) const {
   try
@@ -2669,6 +2739,14 @@
 /*******************************************************************************
   class NonNegativeIntegerItem
 ********************************************************************************/
+
+#ifdef ZORBA_WITH_BIG_INTEGER
+size_t NonNegativeIntegerItem::alloc_size() const
+{
+  return ztd::alloc_size( theValue );
+}
+#endif /* ZORBA_WITH_BIG_INTEGER */
+
 long NonNegativeIntegerItem::compare( Item const *other, long,
                                       const XQPCollator* ) const {
   try
@@ -3330,6 +3408,12 @@
 /*******************************************************************************
   class Base64BinaryItem
 ********************************************************************************/
+
+size_t Base64BinaryItem::alloc_size() const
+{
+  return AtomicItem::alloc_size() + ztd::alloc_sizeof( theValue );
+}
+
 bool
 Base64BinaryItem::equals(
       const store::Item* other,
@@ -3585,6 +3669,13 @@
 /*******************************************************************************
   class HexBinaryItem
 ********************************************************************************/
+
+size_t HexBinaryItem::alloc_size() const
+{
+  return theValue.size();
+}
+
+
 store::Item* HexBinaryItem::getType() const
 {
   return GET_STORE().theSchemaTypeNames[store::XS_HEXBINARY];
@@ -3627,13 +3718,16 @@
 /*******************************************************************************
   class ErrorItem
 ********************************************************************************/
+
 ErrorItem::~ErrorItem()
 {
-  if (theError)
-  {
-    delete theError;
-    theError = NULL;
-  }
+  delete theError;
+}
+
+
+size_t ErrorItem::alloc_size() const
+{
+  return theError ? ztd::mem_sizeof( *theError ) : 0;
 }
 
 

=== modified file 'src/store/naive/atomic_items.h'
--- src/store/naive/atomic_items.h	2012-07-12 17:29:55 +0000
+++ src/store/naive/atomic_items.h	2012-07-13 00:33:23 +0000
@@ -123,6 +123,8 @@
     return theBaseItem->hash(timezone, collation);
   }
 
+  size_t alloc_size() const;
+
   bool equals(
         const store::Item* other,
         long timezone = 0,
@@ -310,6 +312,8 @@
 
   uint32_t hash(long timezone = 0, const XQPCollator* aCollation = 0) const;
 
+  size_t alloc_size() const;
+
   bool equals(
         const store::Item* other,
         long timezone = 0,
@@ -382,6 +386,8 @@
 public:
   virtual ~QNameItem() {}
 
+  size_t alloc_size() const;
+
   // zorba::store::Item interface.
   bool equals(const store::Item* item,
               long timezone = 0,
@@ -554,6 +560,8 @@
       long timezone = 0,
       const XQPCollator* aCollation = 0) const;
 
+  size_t alloc_size() const;
+
   zstring getStringValue() const;
 
   void getStringValue2(zstring& val) const;
@@ -608,6 +616,8 @@
     return item->getString() == theValue;
   }
 
+  size_t alloc_size() const;
+
   long compare(
         const Item* other,
         long timezone = 0,
@@ -730,6 +740,8 @@
   StructuralAnyUriItem() {}
 
 public:
+  size_t alloc_size() const;
+
   bool
   isAncestor(const store::Item_t&) const;
 
@@ -820,6 +832,7 @@
   StringItem() {}
 
 public:
+  size_t alloc_size() const;
 
   virtual store::SchemaTypeCode getTypeCode() const { return store::XS_STRING; }
 
@@ -1379,6 +1392,8 @@
   DecimalItem() {}
 
 public:
+  size_t alloc_size() const;
+
   xs_decimal getDecimalValue() const { return theValue; }
 
   store::SchemaTypeCode getTypeCode() const { return store::XS_DECIMAL; }
@@ -1454,6 +1469,10 @@
   IntegerItemImpl() {}
 
 public:
+#ifdef ZORBA_WITH_BIG_INTEGER
+  size_t alloc_size() const;
+#endif /* ZORBA_WITH_BIG_INTEGER */
+
   xs_decimal getDecimalValue() const;
 
   xs_integer getIntegerValue() const { return theValue; }
@@ -1507,6 +1526,10 @@
   NonPositiveIntegerItem() {}
 
 public:
+#ifdef ZORBA_WITH_BIG_INTEGER
+  size_t alloc_size() const;
+#endif /* ZORBA_WITH_BIG_INTEGER */
+
   xs_decimal getDecimalValue() const;
 
   xs_integer getIntegerValue() const;
@@ -1580,6 +1603,10 @@
   NonNegativeIntegerItem() {}
 
 public:
+#ifdef ZORBA_WITH_BIG_INTEGER
+  size_t alloc_size() const;
+#endif /* ZORBA_WITH_BIG_INTEGER */
+
   xs_decimal getDecimalValue() const;
 
   xs_integer getIntegerValue() const;
@@ -1731,7 +1758,7 @@
   friend class AtomicItem;
 
 protected:
-  int32_t theValue;
+  xs_int theValue;
 
 protected:
   IntItem(xs_int aValue) : theValue(aValue) {}
@@ -1747,7 +1774,7 @@
 
   xs_long getLongValue() const { return static_cast<xs_long>(theValue); }
 
-  int32_t getIntValue() const { return theValue; }
+  xs_int getIntValue() const { return theValue; }
 
   store::SchemaTypeCode getTypeCode() const { return store::XS_INT; }
 
@@ -2404,6 +2431,8 @@
   }
 
 public:
+  size_t alloc_size() const;
+
   const char* getBase64BinaryValue(size_t& data) const;
 
   store::SchemaTypeCode getTypeCode() const { return store::XS_BASE64BINARY; }
@@ -2524,6 +2553,8 @@
   HexBinaryItem() {}
 
 public:
+  size_t alloc_size() const;
+
   xs_hexBinary getHexBinaryValue() const { return theValue; }
 
   store::SchemaTypeCode getTypeCode() const { return store::XS_HEXBINARY; }
@@ -2579,6 +2610,8 @@
 public:
   virtual ~ErrorItem();
 
+  size_t alloc_size() const;
+
   ZorbaException* getError() const { return theError; }
 
   zstring show() const;

=== modified file 'src/store/naive/item.cpp'
--- src/store/naive/item.cpp	2012-07-12 17:29:55 +0000
+++ src/store/naive/item.cpp	2012-07-13 00:33:23 +0000
@@ -225,6 +225,10 @@
 #endif
 }
 
+size_t Item::alloc_size() const {
+  return 0;
+}
+
 
 Item::ItemKind Item::getKind() const
 {

=== modified file 'src/store/naive/node_items.cpp'
--- src/store/naive/node_items.cpp	2012-07-12 17:29:55 +0000
+++ src/store/naive/node_items.cpp	2012-07-13 00:33:23 +0000
@@ -45,6 +45,7 @@
 #include "dataguide.h"
 #include "node_factory.h"
 
+#include "util/mem_sizeof.h"
 #include "util/stl_util.h"
 #include "util/string_util.h"
 
@@ -936,6 +937,10 @@
   }
 }
 
+size_t ConnectorNode::alloc_size() const
+{
+  return XmlNode::alloc_size() + ztd::alloc_sizeof( theNode );
+}
 
 /*******************************************************************************
 
@@ -1012,6 +1017,12 @@
   }
 }
 
+///////////////////////////////////////////////////////////////////////////////
+
+size_t OrdPathNode::alloc_size() const
+{
+  return XmlNode::alloc_size() + ztd::alloc_sizeof( theOrdPath );
+}
 
 /*******************************************************************************
 
@@ -1172,6 +1183,7 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
 bool
 OrdPathNode::getDescendantNodeByOrdPath(
     const OrdPath& aOrdPath,
@@ -1462,6 +1474,13 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
+size_t InternalNode::alloc_size() const
+{
+  return OrdPathNode::alloc_size() + ztd::alloc_sizeof( theNodes );
+}
+
+
 const OrdPath* InternalNode::getFirstChildOrdPathAfter(csize pos) const
 {
   assert((pos == 0 && numChildren() == 0) || pos < numChildren());
@@ -1777,6 +1796,13 @@
 }
 
 
+size_t DocumentNode::alloc_size() const
+{
+  return  InternalNode::alloc_size()
+        + ztd::alloc_sizeof( theBaseUri )
+        + ztd::alloc_sizeof( theDocUri );
+}
+
 /*******************************************************************************
 
 ********************************************************************************/
@@ -2493,6 +2519,16 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
+size_t ElementNode::alloc_size() const
+{
+  return InternalNode::alloc_size()
+#ifdef EMBEDED_TYPE
+      + ztd::alloc_sizeof( theTypeName )
+#endif
+      + ztd::alloc_sizeof( theName );
+}
+
 #ifdef EMBEDED_TYPE
 store::Item* ElementNode::getType() const
 {
@@ -3622,6 +3658,18 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
+size_t AttributeNode::alloc_size() const
+{
+  return  OrdPathNode::alloc_size()
+        + ztd::alloc_sizeof( theName )
+#ifdef EMBEDED_TYPE
+        + ztd::alloc_sizeof( theTypeName )
+#endif
+        + ztd::alloc_sizeof( theTypedValue );
+}
+
+
 XmlNode* AttributeNode::copyInternal(
     InternalNode* rootParent,
     InternalNode* parent,
@@ -4115,6 +4163,18 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
+size_t TextNode::alloc_size() const
+{
+  return  base_type::alloc_size()
+        + (isTyped() ?
+            ztd::alloc_sizeof( theContent.getValue() ) :
+            ztd::alloc_sizeof( theContent.getText() )
+        );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
 XmlNode* TextNode::copyInternal(
     InternalNode* rootParent,
     InternalNode* parent,
@@ -4745,6 +4805,17 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
+size_t PiNode::alloc_size() const
+{
+  return  OrdPathNode::alloc_size()
+        + ztd::alloc_sizeof( theTarget )
+        + ztd::alloc_sizeof( theContent )
+        + ztd::alloc_sizeof( theName );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
 XmlNode* PiNode::copyInternal(
     InternalNode* rootParent,
     InternalNode* parent,
@@ -4879,6 +4950,14 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
+size_t CommentNode::alloc_size() const
+{
+  return  OrdPathNode::alloc_size() + ztd::alloc_sizeof( theContent );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
 XmlNode* CommentNode::copyInternal(
     InternalNode* rootParent,
     InternalNode* parent,

=== modified file 'src/store/naive/node_items.h'
--- src/store/naive/node_items.h	2012-07-12 17:29:55 +0000
+++ src/store/naive/node_items.h	2012-07-13 00:33:23 +0000
@@ -576,6 +576,8 @@
       const XmlNode* child);
 
 public:
+  size_t alloc_size() const;
+
   XmlNode* getNode() const { return theNode.getp(); }
 
   store::Item* getNodeName() const { return theNode->getNodeName(); }
@@ -638,6 +640,8 @@
   virtual ~OrdPathNode() {}
 
 public:
+  size_t alloc_size() const;
+
   const OrdPath& getOrdPath() const { return theOrdPath; }
 
   OrdPath& getOrdPath() { return theOrdPath; }
@@ -748,6 +752,8 @@
   // SimpleStore Methods
   //
 
+  size_t alloc_size() const;
+
   // To be used by the loader ONLY!
   NodeVector& nodes() { return theNodes; }
 
@@ -888,6 +894,8 @@
   // Item methods
   //
 
+  size_t alloc_size() const;
+
   store::Item* getType() const;
 
   void getDocumentURI(zstring& uri) const { uri = theDocUri; }
@@ -977,6 +985,8 @@
   //
   // Item methods
   //
+  size_t alloc_size() const;
+
   store::Item* getNodeName() const { return theName.getp(); }
 
   store::Item* getType() const;
@@ -1164,6 +1174,8 @@
   // Item methods
   //
 
+  size_t alloc_size() const;
+
   store::Item* getNodeName() const { return theName.getp(); }
 
   store::Item* getType() const;
@@ -1258,11 +1270,12 @@
 
 ********************************************************************************/
 #ifdef TEXT_ORDPATH
-class TextNode : public OrdPathNode
+class TextNode : public OrdPathNode {
+  typedef OrdPathNode base_type;
 #else
-class TextNode : public XmlNode
+class TextNode : public XmlNode {
+  typedef XmlNode base_type;
 #endif
-{
   friend class XmlNode;
   friend class InternalNode;
   friend class DocumentDagNode;
@@ -1312,6 +1325,8 @@
   // Item methods
   //
 
+  size_t alloc_size() const;
+
   store::Item* getType() const;
 
   void getTypedValue(store::Item_t& val, store::Iterator_t& iter) const;
@@ -1442,6 +1457,8 @@
   PiNode() {}
 
 public:
+  size_t alloc_size() const;
+
   XmlNode* copyInternal(
         InternalNode* rootParent,
         InternalNode* parent,
@@ -1505,6 +1522,8 @@
   CommentNode() {}
 
 public:
+  size_t alloc_size() const;
+
   XmlNode* copyInternal(
         InternalNode* rootParent,
         InternalNode* parent,

=== modified file 'src/store/naive/ordpath.cpp'
--- src/store/naive/ordpath.cpp	2012-07-12 17:29:55 +0000
+++ src/store/naive/ordpath.cpp	2012-07-13 00:33:23 +0000
@@ -329,12 +329,17 @@
 /*******************************************************************************
 
 ********************************************************************************/
+
+size_t OrdPath::alloc_size() const
+{
+  return isRemote() ? getRemoteByteLength() : 0;
+}
+
 bool OrdPath::isRoot() const
 {
   return isLocal() && theBuffer.local[0] == 0x40;
 }
 
-
 /*******************************************************************************
 
 ********************************************************************************/

=== modified file 'src/store/naive/ordpath.h'
--- src/store/naive/ordpath.h	2012-07-12 17:29:55 +0000
+++ src/store/naive/ordpath.h	2012-07-13 00:33:23 +0000
@@ -65,6 +65,8 @@
     MAX_BIT_LEN = MAX_BYTE_LEN * 8
   };
 
+  size_t alloc_size() const;
+
 protected:
   static const ulong MAX_EMBEDDED_BYTE_LEN = 8;
   static const ulong MAX_EMBEDDED_BIT_LEN = MAX_EMBEDDED_BYTE_LEN * 8 - 1;

=== modified file 'src/unit_tests/CMakeLists.txt'
--- src/unit_tests/CMakeLists.txt	2012-07-12 17:29:55 +0000
+++ src/unit_tests/CMakeLists.txt	2012-07-13 00:33:23 +0000
@@ -18,6 +18,7 @@
   test_base64_streambuf.cpp
   test_fs_iterator.cpp
   test_json_parser.cpp
+  test_mem_sizeof.cpp
   test_string.cpp
   test_uri.cpp
   unit_tests.cpp

=== added file 'src/unit_tests/test_mem_sizeof.cpp'
--- src/unit_tests/test_mem_sizeof.cpp	1970-01-01 00:00:00 +0000
+++ src/unit_tests/test_mem_sizeof.cpp	2012-07-13 00:33:23 +0000
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2006-2008 The FLWOR Foundation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <iostream>
+#include <string>
+
+#include "util/cxx_util.h"
+#include "util/mem_sizeof.h"
+
+using namespace std;
+using namespace zorba;
+
+///////////////////////////////////////////////////////////////////////////////
+
+static int failures;
+
+static bool assert_true( char const *expr, int line, bool result ) {
+  if ( !result ) {
+    cout << "FAILED, line " << line << ": " << expr << endl;
+    ++failures;
+  }
+  return result;
+}
+
+#define ASSERT_TRUE( EXPR ) assert_true( #EXPR, __LINE__, !!(EXPR) )
+
+///////////////////////////////////////////////////////////////////////////////
+
+static void test_int() {
+  int i;
+  ASSERT_TRUE( ztd::mem_sizeof( i ) == sizeof( i ) );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+static void test_map_string_int() {
+  typedef map<string,int> map_type;
+  map_type m;
+  string const key( "a" );
+  m[ key ] = 1;
+
+  size_t const expected_size = sizeof( m )
+    + sizeof( map_type::value_type ) + key.size();
+
+  ASSERT_TRUE( ztd::mem_sizeof( m ) == expected_size );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+struct point {
+  int x, y, z;
+};
+
+static void test_pod() {
+  point p;
+  ASSERT_TRUE( ztd::mem_sizeof( p ) == sizeof( p ) );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+static void test_pointer() {
+  int *p = nullptr;
+  ASSERT_TRUE( ztd::mem_sizeof( p ) == sizeof( p ) );
+  p = new int;
+  ASSERT_TRUE( ztd::mem_sizeof( p ) == sizeof( p ) + sizeof( int ) );
+  delete p;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+template<class StringType>
+static void test_string_empty() {
+  StringType const s;
+  ASSERT_TRUE( ztd::mem_sizeof( s ) == sizeof( s ) );
+}
+
+template<class StringType>
+static void test_string_not_empty() {
+  StringType const s( "hello" );
+  ASSERT_TRUE( ztd::mem_sizeof( s ) == sizeof( s ) + s.size() );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+static void test_vector_int() {
+  typedef vector<int> vector_type;
+  vector_type v;
+  v.push_back( 1 );
+
+  size_t const expected_size = sizeof( v ) + sizeof( vector_type::value_type );
+
+  ASSERT_TRUE( ztd::mem_sizeof( v ) == expected_size );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+struct my_base {
+  virtual size_t alloc_size() const {
+    return ztd::alloc_sizeof( s_ ) + ztd::alloc_sizeof( t_ );
+  }
+
+  string s_, t_;
+};
+
+struct my_derived : my_base {
+  my_derived( string const &s ) : s_( s ) { }
+
+  size_t alloc_size() const {
+    return my_base::alloc_size() + ztd::alloc_sizeof( s_ );
+  }
+
+  string s_;
+};
+
+static void test_base_empty() {
+  my_base b;
+  ASSERT_TRUE( ztd::mem_sizeof( b ) == sizeof( b ) );
+}
+
+static void test_derived_not_empty() {
+  string const s( "hello" );
+  my_derived d( s );
+  ASSERT_TRUE( ztd::mem_sizeof( d ) == sizeof( d ) + s.size() );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+namespace zorba {
+namespace UnitTests {
+
+int test_mem_sizeof( int, char*[] ) {
+  test_int();
+  test_pointer();
+  test_pod();
+
+  test_string_empty<string>();
+  test_string_not_empty<string>();
+  test_string_empty<zstring>();
+  test_string_not_empty<zstring>();
+
+  test_map_string_int();
+  test_vector_int();
+
+  test_base_empty();
+  test_derived_not_empty();
+
+  cout << failures << " test(s) failed\n";
+  return failures ? 1 : 0;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace UnitTests
+} // namespace zorba
+/* vim:set et sw=2 ts=2: */

=== modified file 'src/unit_tests/unit_test_list.h'
--- src/unit_tests/unit_test_list.h	2012-07-12 17:29:55 +0000
+++ src/unit_tests/unit_test_list.h	2012-07-13 00:33:23 +0000
@@ -33,6 +33,7 @@
   int test_icu_streambuf( int, char*[] );
 #endif /* ZORBA_NO_ICU */
   int test_json_parser( int, char*[] );
+  int test_mem_sizeof( int, char*[] );
   int test_string( int, char*[] );
 #ifndef ZORBA_NO_FULL_TEXT
   int test_stemmer( int, char*[] );

=== modified file 'src/unit_tests/unit_tests.cpp'
--- src/unit_tests/unit_tests.cpp	2012-07-12 17:29:55 +0000
+++ src/unit_tests/unit_tests.cpp	2012-07-13 00:33:23 +0000
@@ -43,6 +43,7 @@
 #ifndef ZORBA_NO_ICU
   libunittests["icu_streambuf"] = test_icu_streambuf;
 #endif /* ZORBA_NO_ICU */
+  libunittests["mem_sizeof"] = test_mem_sizeof;
   libunittests["json_parser"] = test_json_parser;
   libunittests["string"] = test_string;
 #ifndef ZORBA_NO_FULL_TEXT

=== added file 'src/util/mem_sizeof.h'
--- src/util/mem_sizeof.h	1970-01-01 00:00:00 +0000
+++ src/util/mem_sizeof.h	2012-07-13 00:33:23 +0000
@@ -0,0 +1,390 @@
+/*
+ * Copyright 2006-2008 The FLWOR Foundation.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ZORBA_MEM_SIZEOF
+#define ZORBA_MEM_SIZEOF
+
+#include <cstring>
+#include <map>
+#include <set>
+#include <stack>
+#include <string>
+#include <vector>
+
+#include <zorba/internal/ztd.h>
+#include <zorba/internal/unique_ptr.h>
+
+#include "util/stl_util.h"
+#include "util/unordered_map.h"
+#include "util/unordered_set.h"
+#include "zorbatypes/rchandle.h"
+#include "zorbatypes/zstring.h"
+
+///////////////////////////////////////////////////////////////////////////////
+
+namespace zorba {
+namespace ztd {
+
+/**
+ * \file
+ * This file provides the \c mem_sizeof() function that returns the total
+ * amount of memory an object is using including any dynamically allocated
+ * memory it has pointers to.  Hence \c mem_sizeof() takes a reference to an
+ * actual object at run-time rather than only a type at compile-time like the
+ * build-in C++ \c sizeof operator does.
+ *
+ * For all built-in types as well as arrays, structs, classes, and unions
+ * composed of only built-in types, <code>mem_sizeof(t) == sizeof(t)</code>.
+ * However, for a \c std::string \s,
+ * <code>mem_sizeof(s) == sizeof(s) + s.size()</code>.
+ *
+ * To implement this, there has to be a distinction between
+ * <em>memory size</em> (<code>mem_sizeof()</code)
+ * and <em>allocation size</em> (<code>alloc_sizeof()</code).
+ * The latter is how much \e additional memory has been dynamically allocated
+ * by an object (if any).
+ *
+ * For all objects of built-in types as well as arrays, structs, classes, and
+ * unions composed of only built-in types, <code>alloc_sizeof(t) == 0</code>
+ * because those objects don't have any additional dynamically allocated memory.
+ *
+ * For structs and classes that \e do have additional dynamically allocated
+ * memory, there has to be a way for \c alloc_sizeof() to be able to know how
+ * much memory.
+ * This can be accomplished by two different methods:
+ *  <ol>
+ *    <li>
+ *      Template partial specialization of \c size_traits.
+ *    </li>
+ *    <li>
+ *      Adding an \c alloc_size() member function to your class.
+ *    </li>
+ *  </ol>
+ * Template partial specialization must be used
+ * when you can not (or do not want to) modify a class.
+ * However, this method may not always be able to report
+ * all the dynamically allocated memory an object may be using.
+ * It also has the code for a class and its template specialization
+ * in different places in the code.
+ *
+ * Adding an \c alloc_size() member function
+ * allows you to report the precise amount of memory an object is using
+ * and also has the code directly in the class.
+ * However, it is intrusive and adds a (possibly virtual) function.
+ *
+ * An example of template partial specialization is for \c std::string.
+ * Since the source code for \c std::string can not be (easily) modified,
+ * you must use template partial specialization for it.
+ * \code
+ *  template<>
+ *  struct size_traits<std::string> {
+ *    static size_t alloc_sizeof( std::string const &s ) {
+ *      return s.size();
+ *    }
+ *  };
+ * \endcode
+ * This specialization, however, does not report all the memory
+ * dynamically allocated by some implementations of \c std::string.
+ * Some implementations use an additional "rep" object.
+ * There's no way to determine whether a given implemenation uses one
+ * much less how much memory it uses programatically,
+ * but it's the best that can be done.
+ *
+ * An example of adding an \c alloc_size() member function is:
+ * \code
+ *  class my_class {
+ *  public:
+ *    // ...
+ *    size_t alloc_size() const {
+ *      return alloc_sizeof( s ) + alloc_sizeof( p );
+ *    }
+ *  private:
+ *    int i;
+ *    std::string s;
+ *    some_other_class *p;
+ *  };
+ * \endcode
+ * Notes:
+ *  <ul>
+ *    <li>
+ *      Doing \c alloc_sizeof(i) isn't necessary since it returns 0.
+ *      (You could do it if you wanted to for the sake of completeness
+ *      without incurring any run-time penalty
+ *      since the implementation of \c alloc_sizeof() for built-in C++ types
+ *      is an \c inline function that always returns 0.
+ *      The function will be optimized away by the compiler.)
+ *    </li>
+ *    <li>
+ *      Calling \c alloc_sizeof(p) for a pointer automatically dereferences it
+ *      (if it's non-null) and adds in the \c mem_sizeof(*p).
+ *    </li>
+ *  </ul>
+ *
+ * If your class has derived classes, then:
+ *  <ol>
+ *    <li>
+ *      \c alloc_size() must be virtual.
+ *    </li>
+ *    <li>
+ *      Derived classes must also implement of \c alloc_size()
+ *      and they must call their base class's implementations.
+ *    </li>
+ *  </ol>
+ * For example:
+ * \code
+ *  class my_derived_class : public my_class {
+ *  public:
+ *    // ...
+ *    size_t alloc_size() const {
+ *      return my_class::alloc_size() + alloc_size( u );
+ *    }
+ *  private:
+ *    std::string u;
+ *  };
+ * \endcode
+ */
+
+///////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Tests the given type to see if it's a class that has a member function
+ * named \c alloc_size having the signature:
+ * \code
+ *  size_t (T::*)() const
+ * \endcode
+ * @tparam T The type to test.
+ */
+template<typename T>
+class has_alloc_size : zorba::internal::ztd::sfinae_base {
+  template<typename SignatureType,SignatureType> struct type_check;
+
+  template<typename U>
+  static yes& test( type_check<size_t (U::*)() const,&U::alloc_size>* );
+
+  template<typename U>
+  static no& test( ... );
+
+public:
+  static bool const value = sizeof( test<T>(0) ) == sizeof( yes );
+};
+
+/**
+ * Size traits for classes that have an \c alloc_size() member function having
+ * the signature:
+ * \code
+ *  size_t (T::*)() const
+ * \endcode
+ *
+ * @tparam T The type to use.
+ */
+template<typename T,bool = has_alloc_size<T>::value>
+struct size_traits {
+  static size_t alloc_sizeof( T const &t ) {
+    return t.alloc_size();
+  }
+};
+
+/**
+ * Specialization for any non-pointer type.
+ * @tparam T The type to use.
+ */
+template<typename T>
+struct size_traits<T,false> {
+  static size_t alloc_sizeof( T const& ) {
+    return 0;
+  }
+};
+
+/**
+ * Gets the size of all the object's additional data.
+ * It does \e not include the size of the object itself.
+ *
+ * @tparam T The type to get the data size of.
+ * @param t An instance of \a T to get the data size of.
+ */
+template<typename T>
+inline size_t alloc_sizeof( T const &t ) {
+  return size_traits<T>::alloc_sizeof( t );
+}
+
+/**
+ * Gets the total memory size of an object.
+ *
+ * @tparam T The type to get the memory size of.
+ * @param t An instance of \a T to get the memory size of.
+ */
+template<typename T>
+inline size_t mem_sizeof( T const &t ) {
+  return sizeof( T ) + alloc_sizeof( t );
+}
+
+/**
+ * Specialization for pointer types.
+ * @tparam T A pointer type.
+ */
+template<typename T>
+struct size_traits<T*,false> {
+  static size_t alloc_sizeof( T *p ) {
+    return p ? mem_sizeof( *p ) : 0;
+  }
+};
+
+/**
+ * Specialization for <code>char const*</code>.
+ */
+template<>
+struct size_traits<char const*> {
+  static size_t alloc_sizeof( char const *s ) {
+    return s ? std::strlen( s ) : 0;
+  }
+};
+
+////////// C++ Specializations ////////////////////////////////////////////////
+
+/**
+ * Specialization for std::string.
+ */
+template<>
+struct size_traits<std::string> {
+  static size_t alloc_sizeof( std::string const &s ) {
+    return s.size();
+  }
+};
+
+/**
+ * Size traits for a MapType.
+ * (This is a base class used by other specializations.)
+ */
+template<class MapType>
+struct map_size_traits {
+  static size_t alloc_sizeof( MapType const &m ) {
+    size_t const padding =
+        sizeof( typename MapType::value_type )
+      - sizeof( typename MapType::key_type )
+      - sizeof( typename MapType::mapped_type );
+    size_t total_size = m.size() * padding;
+    FOR_EACH( typename MapType, i, m )
+      total_size += mem_sizeof( i->first ) + mem_sizeof( i->second );
+    return total_size;
+  }
+};
+
+/**
+ * Specialization for std::map.
+ */
+template<typename K,typename V,typename Comp,class Alloc>
+struct size_traits<std::map<K,V,Comp,Alloc>,false> :
+  map_size_traits< std::map<K,V,Comp,Alloc> >
+{
+};
+
+/**
+ * Specialization for std::unordered_map.
+ */
+template<typename K,typename V,class Hash,class Equal,class Alloc>
+struct size_traits<std::unordered_map<K,V,Hash,Equal,Alloc>,false> :
+  map_size_traits< std::unordered_map<K,V,Hash,Equal,Alloc> >
+{
+};
+
+/**
+ * Size traits for a SequenceType.
+ * (This is a base class used by other specializations.)
+ */
+template<class SequenceType>
+struct sequence_size_traits {
+  static size_t alloc_sizeof( SequenceType const &s ) {
+    size_t total_size = 0;
+    FOR_EACH( typename SequenceType, i, s )
+      total_size += mem_sizeof( *i );
+    return total_size;
+  }
+};
+
+/**
+ * Specialization for std::set.
+ */
+template<typename T,class Comp,class Alloc>
+struct size_traits<std::set<T,Comp,Alloc>,false> :
+  sequence_size_traits< std::set<T,Comp,Alloc> >
+{
+};
+
+/**
+ * Specialization for std::stack.
+ */
+template<typename T,class Container>
+struct size_traits<std::stack<T,Container>,false> :
+  sequence_size_traits< std::stack<T,Container> >
+{
+};
+
+/**
+ * Specialization for std::unique_ptr.
+ */
+template<typename T,class D>
+struct size_traits<std::unique_ptr<T,D>,false> {
+  static size_t alloc_sizeof( std::unique_ptr<T,D> const &p ) {
+    return p ? mem_sizeof( *p ) : 0;
+  }
+};
+
+/**
+ * Specialization for std::unordered_set.
+ */
+template<typename K,class Hash,class Equal,class Alloc>
+struct size_traits<std::unordered_set<K,Hash,Equal,Alloc>,false> :
+  sequence_size_traits< std::unordered_set<K,Hash,Equal,Alloc> >
+{
+};
+
+/**
+ * Specialization for std::vector.
+ */
+template<typename T,class Alloc>
+struct size_traits<std::vector<T,Alloc>,false> :
+  sequence_size_traits< std::vector<T,Alloc> >
+{
+};
+
+////////// Zorba specializations //////////////////////////////////////////////
+
+/**
+ * Specialization for rstring.
+ */
+template<class RepType>
+struct size_traits<rstring<RepType>,false> {
+  static size_t alloc_sizeof( rstring<RepType> const &s ) {
+    return s.size();
+  }
+};
+
+/**
+ * Specialization for rchandle.
+ */
+template<class T>
+struct size_traits<rchandle<T>,false> {
+  static size_t alloc_sizeof( rchandle<T> const &h ) {
+    return h.get() ? mem_sizeof( *h ) : 0;
+  }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace ztd
+} // namespace zorba
+#endif /* ZORBA_MEM_SIZEOF */
+/* vim:set et sw=2 ts=2: */

=== modified file 'src/zorbatypes/decimal.cpp'
--- src/zorbatypes/decimal.cpp	2012-07-12 17:29:55 +0000
+++ src/zorbatypes/decimal.cpp	2012-07-13 00:33:23 +0000
@@ -368,6 +368,10 @@
 
 ////////// miscellaneous //////////////////////////////////////////////////////
 
+size_t Decimal::alloc_size() const {
+  return value_.significant_digits();
+}
+
 uint32_t Decimal::hash( value_type const &value ) {
   char buf[1024];
   char *bufp = value.exponent() + 3 > 1024 ?

=== modified file 'src/zorbatypes/decimal.h'
--- src/zorbatypes/decimal.h	2012-07-12 17:29:55 +0000
+++ src/zorbatypes/decimal.h	2012-07-13 00:33:23 +0000
@@ -211,6 +211,8 @@
 
   ////////// miscellaneous ////////////////////////////////////////////////////
 
+  size_t alloc_size() const;
+
   bool is_xs_int() const;
   bool is_xs_integer() const;
   bool is_xs_long() const;

=== modified file 'src/zorbatypes/integer.cpp'
--- src/zorbatypes/integer.cpp	2012-07-12 17:29:55 +0000
+++ src/zorbatypes/integer.cpp	2012-07-13 00:33:23 +0000
@@ -374,6 +374,10 @@
 #endif /* ZORBA_WITH_BIG_INTEGER */
 
 #ifdef ZORBA_WITH_BIG_INTEGER
+size_t IntegerImpl::alloc_size() const {
+  return value_.significant_digits();
+}
+
 uint32_t IntegerImpl::hash() const {
   return Decimal::hash( value_ );
 }

=== modified file 'src/zorbatypes/integer.h'
--- src/zorbatypes/integer.h	2012-07-12 17:29:55 +0000
+++ src/zorbatypes/integer.h	2012-07-13 00:33:23 +0000
@@ -116,9 +116,9 @@
 
   /////////////////////////////////////////////////////////////////////////////
 
-  #ifndef ZORBA_WITH_BIG_INTEGER
+#ifndef ZORBA_WITH_BIG_INTEGER
   IntType& get_value() { return value_; }
-  #endif
+#endif /* ZORBA_WITH_BIG_INTEGER */
 
   ////////// assignment operators /////////////////////////////////////////////
 
@@ -445,6 +445,9 @@
 
   ////////// miscellaneous ////////////////////////////////////////////////////
 
+#ifdef ZORBA_WITH_BIG_INTEGER
+  size_t alloc_size() const;
+#endif /* ZORBA_WITH_BIG_INTEGER */
   int compare( IntegerImpl const& ) const;
   uint32_t hash() const;
   bool is_cxx_long() const;