graphite-dev team mailing list archive

[Sidnei da Silva <sidnei.da.silva@xxxxxxxxxxxxx>]

Sidnei da Silva has proposed merging lp:~sidnei/graphite/refactor-parse-retention into lp:graphite.

Requested reviews:
  graphite-dev (graphite-dev)

For more details, see:
https://code.launchpad.net/~sidnei/graphite/refactor-parse-retention/+merge/69289

Refactor parseRetentionDef into whisper module, so it can be reused by carbon.storage and whisper-create
-- 
https://code.launchpad.net/~sidnei/graphite/refactor-parse-retention/+merge/69289
Your team graphite-dev is requested to review the proposed merge of lp:~sidnei/graphite/refactor-parse-retention into lp:graphite.

=== modified file 'carbon/lib/carbon/storage.py'
--- carbon/lib/carbon/storage.py	2011-07-10 22:10:06 +0000
+++ carbon/lib/carbon/storage.py	2011-07-26 14:37:24 +0000
@@ -13,6 +13,8 @@
 limitations under the License."""
 
 import os, re
+import whisper
+
 from os.path import join, exists
 from carbon.conf import OrderedConfigParser, settings
 
@@ -30,46 +32,6 @@
   return join(settings.LOCAL_DATA_DIR, metric.replace('.','/')) + '.wsp'
 
 
-UnitMultipliers = {
-  's' : 1,
-  'm' : 60,
-  'h' : 60 * 60,
-  'd' : 60 * 60 * 24,
-  'y' : 60 * 60 * 24 * 365,
-}
-
-def parseRetentionDefinition(retentionDef):
-  (precision, points) = retentionDef.strip().split(':')
-
-  if precision.isdigit():
-    precisionUnit = 's'
-    precision = int(precision)
-  else:
-    precisionUnit = precision[-1]
-    precision = int( precision[:-1] )
-
-  if points.isdigit():
-    pointsUnit = None
-    points = int(points)
-  else:
-    pointsUnit = points[-1]
-    points = int( points[:-1] )
-
-  if precisionUnit not in UnitMultipliers:
-    raise ValueError("Invalid unit: '%s'" % precisionUnit)
-
-  if pointsUnit not in UnitMultipliers and pointsUnit is not None:
-    raise ValueError("Invalid unit: '%s'" % pointsUnit)
-
-  precision = precision * UnitMultipliers[precisionUnit]
-
-  if pointsUnit:
-    points = points * UnitMultipliers[pointsUnit] / precision
-
-  return (precision, points)
-
-
-
 class Schema:
   def test(self, metric):
     raise NotImplementedError()
@@ -79,6 +41,7 @@
 
 
 class DefaultSchema(Schema):
+
   def __init__(self, name, archives):
     self.name = name
     self.archives = archives
@@ -88,6 +51,7 @@
 
 
 class PatternSchema(Schema):
+
   def __init__(self, name, pattern, archives):
     self.name = name
     self.pattern = pattern
@@ -99,6 +63,7 @@
 
 
 class ListSchema(Schema):
+
   def __init__(self, name, listName, archives):
     self.name = name
     self.listName = listName
@@ -130,9 +95,10 @@
 
 
 class Archive:
+
   def __init__(self,secondsPerPoint,points):
-    self.secondsPerPoint = int( secondsPerPoint )
-    self.points = int( points )
+    self.secondsPerPoint = int(secondsPerPoint)
+    self.points = int(points)
 
 
   def getTuple(self):
@@ -141,7 +107,7 @@
 
   @staticmethod
   def fromString(retentionDef):
-    (secondsPerPoint, points) = parseRetentionDefinition(retentionDef)
+    (secondsPerPoint, points) = whisper.parseRetentionDef(retentionDef)
     return Archive(secondsPerPoint, points)
 
 
@@ -172,9 +138,9 @@
     else:
       raise ValueError('schema "%s" has no pattern or list parameter configured' % section)
 
-    schemaList.append( mySchema )
+    schemaList.append(mySchema)
 
-  schemaList.append( defaultSchema )
+  schemaList.append(defaultSchema)
   return schemaList
 
 

=== modified file 'whisper/bin/whisper-create.py'
--- whisper/bin/whisper-create.py	2009-11-02 04:19:45 +0000
+++ whisper/bin/whisper-create.py	2011-07-26 14:37:24 +0000
@@ -4,7 +4,8 @@
 import whisper
 from optparse import OptionParser
 
-option_parser = OptionParser(usage='''%prog path secondsPerPoint:pointsToStore [secondsPerPoint:pointsToStore]* ''')
+option_parser = OptionParser(usage=('%prog path secondsPerPoint:pointsToStore '
+                                    '[secondsPerPoint:pointsToStore]*'))
 option_parser.add_option('--xFilesFactor', default=0.5, type='float')
 option_parser.add_option('--overwrite', default=False, action='store_true')
 
@@ -15,7 +16,8 @@
   sys.exit(1)
 
 path = args[0]
-archives = [ tuple( map(int,archive_str.split(':')) ) for archive_str in args[1:] ]
+archives = [whisper.parseRetentionDef(retentionDef)
+            for retentionDef in args[1:]]
 
 if options.overwrite and os.path.exists(path):
   print 'Overwriting existing file: %s' % path

=== modified file 'whisper/bin/whisper-fetch.py'
--- whisper/bin/whisper-fetch.py	2009-11-02 04:19:45 +0000
+++ whisper/bin/whisper-fetch.py	2011-07-26 14:37:24 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import sys, os, time
+import sys, time
 import whisper
 from optparse import OptionParser
 
@@ -9,7 +9,8 @@
 
 option_parser = OptionParser(usage='''%prog [options] path''')
 option_parser.add_option('--from', default=yesterday, type='int', dest='_from',
-  help="Unix epoch time of the beginning of your requested interval (default: 24 hours ago)")
+  help=("Unix epoch time of the beginning of "
+        "your requested interval (default: 24 hours ago)"))
 option_parser.add_option('--until', default=now, type='int',
   help="Unix epoch time of the end of your requested interval (default: now)")
 option_parser.add_option('--json', default=False, action='store_true',

=== modified file 'whisper/bin/whisper-resize.py'
--- whisper/bin/whisper-resize.py	2011-04-01 16:35:21 +0000
+++ whisper/bin/whisper-resize.py	2011-07-26 14:37:24 +0000
@@ -4,49 +4,10 @@
 import whisper
 from optparse import OptionParser
 
-now = int( time.time() )
-
-UnitMultipliers = {
-  's' : 1,
-  'm' : 60,
-  'h' : 60 * 60,
-  'd' : 60 * 60 * 24,
-  'y' : 60 * 60 * 24 * 365,
-}
-
-
-def parseRetentionDef(retentionDef):
-  (precision, points) = retentionDef.strip().split(':')
-
-  if precision.isdigit():
-    precisionUnit = 's'
-    precision = int(precision)
-  else:
-    precisionUnit = precision[-1]
-    precision = int( precision[:-1] )
-
-  if points.isdigit():
-    pointsUnit = None
-    points = int(points)
-  else:
-    pointsUnit = points[-1]
-    points = int( points[:-1] )
-
-  if precisionUnit not in UnitMultipliers:
-    raise ValueError("Invalid unit: '%s'" % precisionUnit)
-
-  if pointsUnit not in UnitMultipliers and pointsUnit is not None:
-    raise ValueError("Invalid unit: '%s'" % pointsUnit)
-
-  precision = precision * UnitMultipliers[precisionUnit]
-
-  if pointsUnit:
-    points = points * UnitMultipliers[pointsUnit] / precision
-
-  return (precision, points)
-
-
-option_parser = OptionParser(usage='''%prog path precision:retention [precision:retention]*
+now = int(time.time())
+
+option_parser = OptionParser(
+    usage='''%prog path precision:retention [precision:retention]*
 
 precision and retention specify lengths of time, for example:
 
@@ -55,10 +16,19 @@
 1h:7d        1 hour per datapoint, 7 days of retention
 12h:2y       12 hours per datapoint, 2 years of retention
 ''')
-option_parser.add_option('--xFilesFactor', default=None, type='float', help="Change the xFilesFactor")
-option_parser.add_option('--force', default=False, action='store_true', help="Perform a destructive change")
-option_parser.add_option('--newfile', default=None, action='store', help="Create a new database file without removing the existing one")
-option_parser.add_option('--nobackup', action='store_true', help='Delete the .bak file after successful execution')
+
+option_parser.add_option(
+    '--xFilesFactor', default=None,
+    type='float', help="Change the xFilesFactor")
+option_parser.add_option(
+    '--force', default=False, action='store_true',
+    help="Perform a destructive change")
+option_parser.add_option(
+    '--newfile', default=None, action='store',
+    help="Create a new database file without removing the existing one")
+option_parser.add_option(
+    '--nobackup', action='store_true',
+    help='Delete the .bak file after successful execution')
 
 (options, args) = option_parser.parse_args()
 
@@ -67,11 +37,13 @@
   sys.exit(1)
 
 path = args[0]
-new_archives = [ parseRetentionDef(retentionDef) for retentionDef in args[1:] ]
+new_archives = [whisper.parseRetentionDef(retentionDef)
+                for retentionDef in args[1:]]
 
 info = whisper.info(path)
 old_archives = info['archives']
-old_archives.sort(key=lambda a: a['secondsPerPoint'], reverse=True) #sort by precision, lowest to highest
+# sort by precision, lowest to highest
+old_archives.sort(key=lambda a: a['secondsPerPoint'], reverse=True)
 
 if options.xFilesFactor is None:
   xff = info['xFilesFactor']

=== modified file 'whisper/bin/whisper-update.py'
--- whisper/bin/whisper-update.py	2009-11-02 04:19:45 +0000
+++ whisper/bin/whisper-update.py	2011-07-26 14:37:24 +0000
@@ -1,12 +1,13 @@
 #!/usr/bin/env python
 
-import sys, os, time
+import sys, time
 import whisper
 from optparse import OptionParser
 
 now = int( time.time() )
 
-option_parser = OptionParser(usage='''%prog [options] path timestamp:value [timestamp:value]*''')
+option_parser = OptionParser(
+    usage='''%prog [options] path timestamp:value [timestamp:value]*''')
 
 (options, args) = option_parser.parse_args()
 
@@ -16,7 +17,8 @@
 
 path = args[0]
 datapoint_strings = args[1:]
-datapoint_strings = [point.replace('N:', '%d:' % now) for point in datapoint_strings]
+datapoint_strings = [point.replace('N:', '%d:' % now)
+                     for point in datapoint_strings]
 datapoints = [tuple(point.split(':')) for point in datapoint_strings]
 
 if len(datapoints) == 1:

=== modified file 'whisper/whisper.py'
--- whisper/whisper.py	2011-04-01 19:49:53 +0000
+++ whisper/whisper.py	2011-07-26 14:37:24 +0000
@@ -56,6 +56,45 @@
 
 debug = startBlock = endBlock = lambda *a,**k: None
 
+UnitMultipliers = {
+  's' : 1,
+  'm' : 60,
+  'h' : 60 * 60,
+  'd' : 60 * 60 * 24,
+  'y' : 60 * 60 * 24 * 365,
+}
+
+
+def parseRetentionDef(retentionDef):
+  (precision, points) = retentionDef.strip().split(':')
+
+  if precision.isdigit():
+    precisionUnit = 's'
+    precision = int(precision)
+  else:
+    precisionUnit = precision[-1]
+    precision = int( precision[:-1] )
+
+  if points.isdigit():
+    pointsUnit = None
+    points = int(points)
+  else:
+    pointsUnit = points[-1]
+    points = int( points[:-1] )
+
+  if precisionUnit not in UnitMultipliers:
+    raise ValueError("Invalid unit: '%s'" % precisionUnit)
+
+  if pointsUnit not in UnitMultipliers and pointsUnit is not None:
+    raise ValueError("Invalid unit: '%s'" % pointsUnit)
+
+  precision = precision * UnitMultipliers[precisionUnit]
+
+  if pointsUnit:
+    points = points * UnitMultipliers[pointsUnit] / precision
+
+  return (precision, points)
+
 
 class WhisperException(Exception):
     """Base class for whisper exceptions."""