harvest-dev team mailing list archive

[Daniel Holbach <daniel.holbach@xxxxxxxxxx>]

Daniel Holbach has proposed merging lp:~dholbach/harvest/656221 into lp:harvest.

Requested reviews:
  harvest-dev (harvest-dev)
Related bugs:
  #656221 Add proper logging/locking for update jobs
  https://bugs.launchpad.net/bugs/656221

-- 
https://code.launchpad.net/~dholbach/harvest/656221/+merge/37866
Your team harvest-dev is requested to review the proposed merge of lp:~dholbach/harvest/656221 into lp:harvest.

=== modified file 'harvest/common/utils.py'
--- harvest/common/utils.py	2010-08-13 03:10:54 +0000
+++ harvest/common/utils.py	2010-10-07 15:39:44 +0000
@@ -1,6 +1,12 @@
+import subprocess
+import datetime
+import logging
+import logging.handlers
 import email
 import os
 
+from django.conf import settings
+
 def get_harvest_version(version_file, debug):
     """
     return the bzr revision number and version of Harvest
@@ -44,3 +50,39 @@
             version_str = f["versionname"]
     
     return version_str
+
+def instantiate_logger(job_name):
+    logger = logging.getLogger(job_name)
+    stamp_dir = os.path.join(settings.PROJECT_PATH, 'data')
+    log_file = os.path.join(stamp_dir, "%s.log" % job_name)
+    rfh = logging.handlers.RotatingFileHandler(log_file, maxBytes=100000, backupCount=5)
+    logger.addHandler(rfh)
+    logger.setLevel(settings.LOG_LEVEL)
+    return logger
+
+def run_job(which, interval, args=None):
+    logger = instantiate_logger(which)
+    stamp_dir = os.path.join(settings.PROJECT_PATH, 'data')
+    if not os.path.exists(stamp_dir):
+        os.makedirs(stamp_dir)
+    stamp_file = os.path.join(stamp_dir, "%s.stamp" % which)
+    lock_file = os.path.join(stamp_dir, "%s.lock" % which)
+    if not os.path.exists(stamp_file) or \
+       datetime.datetime.utcfromtimestamp(os.path.getmtime(stamp_file)) + interval < datetime.datetime.now():
+        if not os.path.exists(lock_file):
+            open(lock_file, "a").close()
+            pwd = os.getcwd()
+            os.chdir(settings.PROJECT_PATH)
+            p = subprocess.Popen(["./manage.py", which], stdout=subprocess.PIPE, 
+                                 stderr=subprocess.STDOUT, close_fds=True)
+            log = p.stdout.read()
+            p.communicate()
+            os.chdir(pwd)
+            open(stamp_file, "a").close()
+            os.utime(stamp_file, None)
+            if log.strip():
+                logger.error("STDOUT + STDERR Output\n%s" % log)
+            if os.path.exists(lock_file):
+                os.remove(lock_file)
+        else:
+            logger.error("%s exists, job might still be running." % lock_file)

=== modified file 'harvest/local_settings.py.sample'
--- harvest/local_settings.py.sample	2010-08-31 07:58:00 +0000
+++ harvest/local_settings.py.sample	2010-10-07 15:39:44 +0000
@@ -1,3 +1,7 @@
+import logging
+# for debugging set logging.DEBUG
+#LOG_LEVEL = logging.DEBUG
+
 DATABASE_ENGINE = 'sqlite3'            	# Change to postgresql_psycopg2 for postgres
 DATABASE_NAME = 'harvest.db'    	# sqlite3 file or database name
 #DATABASE_USER = ''
@@ -7,3 +11,4 @@
 ADMINS = (
 	('',''),
 )
+

=== modified file 'harvest/opportunities/management/commands/update.py'
--- harvest/opportunities/management/commands/update.py	2010-03-02 10:42:19 +0000
+++ harvest/opportunities/management/commands/update.py	2010-10-07 15:39:44 +0000
@@ -2,19 +2,15 @@
 
 from django.core.management.base import NoArgsCommand
 
-import subprocess
-import os
+from common.utils import run_job
 
-from django.conf import settings
+import datetime
 
 class Command(NoArgsCommand):
     help = "Update everything."
 
     def handle_noargs(self, **options):
-        pwd = os.getcwd()
-        os.chdir(settings.PROJECT_PATH)
-        subprocess.call(["./manage.py", "updatelists"])
-        subprocess.call(["./manage.py", "updateopportunities"])
-        subprocess.call(["./manage.py", "updatepackagesets"])
-        os.chdir(pwd)
+        run_job("updatelists", datetime.timedelta(minutes=20))
+        run_job("updateopportunities", datetime.timedelta(minutes=20))
+        run_job("updatepackagesets", datetime.timedelta(minutes=20))
 

=== modified file 'harvest/opportunities/management/commands/updatelists.py'
--- harvest/opportunities/management/commands/updatelists.py	2010-08-03 16:29:47 +0000
+++ harvest/opportunities/management/commands/updatelists.py	2010-10-07 15:39:44 +0000
@@ -7,6 +7,8 @@
 from common import opportunity_lists
 from opportunities.models import OpportunityList
 
+from common.utils import instantiate_logger
+
 import sys
 import os
 
@@ -14,14 +16,17 @@
     help = "Pull opportunity lists and update them in the DB."
 
     def update_lists(self):
+        logger = instantiate_logger(os.path.splitext(os.path.basename(__file__))[0])
         data_dir = os.path.join(settings.PROJECT_PATH, "data")
         if not os.path.exists(data_dir):
             os.makedirs(data_dir)
+        logger.debug("Pulling lists.")
         list_dir = opportunity_lists.pull_lists(data_dir)
 
         for (list_url, 
              list_description, 
              list_explanation) in opportunity_lists.read_lists(list_dir):
+            logger.debug("Adding/updating record for %s." % list_url)
             op_list, created = OpportunityList.objects.get_or_create(url=list_url)
             op_list.description = list_description
             if list_explanation:
@@ -29,6 +34,8 @@
             op_list.active = True
             op_list.name = self.chop_name(list_url)
             op_list.save()
+            logger.debug("Wrote record for %s." % op_list.name)
+        logger.debug("Completed.")
 
     def chop_name(self, list_url):
         filename = os.path.basename(list_url)

=== modified file 'harvest/opportunities/management/commands/updateopportunities.py'
--- harvest/opportunities/management/commands/updateopportunities.py	2010-03-02 16:04:00 +0000
+++ harvest/opportunities/management/commands/updateopportunities.py	2010-10-07 15:39:44 +0000
@@ -5,17 +5,22 @@
 from django.conf import settings
 
 from common import opportunity_lists
+from common.utils import instantiate_logger
 from opportunities import models
 
 import sys
+import os
 
 class Command(NoArgsCommand):
     help = "Pull opportunity lists and update them in the DB."
 
     def update_lists(self):
+        logger = instantiate_logger(os.path.splitext(os.path.basename(__file__))[0])
         for op_list in models.OpportunityList.objects.all():
+            logger.debug("Checking opportunity list %s" % op_list.url)
             (entries, new_date) = opportunity_lists.read_entries(op_list.url, op_list.last_updated)
             if entries:
+                logger.debug("Has entries and was updated at %s." % new_date)
                 op_list.last_updated = new_date
                 for (sourcepackage, url, description) in filter(lambda a: len(a)==3, entries):
                     sp, created = models.SourcePackage.objects.get_or_create(name=sourcepackage)
@@ -42,7 +47,8 @@
                     opportunity.experience = op_list.experience
                     opportunity.save()
             op_list.save()
-
+            logger.debug("%s updated." % op_list.url)
+        logger.debug("Completed.")
 
     def handle_noargs(self, **options):
         transaction.enter_transaction_management()

=== modified file 'harvest/opportunities/management/commands/updatepackagesets.py'
--- harvest/opportunities/management/commands/updatepackagesets.py	2010-09-01 11:24:57 +0000
+++ harvest/opportunities/management/commands/updatepackagesets.py	2010-10-07 15:39:44 +0000
@@ -3,31 +3,42 @@
 from django.core.management.base import NoArgsCommand
 
 from common import launchpad
+from common.utils import instantiate_logger
 from opportunities.models import SourcePackage, PackageSet
 
 import sys
+import os
 
 class Command(NoArgsCommand):
     help = "Pull packageset information from Launchpad."
 
     def handle_noargs(self, **options):
+        logger = instantiate_logger(os.path.splitext(os.path.basename(__file__))[0])
+        logger.debug("Logging into Launchpad.")
         lp = launchpad.lp_login()
         if not lp:
             sys.exit(1)
+        logger.debug("Succeed logging in. Getting packagesets.")
         lp_packagesets = launchpad.get_packagesets(lp)
+        logger.debug("Packagesets retrieved.")
         package_mapping = {}
+        logger.debug("Checking if packagesets need to be created in Harvest.")
         unseeded, created = PackageSet.objects.get_or_create(name="unseeded")
         if created:
+            logger.debug("'unseeded' packageset created.")
             unseeded.save()
         for lp_packageset in lp_packagesets:
             packageset, created = PackageSet.objects.get_or_create(name=lp_packageset.name)
             if created:
+                logger.debug("'%s' packageset created." % lp_packageset.name)
                 packageset.save()
+            logger.debug("Finding out which package is in %s." % lp_packageset.name)
             # record information which package is in which package sets
             for package_name in lp_packageset.getSourcesIncluded():
                 if not package_mapping.has_key(package_name):
                     package_mapping[package_name] = set()
                 package_mapping[package_name].add((lp_packageset.name))
+        logger.debug("Updating packageset information for all packages.")
         for package_name in package_mapping.keys():
             try:
                 package = SourcePackage.objects.get(name=package_name)
@@ -51,3 +62,4 @@
         for u in unseeded_packages:
             u.packagesets.add(unseeded)
             u.save()
+        logger.debug("Completed.")

=== modified file 'harvest/settings.py'
--- harvest/settings.py	2010-08-31 23:49:08 +0000
+++ harvest/settings.py	2010-10-07 15:39:44 +0000
@@ -1,3 +1,4 @@
+import logging
 import os
 PROJECT_PATH = os.path.dirname(os.path.abspath(__file__))
 
@@ -7,6 +8,7 @@
 STATIC_SERVE = True
 PROJECT_NAME = 'harvest'
 INTERNAL_IPS = ('127.0.0.1',) #for testing
+LOG_LEVEL = logging.ERROR
 
 try:
     import debug_toolbar