zeitgeist team mailing list archive

[Michal Hruby <michal.hruby@xxxxxxxxxxxxx>]

Michal Hruby has proposed merging lp:~mhr3/zeitgeist/fts-optimizations into lp:zeitgeist.

Requested reviews:
  Zeitgeist Framework Team (zeitgeist)

For more details, see:
https://code.launchpad.net/~mhr3/zeitgeist/fts-optimizations/+merge/99350

Make sure FTS doesn't do a single find_events call during reindex - use find_event_ids + get_events. This greatly reduces the mem usage when doing reindex.

Also make FTS quit when Zeitgeist quits.
-- 
https://code.launchpad.net/~mhr3/zeitgeist/fts-optimizations/+merge/99350
Your team Zeitgeist Framework Team is requested to review the proposed merge of lp:~mhr3/zeitgeist/fts-optimizations into lp:zeitgeist.

=== modified file 'extensions/fts++/controller.cpp'
--- extensions/fts++/controller.cpp	2012-02-14 16:56:04 +0000
+++ extensions/fts++/controller.cpp	2012-03-26 15:13:22 +0000
@@ -37,19 +37,21 @@
 void Controller::RebuildIndex ()
 {
   GError *error = NULL;
-  GPtrArray *events;
+  guint32 *event_ids;
+  gint event_ids_size;
   GPtrArray *templates = g_ptr_array_new ();
   ZeitgeistTimeRange *time_range = zeitgeist_time_range_new_anytime ();
 
   g_debug ("asking reader for all events");
-  events = zeitgeist_db_reader_find_events (zg_reader,
-                                            time_range,
-                                            templates,
-                                            ZEITGEIST_STORAGE_STATE_ANY,
-                                            0,
-                                            ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                                            NULL,
-                                            &error);
+  event_ids = zeitgeist_db_reader_find_event_ids (zg_reader,
+                                                  time_range,
+                                                  templates,
+                                                  ZEITGEIST_STORAGE_STATE_ANY,
+                                                  0,
+                                                  ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
+                                                  NULL,
+                                                  &event_ids_size,
+                                                  &error);
 
   if (error)
   {
@@ -58,10 +60,10 @@
   }
   else
   {
-    g_debug ("reader returned %u events", events->len);
+    g_debug ("reader returned %d events", event_ids_size);
 
-    IndexEvents (events);
-    g_ptr_array_unref (events);
+    IndexEvents (event_ids, event_ids_size);
+    g_free (event_ids);
 
     // Set the db metadata key only once we're done
     PushTask (new MetadataTask ("fts_index_version", INDEX_VERSION));
@@ -81,6 +83,19 @@
   }
 }
 
+void Controller::IndexEvents (guint *event_ids, int event_ids_size)
+{
+  const int CHUNK_SIZE = 64;
+  // Break down index tasks into suitable chunks
+  for (int i = 0; i < event_ids_size; i += CHUNK_SIZE)
+  {
+    int num_ids = i + CHUNK_SIZE > event_ids_size ? 
+      event_ids_size - i : CHUNK_SIZE;
+    PushTask (new IndexEventsTask (zg_reader,
+        std::vector<guint> (&event_ids[i], &event_ids[i + num_ids])));
+  }
+}
+
 void Controller::DeleteEvents (guint *event_ids, int event_ids_size)
 {
   // FIXME: Should we break the task here as well?

=== modified file 'extensions/fts++/controller.h'
--- extensions/fts++/controller.h	2012-02-14 16:56:04 +0000
+++ extensions/fts++/controller.h	2012-03-26 15:13:22 +0000
@@ -49,6 +49,7 @@
   void RebuildIndex ();
 
   void IndexEvents (GPtrArray *events);
+  void IndexEvents (guint *event_ids, int event_ids_size);
   void DeleteEvents (guint *event_ids, int event_ids_size);
 
   void PushTask (Task* task);

=== modified file 'extensions/fts++/task.cpp'
--- extensions/fts++/task.cpp	2012-02-14 16:56:04 +0000
+++ extensions/fts++/task.cpp	2012-03-26 15:13:22 +0000
@@ -23,10 +23,36 @@
 
 void IndexEventsTask::Process (Indexer *indexer)
 {
-  unsigned end_index = MIN (start_index + event_count, events->len);
-  for (unsigned i = start_index; i < end_index; i++)
-  {
-    indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (events, i));
+  if (events)
+  {
+    unsigned end_index = MIN (start_index + event_count, events->len);
+    for (unsigned i = start_index; i < end_index; i++)
+    {
+      indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (events, i));
+    }
+  }
+  else if (!event_ids.empty ())
+  {
+    GError *error = NULL;
+    GPtrArray *results = zeitgeist_db_reader_get_events (zg_reader,
+                                                         &event_ids[0],
+                                                         event_ids.size (),
+                                                         NULL,
+                                                         &error);
+    if (error)
+    {
+      g_warning ("Unable to get events: %s", error->message);
+      return;
+    }
+    else
+    {
+      for (unsigned i = 0; i < results->len; i++)
+      {
+        indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (results, i));
+      }
+    }
+
+    g_ptr_array_unref (results);
   }
 }
 

=== modified file 'extensions/fts++/task.h'
--- extensions/fts++/task.h	2012-02-14 16:56:04 +0000
+++ extensions/fts++/task.h	2012-03-26 15:13:22 +0000
@@ -49,16 +49,21 @@
 
   IndexEventsTask (GPtrArray *event_arr, unsigned index, unsigned count)
     : events (event_arr), start_index (index), event_count (count) {}
+  
+  IndexEventsTask (ZeitgeistDbReader *reader, std::vector<unsigned> const &ids)
+    : events (NULL), zg_reader (reader), event_ids (ids) {}
 
   virtual ~IndexEventsTask ()
   {
-    g_ptr_array_unref (events);
+    if (events) g_ptr_array_unref (events);
   }
 
 private:
   GPtrArray *events;
   unsigned start_index;
   unsigned event_count;
+  ZeitgeistDbReader *zg_reader;
+  std::vector<unsigned> event_ids;
 };
 
 class DeleteEventsTask : public Task

=== modified file 'extensions/fts++/zeitgeist-fts.vala'
--- extensions/fts++/zeitgeist-fts.vala	2012-02-10 17:03:50 +0000
+++ extensions/fts++/zeitgeist-fts.vala	2012-03-26 15:13:22 +0000
@@ -54,6 +54,7 @@
         private static FtsDaemon? instance;
         private static MainLoop mainloop;
         private static bool name_acquired = false;
+        private static bool zeitgeist_up = false;
 
         private DbReader engine;
         private Indexer indexer;
@@ -170,6 +171,16 @@
             }
         }
 
+        private static void zeitgeist_vanished ()
+        {
+            if (zeitgeist_up)
+            {
+                // client apis query us via zeitgeist, so quit if zg goes away
+                mainloop.quit ();
+            }
+            zeitgeist_up = false;
+        }
+
         static void run ()
             throws Error
         {
@@ -177,10 +188,7 @@
             var proxy = connection.get_proxy_sync<RemoteDBus> (
                 "org.freedesktop.DBus", "/org/freedesktop/DBus",
                 DBusProxyFlags.DO_NOT_LOAD_PROPERTIES);
-            bool zeitgeist_up = proxy.name_has_owner (ZEITGEIST_DBUS_NAME);
-            // FIXME: throw an error that zeitgeist isn't up? or just start it?
-            bool name_owned = proxy.name_has_owner (DBUS_NAME);
-            if (name_owned)
+            if (proxy.name_has_owner (DBUS_NAME))
             {
                 throw new EngineError.EXISTING_INSTANCE (
                     "The FTS daemon is running already.");
@@ -214,6 +222,10 @@
                 name_acquired_callback,
                 name_lost_callback);
 
+            Bus.watch_name (BusType.SESSION, ZEITGEIST_DBUS_NAME, 0,
+                () => { zeitgeist_up = true; },
+                zeitgeist_vanished);
+
             mainloop = new MainLoop ();
             mainloop.run ();