handle debomatic status with new pre_chroot and post_chroot hooks. refactoried code

debomatic-modules/JSONLogger.py

@@ -17,47 +17,75 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
 #
-# Log information about building packages in a JSON format.
+# Log information about debomatic status in a JSON format.
+#
+# If you want to config this module, please add this info
+# to your debomatic.conf
+# [jsonlogger]
+# jsonfile = /path/to/jsonfile.log
 
 import os
+from time import time
 from json import dumps as toJSON
 
 
 class DebomaticModule_JSONLogger:
 
     def __init__(self):
-        self.logfile = '/var/log/debomatic-json.log'
+        self.jsonfile = '/var/log/debomatic-json.log'
+
+    def _set_jsonfile(self, args):
+        """If debomatic config file has section [jsonlogger] try to get 
+        'jsonfile' option and override the default value."""
+        if 'opts' in args and args['opts'].has_section('jsonlogger'):
+            self.jsonfile = args['opts'].get('jsonlogger', 'jsonfile').strip()
 
-    def _set_logfile(self, args):
-        if args['opts'].has_section('jsonlogger'):
-            self.logfile = args['opts'].get('jsonlogger', 'jsonfile').strip()
+    def _append_info_to_logfile(self, args, info):
+        """Write info to jsonfile converted in JSON format."""
+        self._set_jsonfile(args)
+        info['time'] = int(time())
+        with open(self.jsonfile, 'a') as logfd:
+            json = toJSON(info)
+            logfd.write(json + '\n')
 
-    def _get_info(self, args):
+    def _get_distribution_info(self, args):
+        """From args to distribution info."""
+        info = {}
+        info['status'] = args['cmd']
+        info['distribution'] = args['distribution']
+        if 'success' in args:
+            info['success'] = args['success']
+        return info
+
+    def _get_package_info(self, args):
+        """From args to package info."""
         keys = ['package', 'distribution', 'uploader']
         info = {}
         for k in keys:
-            if args.has_key(k): 
+            if k in args: 
                 info[k] = args[k]
         return info
 
+    def pre_chroot(self, args):
+        distribution = self._get_distribution_info(args)
+        self._append_info_to_logfile(args, distribution)
+
+    def post_chroot(self, args):
+        distribution = self._get_distribution_info(args)
+        self._append_info_to_logfile(args, distribution)
+
     def pre_build(self, args):
-        self._set_logfile(args)
-        with open(self.logfile, 'a') as fd:
-            package = self._get_info(args)
-            package['status'] = 'building'
-            json = toJSON(package)
-            fd.write(json + '\n')
+        package = self._get_package_info(args)
+        package['status'] = 'build'
+        self._append_info_to_logfile(args, package)
 
     def post_build(self, args):
-        self._set_logfile(args)
-        with open(self.logfile, 'a') as fd:
-            status = 'build-failed'
-            package = self._get_info(args)
-            resultdir = os.path.join(args['directory'], 'pool', args['package'])
-            for filename in os.listdir(resultdir):
-                if filename.endswith('.dsc'):
-                    status = 'build-successed'
-                    break
-            package['status'] = status
-            json = toJSON(package)
-            fd.write(json + '\n')
+        package = self._get_package_info(args)
+        package['status'] = 'build'
+        package['success'] = False
+        resultdir = os.path.join(args['directory'], 'pool', args['package'])
+        for filename in os.listdir(resultdir):
+            if filename.endswith('.dsc'):
+                package['success'] = True
+                break
+        self._append_info_to_logfile(args, package)

debomatic-webui/debomatic-webui

@@ -59,15 +59,14 @@ server.listen(config.port, config.host, null, function(err){
   }
 
   // statuses
-  var status = {}
-  status.packages = []
+  var status = []
 
   var broadcast = new Broadcaster(io.sockets, status)
 
   io.sockets.on('connection', function(socket) {
     var client = new Client(socket)
     client.start()
-    if (status.packages.length > 0)
+    if (status.length > 0)
       client.send_status(status)
   });
 

debomatic-webui/lib/broadcaster.js

@@ -2,9 +2,24 @@ var config = require('./config.js')
   , fs = require('fs')
   , Tail = require('./tail.js')
 
+function __watch_status_check_same_obj (obj1, obj2) {
+  if (obj1.status == obj2.status) {
+    if (obj1.distribution == obj2.distribution) {
+      if (obj1.hasOwnProperty('package') && 
+          obj2.hasOwnProperty('package')) 
+      {
+        if (obj1.package == obj2.package)
+          return true;
+        return false
+      }
+      return true
+    }
+  }
+  return false
+}
 
 // watcher on build_status
-function __watch_build_status (socket, status) {
+function __watch_status (socket, status) {
 
   var watcher = new Tail(config.debomatic.jsonfile)
   watcher.on('line', function(new_content) {
@@ -12,39 +27,24 @@ function __watch_build_status (socket, status) {
     try {
       data = JSON.parse(new_content)
     } catch (err) {
-      utils.errors_handler('Broadcaster:__watch_build_status:JSON.parse(new_content) - ', err, socket)
+      utils.errors_handler('Broadcaster:__watch_status:JSON.parse(new_content) - ', err, socket)
       return
     }
-
-    // looking for same package already in status.packages
-    var index = -1
-    for(i = 0; i < status.packages.length; i++)
-    {
-      p = status.packages[i]
-      if ( p.package == data.package
-        && p.distribution == data.distribution )
-      {
-        index = i
-        break
-      }
-    }
-
-    if (data.status == config.status.package.building) {
-      if (index == -1) { // not found in status.packages
-        status.packages.push(data)
-        socket.emit(config.events.broadcast.status_update, data)
-        return
+    // looking for same status already in statuses lists
+    if (data.hasOwnProperty('success')) {
+      for(i = 0; i < status.length; i++) {
+        if (__watch_status_check_same_obj(data, status[i])) {
+          status.splice(i, 1)
+          break;
+        }
+        else
+          continue;
       }
     }
-
-    if (data.status == config.status.package.successed
-      || data.status == config.status.package.failed ) 
-    {
-      if (index >= 0) { // found in status.packages - remove
-        status.packages.splice(index, 1)
-      }
-      socket.emit(config.events.broadcast.status_update, data)
+    else {
+      status.push(data)
     }
+    socket.emit(config.events.broadcast.status_update, data)
   })
   watcher.on('error', function(msg) {
     socket.emit(config.events.error, msg)
@@ -65,7 +65,7 @@ function Broadcaster (sockets, status) {
 
   var sockets = sockets
 
-  __watch_build_status(sockets, status)
+  __watch_status(sockets, status)
 
   __watch_distributions(sockets)
 

debomatic-webui/lib/client.js

@@ -52,6 +52,7 @@ function __send_package_status(socket, data, package_data) {
   new_data.package = package_data
 
   var status_data = {}
+  status_data.status = config.status.build
   status_data.distribution = data.distribution.name
   status_data.package = package_data.orig_name
 
@@ -62,9 +63,9 @@ function __send_package_status(socket, data, package_data) {
   //  + building: wc -l .datestamp == 1 (FIX_ME)
   //  + failed: else
   var base_path = path.join(package_path, package_data.orig_name)
-  fs.exists(base_path + '.dsc', function(changes_exists){
-    if (changes_exists) {
-      status_data.status = config.status.package.successed
+  fs.exists(base_path + '.dsc', function(dsc_exists){
+    if (dsc_exists) {
+      status_data.success = config.status.success
       socket.emit(event_name, status_data)
     }
     else {
@@ -80,10 +81,8 @@ function __send_package_status(socket, data, package_data) {
               if (chunk[i] == 10) count++;
             })
           .on('end', function() {
-            if (count <= 1)
-              status_data.status = config.status.package.building
-            else
-              status_data.status = config.status.package.failed
+            if (count > 1)
+              status_data.success = config.status.fail
             socket.emit(event_name, status_data)
           });
         }

debomatic-webui/lib/config.js

@@ -86,13 +86,13 @@ config.events.client.file = _event_get_set('file')
 config.events.client.file_newcontent = 'file_newcontent'
 config.events.client.status = 'status'
 
-// packages status according with JSONLogger.py module
+// debomatic status according with JSONLogger.py module
 config.status = {}
-config.status.package = {}
-config.status.package.building = 'building'
-config.status.package.failed = 'build-failed'
-config.status.package.successed = 'build-successed'
-
+config.status.build = 'build'
+config.status.create = 'create'
+config.status.update = 'update'
+config.status.success = true
+config.status.fail = false
 
 // read user configuration and merge it
 /*

debomatic-webui/public/javascripts/page_generic.js

@@ -2,20 +2,49 @@ function Page_Generic()
 {
   var _e = config.events
 
-  function __get_status_html(status_package) {
-    var s = status_package
+  function __get_status_html_id(status_data) {
+    var result = 'status-' + status_data.status + '-' + status_data.distribution
+    if (status_data.hasOwnProperty('package'))
+      result += '-' + status_data.package
+    return result
+  }
+
+  function __get_status_html_href(status_data) {
+    result = config.paths.distribution + '#' + status_data.distribution
+    if (status_data.hasOwnProperty('package'))
+      result += '/' + status_data.package.replace('_', '/') + '/datestamp'
+    return result
+  }
+
+  function __get_status_html_title(status_data) {
+    result = status_data.status + ': ' + status_data.distribution 
+    if (status_data.hasOwnProperty('package'))
+      result += ' > ' + status_data.package
+    if (status_data.hasOwnProperty('uploader') && status_data.uploader.length > 0)
+      result += ' by ' + status_data.uploader
+    return result 
+  }
+
+  function __get_status_html_inner(status_data) {
+    if (status_data.hasOwnProperty('package'))
+      return status_data.package;
+    return status_data.distribution
+  }
+
+  function __get_status_html(status_data) {
+    var _s = status_data
     var li = $('<li></li>')
-    li.attr('id', 'status-' + s.distribution + "-" + s.package)
+    li.attr('id', __get_status_html_id(status_data))
     var button = $('<a></a>')
     button.addClass('btn btn-xs')
-    button.addClass(s.status)
-    button.attr('title', s.status + ': ' + s.distribution + ' > ' + s.package)
-    button.attr('href', config.paths.distribution + '#' + s.distribution + '/' + s.package.replace('_', '/') + '/datestamp')
-    button.html(s.package.split('_')[0])
-    var info = Utils.get_status_icon_and_class(s)
+    button.addClass(_s.status)
+    button.attr('title', __get_status_html_title(_s))
+    button.attr('href', __get_status_html_href(_s))
+    button.html(__get_status_html_inner(_s))
+    var info = Utils.get_status_icon_and_class(_s)
     button.addClass('btn-' + info.className)
     // add icon
-    button.html(button.html() + ' ' + Utils.get_status_icon_html(s))
+    button.html(button.html() + ' ' + Utils.get_status_icon_html(_s))
     li.html(button)
     var result = $('<div></div>')
     result.html(li)
@@ -45,24 +74,25 @@ function Page_Generic()
   var status =  {
     set: function(data_status) {
       $("#status ul").html('')
-      if (data_status.packages.length > 0) {
-        data_status.packages.forEach(function(p){
-          status.append(p)
+      if (data_status.length > 0) {
+        data_status.forEach(function(s){
+          status.append(s)
         })
       }
     },
-    append: function(status_package) {
+    append: function(status_data) {
       $('#status .idle').hide()
-      $("#status ul").append(__get_status_html(status_package))
+      $("#status ul").append(__get_status_html(status_data))
     },
-    update: function(status_package) {
+    update: function(status_data) {
 
-      var li = $("#status li[id='status-" + status_package.distribution + "-" + status_package.package + "']")
+      var li = $("#status li[id='" + __get_status_html_id(status_data) + "']")
       if (li.length > 0
-        && status_package.status != config.status.package.building)
+        && status_data.hasOwnProperty('success'))
       {
         // Update color and icon
-        li.html($(__get_status_html(status_package)).children())
+        li = $(li[0])
+        li.html($(__get_status_html(status_data)).children())
         li.attr('id', '')
         // This is a chain to have a fadeOut and correctly
         // delete package from list.
@@ -82,8 +112,8 @@ function Page_Generic()
             }, config.status.delay.remove + 2000) // more delay on remove html
           }, config.status.delay.remove)
       }
-      else if (status_package.status == config.status.package.building) {
-        status.append(status_package)
+      else if (!status_data.hasOwnProperty('success')) {
+        status.append(status_data)
       }
     },
   }
@@ -121,9 +151,9 @@ function Page_Generic()
       distributions.set(socket_distributions)
     });
 
-    socket.on(_e.client.status, function(packages_status) {
-      debug_socket("received", _e.client.status, packages_status)
-      status.set(packages_status)
+    socket.on(_e.client.status, function(socket_status) {
+      debug_socket("received", _e.client.status, socket_status)
+      status.set(socket_status)
     })
 
     socket.on(_e.broadcast.status_update, function(package_status) {

debomatic-webui/public/javascripts/utils.js

@@ -56,17 +56,19 @@ var Utils = {
     var _s = status_data
     var className = null
     var icon = null
-    if (_s.status == config.status.package.building) {
-      className = _c.building
-      icon = _i.building
-    }
-    else if (_s.status == config.status.package.failed) {
-      className = _c.failed
-      icon = _i.failed
+    if (_s.hasOwnProperty('success')) {
+      if (_s.success == true) {
+        className = _c.success
+        icon = _i.success
+      }
+      else {
+        className = _c.fail
+        icon = _i.fail
+      }
     }
     else {
-      className = _c.successed
-      icon = _i.successed
+      className = _c[_s.status]
+      icon = _i[_s.status]
     }
     return {
       className: className,

debomatic-webui/views/footer.ejs

@@ -18,14 +18,18 @@
     var config = <%- JSON.stringify(web) %>
 
     config.status.className = {}
-    config.status.className.building = 'info'
-    config.status.className.successed = 'success'
-    config.status.className.failed = 'danger'
+    config.status.className.build = 'info'
+    config.status.className.create = 'warning'
+    config.status.className.update = 'default'
+    config.status.className.success = 'success'
+    config.status.className.fail = 'danger'
 
     config.status.icons = {}
-    config.status.icons.building = 'refresh'
-    config.status.icons.successed = 'ok'
-    config.status.icons.failed = 'remove'
+    config.status.icons.build = 'refresh'
+    config.status.icons.create = 'asterisk'
+    config.status.icons.update = 'upload'
+    config.status.icons.success = 'ok'
+    config.status.icons.fail = 'remove'
 
     config.status.delay = {}
     config.status.delay.remove = 10000