Merge remote-tracking branch 'github/master' into portable

History.md

-# 0.5.1 (2015-06-17)
+# 0.6.0 (2014-07-10)
+ * [new] [module] write a JSON file about package status in its own directory
+ * [new] update to socket.io 1.x
+ * [new] enable icons and details in static debomatic directory listening
+ * [new] get status of package via socket instead of making a GET request
+ * [new] add clean button to package search
+ * [new] double click on a package in the list make an automatic search
+ * [new] add slide up and down effect while searching packages
+ * [new] set files list to not send to client as configurable
+ * [fix] prevent crashes filtering out request to chroots - send back 403 HTTP status
+ * [fix] escape correctly HTML in file content
+ * [fix] auto-populate page on socket connect instead of on page loads
+ * [fix] preferences does not set correctly if value is false
+ * [fix] better and smaller error messages
+ * [fix] fix some style issues
+
+# 0.5.1 (2014-06-17)
  * [fix] recursive call on receive file new_content while view the whole file
  * [fix] improved style for file view and datestamp
  * [fix] documentation set porter command instead of rebuild

debomatic-modules/JSONLogger.py

@@ -27,6 +27,7 @@
 import os
 from time import time
 from json import dumps as toJSON
+from json import load as fileToJSON
 
 
 class DebomaticModule_JSONLogger:
@@ -34,58 +35,88 @@ class DebomaticModule_JSONLogger:
     def __init__(self):
         self.jsonfile = '/var/log/debomatic-json.log'
 
-    def _set_jsonfile(self, args):
+    def _set_json_logfile_name(self, args):
         """If debomatic config file has section [jsonlogger] try to get
         'jsonfile' option and override the default value."""
         if 'opts' in args and args['opts'].has_section('jsonlogger'):
             self.jsonfile = args['opts'].get('jsonlogger', 'jsonfile').strip()
 
-    def _append_info_to_logfile(self, args, info):
-        """Write info to jsonfile converted in JSON format."""
-        self._set_jsonfile(args)
-        info['time'] = int(time())
+    def _write_json_logfile(self, args, status):
+        """Write status to jsonfile in JSON format."""
+        self._set_json_logfile_name(args)
+        status['time'] = int(time())
         with open(self.jsonfile, 'a') as logfd:
-            json = toJSON(info)
+            json = toJSON(status)
             logfd.write(json + '\n')
 
-    def _get_distribution_info(self, args):
-        """From args to distribution info."""
-        info = {}
-        info['status'] = args['cmd']
-        info['distribution'] = args['distribution']
+    def _get_package_json(self, args):
+        """Get the path of package JSON file"""
+        return '%(directory)s/pool/%(package)s/%(package)s.json' % args
+
+    def _write_package_json(self, args, status):
+        """Write package status to a JSON file."""
+        package_json = self._get_package_json(args)
+        if os.path.isfile(package_json):
+            with open(package_json, 'r') as infofd:
+                try:
+                    info = fileToJSON(infofd)
+                    info['end'] = int(time())
+                except:
+                    return
+        else:
+            info = {'start': int(time())}
+
+        for key in status:
+            if key not in info:
+                info[key] = status[key]
+
+        with open(package_json, 'w') as infofd:
+            json = toJSON(info, indent=4)
+            infofd.write(json + '\n')
+
+    def _get_distribution_status(self, args):
+        """From args to distribution status"""
+        status = {}
+        status['status'] = args['cmd']
+        status['distribution'] = args['distribution']
         if 'success' in args:
-            info['success'] = args['success']
-        return info
+            status['success'] = args['success']
+        return status
 
-    def _get_package_info(self, args):
-        """From args to package info."""
+    def _get_package_status(self, args):
+        """From args to package status"""
         keys = ['package', 'distribution', 'uploader']
-        info = {}
+        status = {}
         for k in keys:
             if k in args:
-                info[k] = args[k]
-        return info
+                status[k] = args[k]
+        return status
 
     def pre_chroot(self, args):
-        distribution = self._get_distribution_info(args)
-        self._append_info_to_logfile(args, distribution)
+        distribution = self._get_distribution_status(args)
+        self._write_json_logfile(args, distribution)
 
     def post_chroot(self, args):
-        distribution = self._get_distribution_info(args)
-        self._append_info_to_logfile(args, distribution)
+        distribution = self._get_distribution_status(args)
+        self._write_json_logfile(args, distribution)
 
     def pre_build(self, args):
-        package = self._get_package_info(args)
+        package = self._get_package_status(args)
         package['status'] = 'build'
-        self._append_info_to_logfile(args, package)
+        package_json = self._get_package_json(args)
+        if os.path.isfile(package_json):
+            os.remove(package_json)
+        self._write_package_json(args, package)
+        self._write_json_logfile(args, package)
 
     def post_build(self, args):
-        package = self._get_package_info(args)
-        package['status'] = 'build'
-        package['success'] = False
+        status = self._get_package_status(args)
+        status['status'] = 'build'
+        status['success'] = False
         resultdir = os.path.join(args['directory'], 'pool', args['package'])
         for filename in os.listdir(resultdir):
             if filename.endswith('.dsc'):
-                package['success'] = True
+                status['success'] = True
                 break
-        self._append_info_to_logfile(args, package)
+        self._write_package_json(args, status)
+        self._write_json_logfile(args, status)

debomatic-webui/lib/client.js

@@ -18,6 +18,8 @@ function __get_files_list_from_package(data, callback) {
             file.orig_name = f;
             file.name = f.split('_')[0];
             file.extension = f.split('.').pop();
+            if (config.debomatic.excluded_files.indexOf(file.extension) >= 0)
+                return;
             if (file.extension == 'deb' || file.extension == 'ddeb' || file.extension == 'udeb') {
                 data.package.debs.push(file);
             } else if (f.indexOf('.tar') >= 0 || file.extension == 'changes' || file.extension == 'dsc') {
@@ -42,50 +44,35 @@ function __send_package_files_list(event_name, socket, data) {
     });
 }
 
-function __send_package_status(socket, data, package_data) {
-
-    var event_name = config.events.client.distribution_packages_status;
-
-    var new_data = {};
-    new_data.distribution = data.distribution;
-    new_data.package = package_data;
-
-    var status_data = {};
-    status_data.status = config.status.build;
-    status_data.distribution = data.distribution.name;
-    status_data.package = package_data.orig_name;
-
-    var package_path = utils.get_package_path(new_data);
-
-    //  status policy:
-    //  + successed: exists .dsc
-    //  + building: wc -l .datestamp == 1 (FIX_ME)
-    //  + failed: else
-    var base_path = path.join(package_path, package_data.orig_name);
-    fs.exists(base_path + '.dsc', function (dsc_exists) {
-        if (dsc_exists) {
-            status_data.success = config.status.success;
-            socket.emit(event_name, status_data);
-        } else {
-            // emulate wc -l .datestamp in nodejs
-            var count = 0;
-            var datestamp = base_path + '.datestamp';
-            fs.exists(datestamp, function (datestamp_exists) {
-                if (datestamp_exists) {
-                    // count lines
-                    fs.createReadStream(datestamp)
-                        .on('data', function (chunk) {
-                            for (var i = 0; i < chunk.length; ++i)
-                                if (chunk[i] == 10) count++;
-                        })
-                        .on('end', function () {
-                            if (count > 1)
-                                status_data.success = config.status.fail;
-                            socket.emit(event_name, status_data);
-                        });
-                }
-            });
+function __read_package_status(data, cb) {
+    var package_path = utils.get_package_path(data);
+    var package_json = path.join(package_path, data.package.orig_name + '.json');
+    fs.readFile(package_json, {
+        encoding: 'utf8'
+    }, function (err, content) {
+        if (err) {
+            utils.errors_handler('Client:__read_package_status:', err);
+            return;
         }
+        try {
+            content = JSON.parse(content);
+        } catch (parse_err) {
+            utils.errors_handler('Client:__read_package_status:parse_err:', parse_err);
+            return;
+        }
+        cb(content);
+    });
+}
+
+function __send_package_info(socket, data) {
+    __read_package_status(data, function (content) {
+        socket.emit(_e.package_info, content);
+    });
+}
+
+function __send_package_status(socket, data) {
+    __read_package_status(data, function (content) {
+        socket.emit(_e.distribution_packages_status, content);
     });
 }
 
@@ -99,8 +86,11 @@ function __send_distribution_packages(event_name, socket, data) {
             pack.name = info[0];
             pack.version = info[1];
             pack.orig_name = p;
+            __send_package_status(socket, {
+                distribution: data.distribution,
+                package: pack
+            });
             data.distribution.packages.push(pack);
-            __send_package_status(socket, data, pack);
         });
         socket.emit(event_name, data);
     });
@@ -162,6 +152,12 @@ function Client(socket) {
             __handler_get_file(socket, data);
         });
 
+        socket.on(_e.package_info, function (data) {
+            if (!utils.check_data_package(data))
+                return;
+            __send_package_info(socket, data);
+        });
+
 
         // on client disconnection close all watchers
         socket.on('disconnect', function () {

debomatic-webui/lib/config.js

@@ -68,7 +68,9 @@ config.web.preferences.debug = 0; // debug level - 0 means disabled
 
 // DO NOT TOUCH these ones
 
-config.version = '0.5.1';
+config.version = '0.6.0';
+
+config.debomatic.excluded_files = ['datestamp', 'json'];
 
 config.events = {};
 config.events.error = 'server-error';
@@ -81,6 +83,7 @@ config.events.client = {};
 config.events.client.distribution_packages = 'c.distribution_packages';
 config.events.client.distribution_packages_status = 'c.distribution_packages_status';
 config.events.client.package_files_list = 'c.package_files_list';
+config.events.client.package_info = 'c.package_info';
 config.events.client.file = 'c.file';
 config.events.client.file_newcontent = 'c.file_newcontent';
 config.events.client.status = 'c.status';

debomatic-webui/lib/utils.js

@@ -8,7 +8,7 @@ var path = require('path'),
 function __errors_handler(from, err, socket) {
     if (!socket)
         from = 'NO SOCKET: ' + from;
-    console.error(from, err);
+    console.error(from, err.message);
     if (socket)
         socket.emit(config.events.error, err.message);
 }

debomatic-webui/public/javascripts/page_distribution.js

@@ -317,27 +317,6 @@ function Page_Distrubion(socket) {
                 query_data.package = view.package;
                 debug_socket('emit', _e.package_files_list, query_data);
                 socket.emit(_e.package_files_list, query_data);
-                files.get_datestamp();
-            }
-        },
-        get_datestamp: function (socket_data) {
-            if (Utils.check_view_package(view)) {
-                if (socket_data && socket_data.package != view.package.orig_name)
-                    return;
-                var url = config.paths.debomatic + '/' +
-                    view.distribution.name + '/pool/' +
-                    view.package.orig_name + '/' +
-                    view.package.orig_name + '.datestamp';
-                debug(2, 'getting datestamp');
-                $.get(url, function (data) {
-                    data = data.replace(/ (\d+:\d+(:\d+)?)$/mg, ' <b>$1</b>');
-                    data = data.replace('Build finished', 'finished');
-                    data = data.replace('Elapsed', 'elapsed');
-                    data = data.replace(/\n$/g, '');
-                    data = data.replace(/\n/g, ' - ');
-                    data = data.replace(/at /g, '');
-                    $("#file .datestamp").html(data);
-                });
             }
         },
         select: function () {
@@ -358,6 +337,56 @@ function Page_Distrubion(socket) {
         },
     };
 
+    var package_info = {
+        set: function (socket_data) {
+            if (!Utils.check_view_package(view) || socket_data.package != view.package.orig_name) {
+                return;
+            }
+
+            function _get_time(timestamp) {
+                var date = new Date(timestamp * 1000);
+                var locale = navigator.language || 'en-US';
+                var options = {
+                    weekday: "long",
+                    year: "numeric",
+                    month: "long",
+                    day: "numeric",
+                };
+                var result = date.toLocaleDateString(locale, options);
+                result += ' <b>' + date.getHours() + ':' + date.getMinutes() + '</b>';
+                return result;
+            }
+
+            var info = "";
+
+            if (socket_data.uploader)
+                info += "Uploaded by " + socket_data.uploader + ' - ';
+
+            info += "Build started " + _get_time(socket_data.start);
+
+            if (socket_data.end) {
+                info += ' - finished ' + _get_time(socket_data.end);
+                info += ' - elapsed time: <b>';
+                var elapsed = new Date((socket_data.end - socket_data.start) * 1000);
+                info += ("0" + elapsed.getUTCHours()).slice(-2) + ':';
+                info += ("0" + elapsed.getUTCMinutes()).slice(-2) + ':';
+                info += ("0" + elapsed.getUTCSeconds()).slice(-2);
+            }
+            $("#package_info").html(info);
+
+        },
+        get: function () {
+            if (Utils.check_view_package(view)) {
+                var query_data = {};
+                query_data.distribution = view.distribution;
+                query_data.package = view.package;
+                debug_socket('emit', _e.package_info, query_data);
+                socket.emit(_e.package_info, query_data);
+                $("#package_info").html('...');
+            }
+        }
+    };
+
     var file = {
         set: function (socket_data) {
             var new_content = Utils.escape_html(socket_data.file.content);
@@ -628,6 +657,7 @@ function Page_Distrubion(socket) {
                 return;
             } else if (!Utils.check_view_package(old_view) || !Utils.check_view_package(view) ||
                 view.package.orig_name != old_view.package.orig_name) { // new package view
+                package_info.get();
                 files.get();
                 file.get();
             } else if (!Utils.check_view_file(old_view) || !Utils.check_view_file(view) ||
@@ -650,6 +680,7 @@ function Page_Distrubion(socket) {
     var populate = function () {
         clean();
         packages.get();
+        package_info.get();
         files.get();
         file.get();
         update.view();
@@ -687,7 +718,9 @@ function Page_Distrubion(socket) {
         socket.on(config.events.broadcast.status_update, function (socket_data) {
             packages.set_status(socket_data);
             sticky.set_status(socket_data);
-            files.get_datestamp(socket_data);
+            if (socket_data.distribution == view.distribution.name && socket_data.package == view.package.orig_name) {
+                package_info.get();
+            }
         });
 
         socket.on(_e.package_files_list, function (socket_data) {
@@ -705,6 +738,11 @@ function Page_Distrubion(socket) {
             new_lines.push(socket_data.file.new_content);
         });
 
+        socket.on(_e.package_info, function (socket_data) {
+            debug_socket('received', _e.package_info, socket_data);
+            package_info.set(socket_data);
+        });
+
         $(window).on('hashchange', function () {
             if (!__check_hash_makes_sense())
                 return;

debomatic-webui/public/stylesheets/style.css

@@ -169,10 +169,6 @@ footer .info {
     display: none;
 }
 
-#files li[id$='datestamp'] {
-  display: none;
-}
-
 #preferences .select label {
   font-weight: normal;
   line-height: 35px;
@@ -189,15 +185,6 @@ footer .info {
   color: #101010;
 }
 
-#file .datestamp {
-  display: inline-block;
-  background: #f4f4f4;
-  padding: 4px 10px;
-  margin-bottom: 10px;
-  border-radius: 4px;
-  font-size: 75%;
-}
-
 #file .preview {
   overflow:hidden;
 }
@@ -206,3 +193,12 @@ footer .info {
   padding: 10px 12px;
   float: right;
 }
+
+#package_info {
+  display: inline-block;
+  background: #f4f4f4;
+  padding: 4px 10px;
+  margin-bottom: 10px;
+  border-radius: 4px;
+  font-size: 75%;
+}

debomatic-webui/scripts/install.sh

@@ -2,7 +2,6 @@
 
 export SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-bash ${SCRIPTS_DIR}/install/patch_css_directory_listing.sh
 bash ${SCRIPTS_DIR}/install/download_external_libs.sh
 
 python ${SCRIPTS_DIR}/install/create-user-config.py

debomatic-webui/scripts/install/patch_css_directory_listing.sh

-#!/bin/bash
-
-CSS="${SCRIPTS_DIR}/../node_modules/serve-index/public/style.css"
-
-if [ "`grep debomatic-webui $CSS`" == "" ] ; then
-  echo "Patching directory listing style.css"
-  cp $CSS $CSS.orig
-  echo "
-  /* debomatic-webui style patch */
-  ul#files li {
-    float: none;
-  }
-  /* end debomatic-webui patch */
-  " >> $CSS
-fi
-

debomatic-webui/views/distribution.ejs

@@ -72,7 +72,7 @@
     </header>
     <div id="welcome"></div>
     <div id="file">
-      <div class="datestamp"></div>
+      <div id="package_info"></div>
       <pre class="content"></pre>
       <div id="fileOffset"></div>
     </div>