Initial commit

.gitignore

+# Setuptools stuff
+/managetests.egg-info
+/dist
+/build
+
+# Python stuff
+__pycache__
+.py[co]

README.md

+# Managetests
+
+Managetests è un semplice tool realizzato dal Gruppo Web di Ubuntu-it per la
+gestione dei branch di test per il suo sito. Esso è integrato con GitLab, e
+permette una sincronizzazione dei branch con i merge request presenti sul
+repository. È anche altamente specifico per la nostra configurazione, quindi
+potrebbe non funzionare in contesti differenti.
+
+Managetests è rilasciato sotto licenza GNU-AGPL v3.
+
+## Inizializzazione di managetests
+
+Managetests tiene tutti i file necessari al suo funzionamento in una directory
+specifica. È possibile crearla con questo comando:
+
+```
+$ managetests -r path/to/directory init http://code.ubuntu-it.org/ubuntu-it-web/www
+```
+
+È necessario specificare il percorso della directory e l'indirizzo del
+repository git. Una volta inizializzata la directory, bisogna modificare il
+file di configurazione `config.json` presente in essa, in questo modo:
+
+```json
+{
+    "gitlab-token": "YOUR-API-KEY",
+    "gitlab-url": "http://code.ubuntu-it.org",
+    "gitlab-project": 22,
+    "keep-branches": [
+        "master"
+    ]
+}
+```
+
+I parametri in esso sono i seguenti:
+
+* `gitlab-token` è l'API key di GitLab del tuo account
+* `gitlab-url` è l'URL dell'istanza GitLab
+* `gitlab-project` è l'ID del progetto su GitLab
+* `keep-branches` è una lista di branch da caricare anche se non sono in una
+  merge request (per esempio `master`)
+
+## Esecuzione di managetests
+
+Una volta configurato inizializzato e configurato managetests è possibile
+eseguirlo, con questo comando:
+
+```
+$ managetests -r path/to/directory run
+```
+
+La prima volta che il comando viene eseguito sarà necessario aspettare che ogni
+branch richiesto venga scaricato e preparato, e dopo ciò ogni branch verrà
+eseguito.
+
+## Generazione di una build
+
+Per creare i pacchetti di managetests, è necessario aver installato
+[pyinvoke](http://www.pyinvoke.org), ed eseguire questo comando:
+
+```
+$ invoke build
+```
+
+La prima esecuzione richiede una connessione ad internet per scaricare le
+dipendenze richieste. Per rimuovere tutti i file generati:
+
+```
+$ invoke clean
+```
+
+## Hacking
+
+Per contribuire a managetests è consigliata la creazione di un virtualenv per
+lo sviluppo. È possibile crearlo con il seguente comando:
+
+```
+$ invoke devel
+```
+
+Una volta creato il virtualenv è disponibile in ``build/envs/devel``. Per
+attivarlo bisogna ad ogni sessione di terminale eseguire:
+
+```
+$ source build/envs/devel/bin/activate
+```

managetests/__init__.py

+"""
+    testsmanager
+    Test instances manager for the Ubuntu-it website
+
+    Copyright (c) 2015 Pietro Albini
+    Released under the GNU-AGPL v3
+"""

managetests/__main__.py

+#!/bin/bash
+# A program which manages Ubuntu-it's web test server
+# Copyright (C) 2015  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; witout even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import json
+import subprocess
+
+import click
+
+from . import app
+from . import utils
+
+
+def error(message, *format):
+    """Show an error message"""
+    click.echo("managetests: error: {}".format(message.format(*format)))
+    sys.exit(1)
+
+
+@click.group()
+@click.option("-r", "--root", help="The managetests root")
+@click.pass_context
+def cli(ctx, root):
+    """Manage a managetests instance"""
+    ctx.obj = {}
+    skip_check = ["init"]
+
+    if root is not None and "MANAGETESTS_ROOT" in os.environ:
+        root = os.environ["MANAGETESTS_ROOT"]
+
+    skip = ctx.invoked_subcommand in skip_check
+    if root is None or not (skip or utils.is_root_valid(root)):
+        error("please provide a valid root path")
+    else:
+        ctx.obj["root"] = os.path.abspath(root)
+
+
+@cli.command("init")
+@click.argument("git-url")
+@click.pass_context
+def init_command(ctx, git_url):
+    """Init a new root directory"""
+    root = ctx.obj["root"]
+
+    # Check if the directory is empty
+    if os.path.exists(root) and os.listdir(root):
+        error("root directory not empty")
+
+    os.makedirs(root, exist_ok=True)
+    for dir in ("git", "envs", "caches", "public", "socks", "socks/branches"):
+        os.mkdir(os.path.join(root, dir))
+
+    with open(os.path.join(root, "config.json"), "w") as f:
+        content = {
+            "gitlab-url": "http://example.com",
+            "gitlab-project": 1,
+            "gitlab-token": "abcdefghi",
+            "keep-branches": [
+                "master",
+            ],
+        }
+        json.dump(content, f, indent=4)
+        f.write("\n")
+
+    with open(os.path.join(root, "details.json"), "w") as f:
+        content = {
+            "branches": {},
+        }
+        json.dump(content, f, indent=4)
+        f.write("\n")
+
+    subprocess.call(["git", "clone", git_url, os.path.join(root, "git")])
+
+
+@cli.command("run")
+@click.pass_obj
+def run_command(obj):
+    """Run managetests"""
+    root = obj["root"]
+
+    with open(os.path.join(root, "config.json")) as f:
+        config = json.load(f)
+
+    inst = app.TestsManager(root, config)
+    inst.run()

managetests/app.py

+# A program which manages Ubuntu-it's web test server
+# Copyright (C) 2015  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; witout even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import json
+
+import pkg_resources
+
+from . import utils
+from . import branches
+from . import instances
+from . import webhooks
+from . import gitlab
+
+
+class TestsManager:
+    """Main instance of the application"""
+
+    def __init__(self, root, config):
+        self.config = config
+        self.root = os.path.abspath(root)
+
+        self.gitlab = gitlab.GitLabAPI(self)
+
+        hooks_processor = webhooks.HooksProcessor(self)
+        self.webhook = webhooks.create_app(self, hooks_processor)
+        self.instances = instances.InstancesManager(self, self.webhook,
+                                                    hooks_processor)
+
+        if not utils.is_root_valid(root):
+            raise RuntimeError("Invalid root directory: %s" % root)
+
+        # Shortcut for creating multiple attributes
+        dirs = ["git", "envs", "caches", "public", "socks", "build"]
+        for dir in dirs:
+            setattr(self, "%s_dir" % dir, os.path.join(root, dir))
+
+        self.config_file = os.path.join(root, "config.json")
+        self.gunicorn_config_file = pkg_resources.resource_filename(
+            "managetests", "gunicorn_config"
+        )
+
+        self._load_details()
+        self._load_branches()
+
+    def _load_details(self):
+        """Load details from the root directory"""
+        with open(os.path.join(self.root, "details.json")) as f:
+            self.details = json.load(f)
+
+    def _load_branches(self):
+        """Load all the branches"""
+        self.branches = {}
+        for branch_name, mr in self.details["branches"].copy().items():
+            self.load_branch(branch_name, mr)
+
+        # Load also new forced-to-be-keeped branches
+        for branch_name in self.config["keep-branches"]:
+            if branch_name in self.branches:
+                continue
+            self.load_branch(branch_name, -1)
+
+        # Load also new branches from merge requests
+        new = self.gitlab.merge_requests()
+        if not new:
+            raise RuntimeError("Can't get merge requests from GitLab!")
+        for request in new:
+            # Skip already loaded requests
+            if request["source_branch"] in self.branches:
+                continue
+            self.load_branch(request["source_branch"], request["id"])
+
+    def load_branch(self, name, mr):
+        """Load a branch"""
+        branch = branches.Branch(self, name, mr)
+        if not branch.present and branch.active:
+            branch.deploy()
+        elif not branch.active:
+            if branch.present:
+                branch.destroy()
+            return
+        self.branches[name] = branch
+
+        # Load the branch in the runner
+        self.instances.load_branch(branch)
+
+        # Save the branch details
+        self.details["branches"][name] = mr
+        self.save_details()
+
+    def remove_branch(self, name):
+        """Remove a branch"""
+        if name not in self.branches:
+            return
+
+        self.instances.remove_branch(name)
+
+        self.branches[name].destroy()
+        del self.branches[name]
+
+        del self.details["branches"][name]
+        self.save_details()
+
+    def save_details(self):
+        """Save details on disk"""
+        with open(os.path.join(self.root, "details.json"), "w") as f:
+            json.dump(self.details, f)
+
+    def run(self):
+        """Run all the test instances"""
+        self.instances.run()
+
+    def process_hook(self, data):
+        """Process a webhook received by GitLab"""
+        obj = data["object_attributes"]
+        branch = obj["source_branch"]
+
+        # The branch was loaded
+        if branch in self.branches:
+            # Delete old branches
+            if obj["state"] != "opened":
+                self.remove_branch(branch)
+                self.gitlab.post_comment(obj["id"], "L'istanza è stata rimossa.")
+                return
+
+            # Get the local commit
+            with open("%s/%s/commit" % (self.envs_dir, branch)) as f:
+                commit = f.read()
+
+            # Rebuild if there is a new commit
+            if commit != obj["last_commit"]["id"]:
+                self.remove_branch(branch)
+                self.load_branch(branch, obj["id"])
+                self.gitlab.post_comment(obj["id"], "L'istanza è stata "
+                    "aggiornata con il nuovo codice.")
+
+        # The instance should be created
+        elif obj["state"] == "opened":
+            self.load_branch(branch, obj["id"])
+            self.gitlab.post_comment(obj["id"], "Istanza live disponibile su "
+                "http://wwwtest.ubuntu-it.org/%s." % branch)

managetests/branches.py

+"""
+    testsmanager.branches
+    Representation of the branches
+
+    Copyright (c) 2015 Pietro Albini <pietroalbini@ubuntu.com>
+    Released under the GNU-AGPL v3 license
+"""
+
+import sys
+import subprocess
+import os
+import shutil
+
+
+class Branch:
+    """Representation of a branch"""
+
+    def __init__(self, manager, name, merge_request):
+        self.manager = manager
+
+        self.name = name
+        self.merge_request = merge_request
+
+        self.check_remote_status()
+        self.check_local_status()
+
+    def check_remote_status(self):
+        """Check the branch merging status on GitLab"""
+        if self.name in self.manager.config["keep-branches"]:
+            self.active = True
+            return
+
+        result = self.manager.gitlab.merge_request(self.merge_request)
+        if result:
+            active = result["state"] == "opened"
+        else:
+            active = False
+        self.active = active
+
+    def check_local_status(self):
+        """Check if the branch is present locally"""
+        present_in = ["envs", "caches", "public"]
+        for one in present_in:
+            path = os.path.join(self.manager.root, one, self.name)
+            if not os.path.exists(path):
+                self.present = False
+                return
+        self.present = True
+
+    def deploy(self):
+        """Deploy the branch"""
+        self.check_local_status()
+        # Ok, branch already deployed
+        if self.present:
+            return
+
+        make_dirs_in = ["envs", "caches", "public"]
+        for dir in make_dirs_in:
+            os.mkdir(os.path.join(self.manager.root, dir, self.name))
+
+        commands = [
+            "rm -rf %s" % self.manager.build_dir,
+            "mkdir -p %s" % self.manager.build_dir,
+            "cd %s && git fetch" % self.manager.git_dir,
+            "git --git-dir=%s/.git --work-tree=%s checkout -f %s" %
+                (self.manager.git_dir, self.manager.build_dir, self.name),
+            "%s/scripts/build.sh" % self.manager.build_dir,
+            "virtualenv %s/%s" % (self.manager.envs_dir, self.name),
+            "cd %s && git rev-parse HEAD > %s/%s/commit" %
+                (self.manager.build_dir, self.manager.envs_dir, self.name),
+            "%s/%s/bin/pip install %s/out.tar.gz" % (self.manager.envs_dir,
+                self.name, self.manager.build_dir),
+            ("ln -s %s/%s/lib/python3.%s/site-packages/uitwww/static "
+             "%s/%s/static" % (self.manager.envs_dir, self.name,
+                sys.version_info[1], self.manager.public_dir, self.name)),
+            "rm -rf %s" % self.manager.build_dir,
+        ]
+
+        for command in commands:
+            subprocess.call(command, shell=True)
+
+        self.manager.details["branches"][self.name] = self.merge_request
+        self.manager.save_details()
+
+    def destroy(self):
+        """Destroy the local copy"""
+        self.check_local_status()
+        # Ok, branch not present
+        if not self.present:
+            return
+
+        remove_from = ["envs", "caches", "public"]
+        for dir in remove_from:
+            shutil.rmtree(os.path.join(self.manager.root, dir, self.name))
+
+        try:
+            del self.manager.details["branches"][self.name]
+        except KeyError:
+            pass
+        self.manager.save_details()

managetests/gitlab.py

+"""
+    testsmanager.gitlab
+    Wrapper for the GitLab API
+
+    Copyright (c) 2015 Pietro Albini <pietroalbini@ubuntu.com>
+    Released under the GNU-AGPL v3 license
+"""
+
+import requests
+
+
+class GitLabAPI:
+    """Wrapper for the GitLab API"""
+
+    def __init__(self, manager):
+        self.manager = manager
+        self.url = manager.config["gitlab-url"]
+        self.token = manager.config["gitlab-token"]
+        self.project = manager.config["gitlab-project"]
+
+    def call(self, method, endpoint, params=None, data=None):
+        """Make a raw call to the GitLab API"""
+        if params == None:
+            params = {}
+        if data is None:
+            data = {}
+
+        url = self.url+"/api/v3/"+endpoint
+        params["private_token"] = self.token
+
+        response = requests.request(method, url, params=params, data=data)
+        if response.status_code == 200:
+            return response.json()
+        return False
+
+    def merge_requests(self):
+        """Get all the merge requests of the project"""
+        return self.call("get", "projects/%s/merge_requests" % self.project, {
+            "state": "opened",
+        })
+
+    def merge_request(self, id):
+        """Get a single merge request on the project"""
+        return self.call("get", "projects/%s/merge_request/%s" % (
+            self.project, id))
+
+    def post_comment(self, id, text):
+        """Post a comment on a merge request"""
+        return self.call("post", "projects/%s/merge_request/%s/comments" % (
+            self.project, id), args={"note": text})

managetests/gunicorn_config.py

+import sys
+import os
+
+# An hacky way to discover if we're inside a virtualenv
+if not hasattr(sys, "real_prefix"):
+    raise RuntimeError("You must use this inside a virtualenv")
+
+_socks_path = os.path.realpath(os.path.join(sys.prefix,
+                                            "../../socks/branches"))
+_branch_name = sys.prefix.rsplit("/", 1)[-1]
+
+
+bind = "unix:%s/%s.sock" % (_socks_path, _branch_name)
+workers = 2
+
+
+del sys, os, _socks_path, _branch_name

managetests/instances.py

+import subprocess
+import atexit
+import time
+
+from werkzeug import serving
+
+
+class InstancesManager:
+    """This class will manage all the running branches"""
+
+    def __init__(self, manager, main, processor):
+        self.manager = manager
+        self.running = False
+
+        self._stop = False
+        self._processes = {}
+        self._branches = {}
+
+        self._main_app = main
+        self._hooks_processor = processor
+
+        atexit.register(_stop(self))
+
+    def load_branch(self, branch):
+        """Load a branch into the instances manager"""
+        self._branches[branch.name] = branch
+
+        if self.running:
+            self._run_branch(branch.name)
+
+    def remove_branch(self, name):
+        """Remove a branch from the instances manager"""
+        if name not in self._branches:
+            return
+
+        del self._branches[name]
+        if self.running:
+            self._processes[name].terminate()
+            del self._processes[name]
+
+    def run(self):
+        """Run all the instances"""
+        if self.running:
+            raise RuntimeError("Already running!")
+        self.running = True
+        self._stop = False
+
+        for branch in self._branches.values():
+            self._run_branch(branch)
+
+        # Starts also the receiver part
+        self._hooks_processor.start()
+        serving.run_simple("127.0.0.1", 8080, self._main_app)
+        self._hooks_processor.stop = True
+
+        # Stop all the running processes
+        for process in self._processes.values():
+            process.terminate()
+        self._processes = {}
+
+        print()
+
+    def _run_branch(self, branch):
+        """Run a single branch"""
+        command = [
+            "%s/%s/bin/uitwww" % (self.manager.envs_dir, branch.name),
+            "run", "-g", "%s.py" % self.manager.gunicorn_config_file,
+            "%s/%s" % (self.manager.caches_dir, branch.name),
+        ]
+        process = subprocess.Popen(command)
+        self._processes[branch.name] = process
+
+    def stop(self):
+        """Stop the server"""
+        if not self.running:
+            return
+
+        self._stop = True
+
+
+def _stop(inst):
+    """Function passed to atexit which stops the know world"""
+    def __(*_):
+        if inst.running:
+            inst.stop()
+    return __

managetests/utils.py

+# A program which manages Ubuntu-it's web test server
+# Copyright (C) 2015  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; witout even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+
+def is_root_valid(path):
+    """Check if a root directory is valid"""
+    # Eh...
+    if not os.path.exists(path):
+        return False
+
+    required_dirs = ["git", "envs", "caches", "public", "socks",
+                     "socks/branches"]
+    required_files = ["config.json", "details.json"]
+    base = os.path.abspath(path)
+
+    for dir in required_dirs:
+        dir_path = os.path.join(base, dir)
+        if not (os.path.exists(dir_path) and os.path.isdir(dir_path)):
+            return False
+    for file in required_files:
+        file_path = os.path.join(base, file)
+        if not (os.path.exists(file_path) and os.path.isfile(file_path)):
+            return False
+
+    return True

managetests/webhooks.py

+import queue
+import threading
+
+import flask
+
+
+class HooksProcessor(threading.Thread):
+    """A class which processes hooks"""
+
+    def __init__(self, manager):
+        self.manager = manager
+        self.q = queue.Queue()
+        self.stop = False
+
+        super(HooksProcessor, self).__init__()
+
+    def append(self, data):
+        """Append something to the queue"""
+        self.q.put(data)
+
+    def run(self):
+        """Run the thread"""
+        while not self.stop:
+            try:
+                data = self.q.get(timeout=0.2)
+            except queue.Empty:
+                continue
+
+            self.manager.process_hook(data)
+
+
+def create_app(manager, processor):
+    """Create an instance of the app which captures webhooks"""
+    app = flask.Flask(__name__)
+
+    @app.route("/+hook", methods=["POST"])
+    def hook():
+        processor.append(flask.request.data)
+        return "OK"
+
+    return app

requirements-build.txt

+wheel==0.26.0

setup.py

+import setuptools
+
+
+setuptools.setup(
+
+    name = "managetests",
+    version = "1",
+    url = "http://www.ubuntu-it.org",
+
+    license = "GNU-GPL v3",
+
+    author = "Ubuntu-it website team",
+    author_email = "gruppo-web@liste.ubuntu-it.org",
+
+    description = "An application to manage our test instances",
+
+    install_requires = [
+        "flask",
+        "click",
+        "gunicorn",
+        "requests",
+    ],
+
+    packages = [
+        "managetests",
+    ],
+
+    entry_points = {
+        "console_scripts": [
+            "managetests = managetests.__main__:cli",
+        ],
+    },
+
+    zip_safe = False,
+
+    classifiers = [
+        "Not on PyPI"
+    ],
+
+)

tasks.py

+# Manage test servers for the Ubuntu-it's websites
+# Copyright (C) 2015 Pietro Albini
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import glob
+
+import invoke
+
+
+BASE = os.path.dirname(__file__)
+PYTHON = "python3"
+PROJECT = "managetests"
+
+
+def create_env(name, requirements=False, self=False, force=False):
+    """Create a new virtual environment"""
+    path = os.path.join(BASE, "build", "envs", name)
+
+    # Don't re-create the environment if force is False
+    if os.path.exists(path):
+        if force:
+            shutil.rmtree(path)
+        else:
+            return path
+
+    invoke.run("virtualenv -p %s %s" % (PYTHON, path))
+    if requirements:
+        invoke.run("%s/bin/pip install -r requirements-%s.txt" % (path, name))
+    if self:
+        invoke.run("%s/bin/pip install -e ." % path)
+
+    return path
+
+
+@invoke.task
+def devel():
+    """Setup the development environment"""
+    create_env("devel", self=True, force=True)
+
+
+@invoke.task
+def build():
+    """Create a new build"""
+    env = create_env("build", requirements=True)
+
+    out = os.path.join(BASE, "build", "packages")
+    if os.path.exists(out):
+        for file in glob.glob(os.path.join(out, "*")):
+            os.remove(file)
+
+    for type in "sdist", "bdist_wheel":
+        invoke.run("%s/bin/python setup.py %s -d %s" % (env, type, out))
+
+
+@invoke.task
+def clean():
+    """Clean all the build things"""
+    for dir in "build", "%s.egg-info" % PROJECT:
+        path = os.path.join(BASE, dir)
+        if not os.path.exists(path):
+            continue
+        shutil.rmtree(path)