Use a new directory structure for the root directory

This new structure allows parallel builds, is versioned, and also is
cleaner, with every branch in their own directory.

managetests/__main__.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import pathlib
 import os
 import sys
 import json
@@ -28,6 +29,9 @@ from . import app
 from . import utils
 
 
+DEFAULT_GIT_URL = "http://code.ubuntu-it.org/ubuntu-it-web/www.git"
+
+
 def error(message, *format):
     """Show an error message"""
     click.echo("managetests: error: {}".format(message.format(*format)))
@@ -56,25 +60,28 @@ def cli(ctx, root):
     if root is None or not (skip or utils.is_root_valid(root)):
         error("please provide a valid root path")
     else:
-        ctx.obj["root"] = os.path.abspath(root)
+        ctx.obj["root"] = pathlib.Path(root)
 
 
 @cli.command("init")
-@click.argument("git-url")
+@click.argument("git-url", default=DEFAULT_GIT_URL)
 @click.pass_context
 def init_command(ctx, git_url):
     """Init a new root directory"""
     root = ctx.obj["root"]
 
     # Check if the directory is empty
-    if os.path.exists(root) and os.listdir(root):
+    if root.exists() and len(list(root.iterdir())):
         error("root directory not empty")
 
-    os.makedirs(root, exist_ok=True)
-    for dir in ("git", "envs", "caches", "public", "socks", "socks/branches"):
-        os.mkdir(os.path.join(root, dir))
+    root.mkdir(parents=True)
+    for dir in ("git", "branches"):
+        (root / dir).mkdir()
+
+    with (root / "version").open("w") as f:
+        f.write("%s\n" % utils.ROOT_DIR_VERSION)
 
-    with open(os.path.join(root, "config.json"), "w") as f:
+    with (root / "config.json").open("w") as f:
         content = {
             "gitlab-url": "http://code.ubuntu-it.org",
             "gitlab-project": "ubuntu-it-web/www",
@@ -87,14 +94,14 @@ def init_command(ctx, git_url):
         json.dump(content, f, indent=4)
         f.write("\n")
 
-    with open(os.path.join(root, "details.json"), "w") as f:
+    with (root / "details.json").open("w") as f:
         content = {
             "branches": {},
         }
         json.dump(content, f, indent=4)
         f.write("\n")
 
-    subprocess.call(["git", "clone", git_url, os.path.join(root, "git")])
+    subprocess.call(["git", "clone", git_url, str(root / "git"), "--bare"])
 
 
 @cli.command("run")
@@ -103,9 +110,9 @@ def init_command(ctx, git_url):
 @click.pass_obj
 def run_command(obj, port):
     """Run managetests"""
-    root = obj["root"]
+    root = obj["root"].resolve()
 
-    with open(os.path.join(root, "config.json")) as f:
+    with (root / "config.json").open() as f:
         config = json.load(f)
 
     inst = app.TestsManager(root, config, port)

managetests/app.py

@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
 import json
+import pathlib
 
 import pkg_resources
 
@@ -31,7 +31,7 @@ class TestsManager:
 
     def __init__(self, root, config, port):
         self.config = config
-        self.root = os.path.abspath(root)
+        self.root = root
 
         self.gitlab = gitlab.GitLabAPI(self)
 
@@ -43,12 +43,7 @@ class TestsManager:
         if not utils.is_root_valid(root):
             raise RuntimeError("Invalid root directory: %s" % root)
 
-        # Shortcut for creating multiple attributes
-        dirs = ["git", "envs", "caches", "public", "socks", "build"]
-        for dir in dirs:
-            setattr(self, "%s_dir" % dir, os.path.join(root, dir))
-
-        self.config_file = os.path.join(root, "config.json")
+        self.config_file = root / "config.json"
         self.gunicorn_config_file = pkg_resources.resource_filename(
             "managetests", "gunicorn_config"
         )
@@ -58,12 +53,14 @@ class TestsManager:
 
     def _load_details(self):
         """Load details from the root directory"""
-        with open(os.path.join(self.root, "details.json")) as f:
+        with (self.root / "details.json").open() as f:
             self.details = json.load(f)
 
     def _load_branches(self):
         """Load all the branches"""
         self.branches = {}
+
+        # Load already present branches from merge requests
         for branch_name, mr in self.details["branches"].copy().items():
             self.load_branch(branch_name, mr)
 
@@ -117,7 +114,7 @@ class TestsManager:
 
     def save_details(self):
         """Save details on disk"""
-        with open(os.path.join(self.root, "details.json"), "w") as f:
+        with (self.root / "details.json").open("w") as f:
             json.dump(self.details, f)
 
     def run(self):

managetests/branches.py

@@ -20,6 +20,9 @@ import os
 import shutil
 
 
+BRANCH_DIR_VERSION = "1"
+
+
 class Branch:
     """Representation of a branch"""
 
@@ -66,49 +69,103 @@ class Branch:
 
     def check_local_status(self):
         """Check if the branch is present locally"""
-        present_in = ["envs", "caches", "public"]
-        for one in present_in:
-            path = os.path.join(self.manager.root, one, self.name)
-            if not os.path.exists(path):
-                self.present = False
-                return
-        self.present = True
+        if (self.manager.root / "branches" / self.name).exists():
+            self.present = self.valid()
+        else:
+            self.present = False
+
+    def valid(self):
+        """Check if a branch is valid"""
+        root = self.manager.root / "branches" / self.name
+
+        # The branch must be a valid directory
+        if not (root.exists() and root.is_dir()):
+            return False
+
+        # All the subdirectories must be present
+        dirs = ["env", "public", "data"]
+        for dir in dirs:
+            dir = root / dir
+            if not (dir.exists() and dir.is_dir()):
+                return False
+
+        # All the files must be be present
+        files = ["version"]
+        for file in files:
+            file = root / file
+            if not (file.exists() and file.is_file()):
+                return False
+
+        # The directory version must be correct
+        with (root / "version").open() as f:
+            if f.read().strip() != BRANCH_DIR_VERSION:
+                return False
+
+        return True
 
     def deploy(self):
         """Deploy the branch"""
         self.check_local_status()
+
         # Ok, branch already deployed
         if self.present:
             return
 
-        make_dirs_in = ["envs", "caches", "public"]
-        for dir in make_dirs_in:
-            os.mkdir(os.path.join(self.manager.root, dir, self.name))
-
-        commands = [
-            "rm -rf %s" % self.manager.build_dir,
-            "mkdir -p %s" % self.manager.build_dir,
-            "cd %s && git fetch" % self.manager.git_dir,
-            "git --git-dir=%s/.git --work-tree=%s checkout -f origin/%s" %
-                (self.manager.git_dir, self.manager.build_dir, self.name),
-            "cd %s && invoke build" % self.manager.build_dir,
-            "virtualenv -p python3 %s/%s" % (self.manager.envs_dir, self.name),
-            "%s/%s/bin/pip install %s/build/packages/*.whl" %
-                (self.manager.envs_dir, self.name, self.manager.build_dir),
-            ("ln -s %s/%s/lib/python3.%s/site-packages/uitwww/static "
-             "%s/%s/static" % (self.manager.envs_dir, self.name,
-                sys.version_info[1], self.manager.public_dir, self.name)),
-            "rm -rf %s" % self.manager.build_dir,
-            "rm -rf %s/%s/*" % (self.manager.caches_dir, self.name),
-        ]
-
-        # Provide to the command the current environment plus $HOME
-        # gulp fails otherwise
-        env = dict(os.environ)
-        env["HOME"] = self.manager.build_dir
+        # Start from a fresh branch dir
+        branch = self.manager.root / "branches" / self.name
+        if branch.exists():
+            shutil.rmtree(str(branch))
+        branch.mkdir()
+
+        git_dir = str(self.manager.root / "git")
+        build_dir = branch / "build"
+
+        # Create subdirectories
+        subdirs = ["env", "public", "data", "build"]
+        for dir in subdirs:
+            (branch / dir).mkdir()
 
-        for command in commands:
-            subprocess.call(command, shell=True, env=env)
+        # Create the version file
+        with (branch / "version").open("w") as f:
+            f.write("%s\n" % BRANCH_DIR_VERSION)
+
+        # Update the bare git repository and checkout the branch
+        subprocess.call(["git", "--git-dir", git_dir, "fetch", "origin"])
+        subprocess.call([
+            "git", "--git-dir", git_dir, "--work-tree", str(build_dir),
+            "checkout", "-f", self.name,
+        ])
+
+        # Build the website
+        env = dict(os.environ)
+        env["HOME"] = str(build_dir)
+        subprocess.call(["invoke", "build"], cwd=str(build_dir), env=env)
+
+        # Check if the build results exists
+        results = list(build_dir.glob("build/packages/*.whl"))
+        if not results:
+            return False
+
+        # Create a new environment and install all the results in it
+        subprocess.call(["virtualenv", "-p", "python3", str(branch / "env")])
+        for result in results:
+            subprocess.call([
+                "%s/bin/pip" % str(branch / "env"), "install", str(result)
+            ])
+
+        # Calculate the static files directory
+        py_version = ".".join(str(i) for i in tuple(sys.version_info)[:2])
+        static_dir = (
+            branch / "env" / "lib" / ("python%s" % py_version) /
+            "site-packages" / "uitwww" / "static"
+        )
+
+        # Link the assets directory to the static files one
+        assets = branch / "public" / "+assets"
+        assets.symlink_to(static_dir)
+
+        # Delete the build directory
+        shutil.rmtree(str(build_dir))
 
         self.manager.details["branches"][self.name] = self.mr
         self.manager.save_details()
@@ -116,13 +173,12 @@ class Branch:
     def destroy(self):
         """Destroy the local copy"""
         self.check_local_status()
+
         # Ok, branch not present
         if not self.present:
             return
 
-        remove_from = ["envs", "caches", "public"]
-        for dir in remove_from:
-            shutil.rmtree(os.path.join(self.manager.root, dir, self.name))
+        shutil.rmtree(str(self.manager.root_dir / "branches" / name))
 
         try:
             del self.manager.details["branches"][self.name]

managetests/gunicorn_config.py

 # A program which manages Ubuntu-it's web test server
-# Copyright (C) 2015  Pietro Albini <pietroalbini@ubuntu.com>
+# Copyright (C) 2015-2016  Pietro Albini <pietroalbini@ubuntu.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published
@@ -21,13 +21,11 @@ import os
 if not hasattr(sys, "real_prefix"):
     raise RuntimeError("You must use this inside a virtualenv")
 
-_socks_path = os.path.realpath(os.path.join(sys.prefix,
-                                            "../../socks/branches"))
-_branch_name = sys.prefix.rsplit("/", 1)[-1]
+_branch_dir = os.path.realpath(os.path.join(sys.prefix, ".."))
 
 
-bind = "unix:%s/%s.sock" % (_socks_path, _branch_name)
+bind = "unix:%s/branch.sock" % (_branch_dir)
 workers = 2
 
 
-del sys, os, _socks_path, _branch_name
+del sys, os, _branch_dir

managetests/instances.py

@@ -79,10 +79,13 @@ class InstancesManager:
 
     def _run_branch(self, branch):
         """Run a single branch"""
+        branch_dir = self.manager.root / "branches" / branch.name
+        command_path = str(branch_dir / "env" / "bin" / "uitwww")
+
         command = [
-            "%s/%s/bin/uitwww" % (self.manager.envs_dir, branch.name),
-            "run", "-g", "%s.py" % self.manager.gunicorn_config_file,
-            "%s/%s" % (self.manager.caches_dir, branch.name),
+            command_path, "run", "-g",
+            "%s.py" % self.manager.gunicorn_config_file,
+            str(branch_dir / "data")
         ]
         try:
             process = subprocess.Popen(command)

managetests/utils.py

@@ -14,27 +14,34 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
+import pathlib
+
+
+ROOT_DIR_VERSION = "1"
 
 
 def is_root_valid(path):
     """Check if a root directory is valid"""
-    # Eh...
-    if not os.path.exists(path):
+    path = pathlib.Path(path)
+
+    if not (path.exists() and path.is_dir()):
         return False
 
-    required_dirs = ["git", "envs", "caches", "public", "socks",
-                     "socks/branches"]
-    required_files = ["config.json", "details.json"]
-    base = os.path.abspath(path)
+    required_dirs = ["git", "branches"]
+    required_files = ["config.json", "details.json", "version"]
 
     for dir in required_dirs:
-        dir_path = os.path.join(base, dir)
-        if not (os.path.exists(dir_path) and os.path.isdir(dir_path)):
+        dir = path / dir
+        if not (dir.exists() and dir.is_dir()):
             return False
+
     for file in required_files:
-        file_path = os.path.join(base, file)
-        if not (os.path.exists(file_path) and os.path.isfile(file_path)):
+        file = path / file
+        if not (file.exists() and file.is_file()):
+            return False
+
+    with (path / "version").open("r") as f:
+        if f.read().strip() != ROOT_DIR_VERSION:
             return False
 
     return True

nginx.conf

@@ -17,13 +17,13 @@ server {
     location ~ ^/([a-zA-Z0-9\-\._]+)/ {
         set $branch $1;
 
-        root $managetests_root/caches;
+        alias $managetests_root/branches/$branch/data/cache;
         try_files $request_uri.html $request_uri/index.html $request_uri
                   @branch;
     }
 
     location @branch {
-        proxy_pass http://unix:$managetests_root/socks/branches/$branch.sock;
+        proxy_pass http://unix:$managetests_root/branches/$branch/branch.sock;
         proxy_set_header Host $host;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Scheme $scheme;