Add most of the backend logic of the download pages

This commit adds most of the backend needed to implement the download
pages. It implements:

- Automatic fetch of the .iso's md5s the first time the application is
  started (and when they change), so they're available to show in the
  "Thank you" page

- Automatic fetch of the list of official mirrors for Ubuntu from the
  Launchpad API

- Download URL generation, which randomizes between mirrors to avoid
  overloading one of them

MANIFEST.in

 recursive-include uitwww/templates *
 recursive-include uitwww/static *
 include uitwww/navbar.json
+include uitwww/download.json

setup.py

@@ -34,6 +34,7 @@ setuptools.setup(
         "flask",
         "click",
         "gunicorn",
+        "requests",
     ],
 
     packages = [

uitwww/__init__.py

@@ -21,6 +21,7 @@ import flask
 from . import pages
 from . import cache
 from . import utils
+from . import download
 
 
 def create_app(data_path):
@@ -78,3 +79,7 @@ def init_data_directory(data_path):
         with open(secret_key_path, "w") as f:
             f.write("%s\n" % utils.random_key(64))
         os.chmod(secret_key_path, 0o400)
+
+    # Initialize the download files
+    download_inst = download.Downloads(data_path)
+    download_inst.store_cache_file()

uitwww/download.json

+{
+    "mirrors": {
+        "country": "IT",
+        "for": ["ubuntu"]
+    },
+    "releases": {
+        "latest": {
+            "version": "16.10",
+            "codename": "yakkety",
+            "lts": false
+        },
+        "lts": {
+            "version": "16.04.1",
+            "codename": "xenial",
+            "lts": true
+        }
+    },
+    "distros": {
+        "ubuntu-desktop": {
+            "pretty-name": "Ubuntu",
+            "releases": ["latest", "lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "{mirror:ubuntu}/{codename}/ubuntu-{version}-desktop-{arch}.iso"
+            ],
+            "download_torrent": [
+                "{mirror:ubuntu}/{codename}/ubuntu-{version}-desktop-{arch}.iso.torrent"
+            ]
+        },
+        "ubuntu-server": {
+            "pretty-name": "Ubuntu Server",
+            "releases": ["latest", "lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "{mirror:ubuntu}/{codename}/ubuntu-{version}-server-{arch}.iso"
+            ],
+            "download_torrent": [
+                "{mirror:ubuntu}/{codename}/ubuntu-{version}-server-{arch}.iso.torrent"
+            ]
+        },
+        "kubuntu": {
+            "pretty-name": "Kubuntu",
+            "releases": ["latest", "lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso"
+            ],
+            "download_torrent": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso.torrent"
+            ]
+        },
+        "xubuntu": {
+            "pretty-name": "Xubuntu",
+            "releases": ["latest", "lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso"
+            ],
+            "download_torrent": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso.torrent"
+            ]
+        },
+        "lubuntu": {
+            "pretty-name": "Lubuntu",
+            "releases": ["latest", "lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso"
+            ],
+            "download_torrent": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso.torrent"
+            ]
+        },
+        "ubuntu-gnome": {
+            "pretty-name": "Ubuntu GNOME",
+            "releases": ["latest", "lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso"
+            ],
+            "download_torrent": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso.torrent"
+            ]
+        },
+        "ubuntu-mate": {
+            "pretty-name": "Ubuntu MATE",
+            "releases": ["latest", "lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso"
+            ],
+            "download_torrent": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso.torrent"
+            ]
+        },
+        "ubuntustudio": {
+            "pretty-name": "Ubuntu Studio",
+            "releases": ["lts"],
+            "archs": ["i386", "amd64"],
+            "download_http": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-dvd-{arch}.iso"
+            ],
+            "download_torrent": [
+                "http://cdimage.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-dvd-{arch}.iso.torrent"
+            ]
+        }
+    }
+}

uitwww/download.py

+# Source code of the Ubuntu-it website
+# Copyright (C) 2016  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; witout even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import hashlib
+import json
+import os
+import random
+
+import requests
+import pkg_resources
+
+from . import launchpad
+
+
+CACHE_FILE_VERSION = 1
+
+
+class Downloads:
+
+    def __init__(self, data_path):
+        # Load the configuration
+        config_raw = pkg_resources.resource_string("uitwww", "download.json")
+        self.config = json.loads(config_raw.decode("utf-8"))
+
+        # Save the hash of the configuration
+        self._config_hash = "sha1=%s" % hashlib.sha1(config_raw).hexdigest()
+
+        self._cache_file = os.path.join(data_path, "download-cache.json")
+
+    @property
+    def _cache(self):  # Cacheception
+        """Get the cache from the data directory"""
+        if not hasattr(self, "_cache_cache"):  # Naming things is hard
+            self._cache_cache = self._cache_content()
+        return self._cache_cache
+
+    @property
+    def mirrors(self):
+        """Get a list of CD mirrors needed by the website"""
+        if not hasattr(self, "_mirrors"):
+            if self._cache is not None:
+                self._mirrors = self._cache["mirrors"]
+            else:
+                self._mirrors = {
+                    distro: list(sorted(launchpad.get_cdimage_mirrors(
+                         distro, self.config["mirrors"]["country"]
+                    ))) for distro in self.config["mirrors"]["for"]
+                }
+
+        return self._mirrors
+
+    @property
+    def md5sums(self):
+        """Get a list of all the MD5SUMS"""
+        if not hasattr(self, "_md5sums"):
+            if self._cache is not None:
+                self._md5sums = self._cache["md5sums"]
+            else:
+                self._md5sums = self._fetch_md5sums()
+        return self._md5sums
+
+    def _fetch_md5sums(self):
+        """Fetch all the needed MD5SUMS"""
+        result = {}
+        files_content = {}
+
+        for distro, config in self.config["distros"].items():
+            for release in config["releases"]:
+                for arch in config["archs"]:
+                    url = self.url_for(distro, release, arch, use_random=False)
+                    path, file = url.rsplit("/", 1)
+
+                    # Fetch the file from the remote if it wasn't done already
+                    if path not in files_content:
+                        md5s = {}
+                        raw = requests.get("%s/MD5SUMS" % path).text
+                        for line in raw.split("\n"):
+                            if line.strip() == "":
+                                continue
+
+                            hash, name = line.split(" ", 1)
+                            if name.startswith("*"):
+                                name = name[1:]
+                            md5s[name] = hash
+                        files_content[path] = md5s
+
+                    # Add the MD5 to the result if it was found
+                    if file in files_content[path]:
+                        key = "%s:%s:%s" % (distro, release, arch)
+                        result[key] = files_content[path][file]
+
+        return result
+
+    def _cache_content(self):
+        """Get the content of the cache in the data directory"""
+        # The cache is returned if the file exists, the download configuration
+        # didn't change, the file is valid JSON and the version is correct
+        if not os.path.exists(self._cache_file):
+            return None
+        try:
+            with open(self._cache_file) as f:
+                content = json.load(f)
+        except ValueError:
+            return None
+        if content["config-hash"] != self._config_hash:
+            return None
+        if content["version"] != CACHE_FILE_VERSION:
+            return None
+        return content["content"]
+
+    def store_cache_file(self):
+        """Store the cache file in the data directory"""
+        # Store only if the cache isn't available
+        if self._cache is None:
+            with open(self._cache_file, "w") as f:
+                json.dump({
+                    "config-hash": self._config_hash,
+                    "version": CACHE_FILE_VERSION,
+                    "content": {
+                        "mirrors": self.mirrors,
+                        "md5sums": self.md5sums,
+                    },
+                }, f, separators=(",", ":"))
+
+            # Invalidate the cache in this instance
+            delattr(self, "_cache_cache")
+
+    def url_for(self, distro, release, arch, torrent=False, use_random=True):
+        """Get the URL of a download file"""
+        # Build the dict of keys to replace
+        replaces = {
+            "distro": distro,
+            "release": release,
+            "arch": arch,
+            "version": self.config["releases"][release]["version"],
+            "codename": self.config["releases"][release]["codename"],
+        }
+
+        # Add mirrors to the replaces key
+        for name, choices in self.mirrors.items():
+            if use_random:
+                replaces["mirror:%s" % name] = random.choice(choices)
+            else:
+                replaces["mirror:%s" % name] = choices[0]
+
+        # Download URLs are different for torrent and HTTP
+        if torrent:
+            urls = self.config["distros"][distro]["download_torrent"]
+        else:
+            urls = self.config["distros"][distro]["download_http"]
+
+        # Choose a random download URL
+        if use_random:
+            url = random.choice(urls)
+        else:
+            url = urls[0]
+
+        for key, value in replaces.items():
+            url = url.replace("{%s}" % key, value)
+
+        return url

uitwww/launchpad.py

+# Source code of the Ubuntu-it website
+# Copyright (C) 2016  Pietro Albini <pietroalbini@ubuntu.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; witout even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import requests
+
+
+BASE = "https://api.launchpad.net/devel"
+
+
+def fetch_paginated(initial):
+    """Fetch paginated data"""
+    next_url = initial
+    while True:
+        data = requests.get(next_url).json()
+
+        for entry in data["entries"]:
+            yield entry
+
+        if "next_collection_link" not in data:
+            break
+        next_url = data["next_collection_link"]
+
+
+def get_cdimage_mirrors(distro, country):
+    """Get a list of cdimage mirrors"""
+    result = []
+
+    for mirror in fetch_paginated("%s/%s/cdimage_mirrors" % (BASE, distro)):
+        # Ensure we get only CD Image mirrors
+        if mirror["content"] != "CD Image":
+            continue
+
+        # Only get Official mirrors, please
+        if mirror["status"] != "Official":
+            continue
+
+        # Only get mirrors of the specific country
+        if not mirror["country_link"].endswith("/+countries/%s" % country):
+            continue
+
+        if mirror["http_base_url"] is None:
+            continue
+
+        url = mirror["http_base_url"]
+        if url.endswith("/"):
+            url = url[:-1]
+        result.append(url)
+
+    return result