Commit 5fe66770 authored by Mattia Rizzolo's avatar Mattia Rizzolo

Merge remote-tracking branch 'origin/dev-shadmod-01-debug-downloads-versions' into develop

MR: https://code.ubuntu-it.org/ubuntu-it-web/www/-/merge_requests/48Signed-off-by: Mattia Rizzolo's avatarMattia Rizzolo <mapreri@ubuntu.com>
parents 8d14fd3e 8bfb2ec0
Pipeline #339 passed with stage
in 0 seconds
# Source code of the Ubuntu-it website
# Copyright (C) 2015-2018 Pietro Albini <pietroalbini@ubuntu.com>
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
......@@ -29,7 +30,7 @@ from . import redirects
from . import utils
def create_app(data_path):
def create_app(data_path, debug=False):
"""Create a new instance of the application"""
# Normalize the data path
data_path = os.path.expanduser(os.path.abspath(data_path))
......@@ -44,8 +45,8 @@ def create_app(data_path):
init_data_directory(data_path)
# Load the secret key
with open(os.path.join(data_path, "secret_key")) as f:
app.secret_key = f.read().strip()
with open(os.path.join(data_path, "secret_key")) as fn:
app.secret_key = fn.read().strip()
# Initialize the database
app.db = db.Database(os.path.join(data_path, "database.db"))
......@@ -79,14 +80,14 @@ def create_app(data_path):
return app
def init_data_directory(data_path):
def init_data_directory(data_path: str):
"""Initialize the data directory"""
src_directory = os.path.dirname(os.path.abspath(__file__))
# Create all the directories
dirs = ["", "cache"]
for dir in dirs:
os.makedirs(os.path.join(data_path, dir), exist_ok=True)
for _dir in dirs:
os.makedirs(os.path.join(data_path, _dir), exist_ok=True)
# Initialize the cache
static_dirs = {"static": "+assets"}
......@@ -105,6 +106,10 @@ def init_data_directory(data_path):
f.write("%s\n" % utils.random_key(64))
os.chmod(secret_key_path, 0o400)
def init_data_downloads(data_path, debug=False):
# Initialize the download files
download_inst = download.Downloads(data_path)
download_inst = download.Downloads(data_path, debug=debug)
if debug is False:
# mk cache file
download_inst.store_cache_file()
# Configurazione delle release disponibili
# Quando viene rilasciata una release o una point release è solamente
# necessario aggiornare questa sezione.
[releases.latest]
version = "21.10"
codename = "impish"
lts = false
[releases.lts]
version = "22.04.3"
codename = "jammy"
lts = true
# Distribuzioni disponibili per il download
# Tutte le pagine in /download sono generate automaticamente da questa lista.
#
# name: Il nome della distro
# description: Una breve descrizione della distro
# lts-support-years: Il numero di anni per cui la LTS è supportata
# lts-only: Se impostato a `true`, solo le LTS verranno offerte
# releases: Una lista di release da offrire -- viene filtrata da lts-only
# archs: Una tabella di architettura e sorgente delle ISO
#
# Se è necessario sovrascrivere la sorgente delle ISO per una singola coppia
# release-architettura, è possibile aggiungere la chiave:
#
# source-override-RELEASE-ARCH = "SOURCE"
[distros.desktop]
name = "Ubuntu"
description = "L'originale, con GNOME"
lts-support-years = 5
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "releases-ubuntu" }
[distros.live-server]
name = "Ubuntu Server"
description = "Tutta la potenza di Ubuntu nel tuo server"
lts-support-years = 5
lts-only = true
releases = ["lts", "latest"] # Invertiti intenzionalmente
# amd64 è su releases.u.c, mentre arm64 e ppc64el sono su cdimages.u.c
archs = { amd64 = "releases-ubuntu", arm64 = "cdimages-ubuntu", ppc64el = "cdimages-ubuntu" }
[distros.kubuntu]
name = "Kubuntu"
description = "L'esperienza Ubuntu con desktop KDE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.lubuntu]
name = "Lubuntu"
description = "La derivata più leggera, con LXDE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntu-budgie]
name = "Ubuntu Budgie"
description = "La potenza di Ubuntu e la leggerezza di Budgie"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntukylin]
name = "Ubuntu Kylin"
description = "La derivata di Ubuntu con desktop UKUI"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntu-mate]
name = "Ubuntu MATE"
description = "Ubuntu si unisce a MATE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntustudio]
name = "Ubuntu Studio"
description = "La derivata di Ubuntu dedicata alla multimedialità"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-studio" }
[distros.xubuntu]
name = "Xubuntu"
description = "La derivata di Ubuntu leggera ma personalizzabile, con desktop XFCE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
# Nomi delle architetture
# Vengono usati per scegliere l'architettura da scaricare
[archs]
amd64 = "64bit"
#i386 = "32bit (computer più datati)"
arm64 = "ARM 64bit"
ppc64el = "POWER"
# Tabella delle sorgenti delle ISO
# Le distro possono scegliere da quale sorgente prendere ogni architettura
[sources.releases-ubuntu]
http = "{mirror:ubuntu}/{codename}/ubuntu-{version}-{distro}-{arch}.iso"
torrent = "{mirror:ubuntu}/{codename}/ubuntu-{version}-{distro}-{arch}.iso.torrent"
[sources.cdimages-ubuntu]
http = "http://cdimages.ubuntu.com/releases/{codename}/release/ubuntu-{version}-{distro}-{arch}.iso"
torrent = "http://cdimages.ubuntu.com/releases/{codename}/release/ubuntu-{version}-{distro}-{arch}.iso.torrent"
[sources.cdimages-derivatives]
http = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso"
torrent = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso.torrent"
[sources.cdimages-studio]
http = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-dvd-{arch}.iso"
torrent = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-dvd-{arch}.iso.torrent"
# Configurazione dei mirror
# I mirror vengono caricati da Launchpad in automatico
[mirrors]
# apparentemente non ci sono più mirror ufficiali in Italia ☹
country = "DE"
for = ["ubuntu"]
# Source code of the Ubuntu-it website
# Copyright (C) 2016 Pietro Albini <pietroalbini@ubuntu.com>
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
......@@ -29,16 +30,20 @@ from . import cache
from . import launchpad
from .constants import UITWWW_DIR
CONFIG_FILE = "/data/downloads.toml"
DOWNLOAD_FILE = "downloads.toml"
CACHE_FILE = "download-cache.json"
CACHE_FILE_VERSION = 1
class Downloads:
def __init__(self, data_path):
def __init__(self, data_path, debug: bool = False):
# init _mirrors and _sha256sums
self._mirrors = None
self._sha256sums = None
# Load the configuration
path = UITWWW_DIR + CONFIG_FILE
path = os.path.join(data_path, DOWNLOAD_FILE)
self.config = toml.load(
path, _dict=collections.OrderedDict,
)
......@@ -49,6 +54,7 @@ class Downloads:
with open(path, "rb") as raw:
self._config_hash = "sha1=%s" % hashlib.sha1(raw.read()).hexdigest()
if debug is False:
self._cache_file = os.path.join(data_path, CACHE_FILE)
@property
......@@ -61,7 +67,7 @@ class Downloads:
@property
def mirrors(self):
"""Get a list of CD mirrors needed by the website"""
if not hasattr(self, "_mirrors"):
if not hasattr(self, "_mirrors") or self._mirrors is None:
if self._cache is not None:
self._mirrors = self._cache["mirrors"]
else:
......@@ -70,6 +76,8 @@ class Downloads:
found_mirrors = list(sorted(launchpad.get_cdimage_mirrors(
distro, self.config["mirrors"]["country"]
)))
if "it1.mirror.vhosting-it.com" in found_mirrors[0]:
found_mirrors = ["https://releases.ubuntu.com/"]
if found_mirrors:
self._mirrors[distro] = found_mirrors
if not self._mirrors:
......@@ -80,14 +88,14 @@ class Downloads:
return self._mirrors
@property
def md5sums(self):
"""Get a list of all the MD5SUMS"""
if not hasattr(self, "_md5sums"):
def sha256sums(self):
"""Get a list of all the SHA256SUMS"""
if getattr(self, "_sha256sums") is None:
if self._cache is not None:
self._md5sums = self._cache["md5sums"]
self._sha256sums = self._cache["sha256sums"]
else:
self._md5sums = self._fetch_md5sums()
return self._md5sums
self._sha256sums = self._fetch_sha256sums()
return self._sha256sums
def _strip_non_lts_releases(self):
"""Process the lts-only distro configuration"""
......@@ -99,14 +107,14 @@ class Downloads:
if not self.config["releases"][release]["lts"]:
config["releases"].remove(release)
def _fetch_md5sums(self):
"""Fetch all the needed MD5SUMS"""
def _fetch_sha256sums(self):
"""Fetch all the needed SHA256SUMS"""
result = {}
files_content = {}
for distro, config in self.config["distros"].items():
for release in config["releases"]:
if config["lts-only"] and not self.config["releases"][release]["lts"]:
if config["lts-only"]:
continue
for arch in config["archs"].keys():
......@@ -115,7 +123,7 @@ class Downloads:
# Fetch the file from the remote if it wasn't done already
if path not in files_content:
md5s = {}
sha_hash = {}
response = requests.get("%s/SHA256SUMS" % path)
if response.status_code == 404:
......@@ -127,19 +135,19 @@ class Downloads:
if line.strip() == "":
continue
hash, name = line.split(" ", 1)
_hash, name = line.split(" ", 1)
if name.startswith("*"):
name = name[1:]
md5s[name] = hash
files_content[path] = md5s
sha_hash[name] = _hash
files_content[path] = sha_hash
# Add the MD5 to the result if it was found
# Add the SHA256 to the result if it was found
if file in files_content[path]:
key = "%s:%s:%s" % (distro, release, arch)
result[key] = files_content[path][file]
else:
print(files_content[path])
raise RuntimeError("Missing MD5 for {}".format(file))
raise RuntimeError("Missing SHA256SUMS for {}".format(file))
return result
......@@ -173,7 +181,7 @@ class Downloads:
"version": CACHE_FILE_VERSION,
"content": {
"mirrors": self.mirrors,
"md5sums": self.md5sums,
"sha256sums": self.sha256sums,
},
}, f, separators=(",", ":"))
......@@ -183,12 +191,13 @@ class Downloads:
def url_for(self, distro, release, arch, torrent=False, use_random=True):
"""Get the URL of a download file"""
# Build the dict of keys to replace
data_rel = self.config["distros"][distro]["archs"][arch][release]
replaces = {
"distro": distro,
"release": release,
"arch": arch,
"version": self.config["releases"][release]["version"],
"codename": self.config["releases"][release]["codename"],
"version": data_rel["version"],
"codename": data_rel["codename"],
}
# Add mirrors to the replaces key
......@@ -199,14 +208,14 @@ class Downloads:
replaces["mirror:%s" % name] = choices[0]
# Get the right source
source_name = self.config["distros"][distro]["archs"][arch]
source = self.config["sources"][source_name]
host = data_rel["host"]
source = self.config["sources"][host]
# Download URLs are different for torrent and HTTP
if torrent:
url = source["torrent"]
else:
url = source["http"]
url = source["https"]
for key, value in replaces.items():
url = url.replace("{%s}" % key, value)
......@@ -233,11 +242,12 @@ class Downloads:
if distro not in self.config["distros"]:
flask.abort(404)
data = self.config["distros"][distro]
return flask.render_template(
"download/landing.html",
distro_name=distro,
distro=self.config["distros"][distro],
releases=self.config["releases"],
distro=data,
releases=data["archs"]["amd64"],
archs=self.config["archs"],
)
......@@ -261,16 +271,19 @@ class Downloads:
torrent = "torrent" in flask.request.form
payload = signer.dumps({
data_rel = self.config["distros"][distro]["archs"][arch][release]
payload = signer.dumps(
{
"url": self.url_for(distro, release, arch, torrent),
"md5": self.md5sums["%s:%s:%s" % (distro, release, arch)],
"sha256": self.sha256sums[f"{distro}:{release}:{arch}"],
"name": "%s %s%s" % (
self.config["distros"][distro]["name"],
self.config["releases"][release]["version"],
" LTS" if self.config["releases"][release]["lts"] else "",
data_rel["version"],
" LTS" if data_rel["lts"] else "",
),
"theme": distro,
})
}
)
return flask.redirect(flask.url_for(".thanks", payload=payload))
@bp.route("/grazie/<payload>")
......@@ -288,10 +301,11 @@ class Downloads:
"""Generate the sub-navbar for /download"""
result = []
for name, distro in self.config["distros"].items():
result.append({
result.append(
{
"name": distro["name"],
"endpoint": "download.landing",
"endpoint-args": {"distro": name},
})
}
)
return result
# Source code of the Ubuntu-it website
# Copyright (C) 2015-2016 Pietro Albini <pietroalbini@ubuntu.com>
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
......@@ -19,28 +20,31 @@ import os
import click
import uitwww
from scss.compiler import compile_file
from .constants import UITWWW_DIR, BASE_DIR
from .utils import ReverseProxied, GunicornInstance
from scss.compiler import compile_file
from .src.download.compile_download import CompileVersion
@click.group()
def cli():
""" Ubuntu-it website's command line utility """
"""Ubuntu-it website's command line utility"""
pass
@cli.command("run")
@click.argument("data")
@click.option("-g", "--gunicorn-config", default=None, help="Path to a"
"gunicorn config file")
@click.option(
"-g", "--gunicorn-config", default=None, help="Path to a gunicorn config file"
)
@click.option("-p", "--port", default=8000, help="Bind that port")
@click.option("--public", help="Make available to the public", is_flag=True)
@click.option("-w", "--workers", help="Number of workers to start", default=3)
@click.option("-d", "--debug", help="Enable debug mode", is_flag=True)
def run(data, gunicorn_config, port, public, workers, debug):
"""Run the application"""
app = uitwww.create_app(data)
app = uitwww.create_app(data, debug)
app.wsgi_app = ReverseProxied(app.wsgi_app)
host = "127.0.0.1"
......@@ -76,11 +80,21 @@ def run(data, gunicorn_config, port, public, workers, debug):
@cli.command("init")
@click.argument("data")
def init(data):
@click.option("-d", "--debug", help="Enable debug mode", is_flag=True)
def init(data, debug):
"""Initialize the data directory"""
print("Initializing data directory:", data)
data_path = os.path.expanduser(os.path.abspath(data))
print("Initializing data directory:", data)
uitwww.init_data_directory(data_path)
print("Created data directory:", data)
compiler = CompileVersion(
path_out=data_path + "/downloads.toml",
constants=UITWWW_DIR + "/src/download/assets/constants.json",
)
compiler.compile_download()
print("Compile download file")
uitwww.init_data_downloads(data_path, debug)
print("Populate data download:", data)
@cli.command("build_scss")
......
# Source code of the Ubuntu-it website
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; witout even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Source code of the Ubuntu-it website
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; witout even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
{
"archs": {
"amd64": "64bit",
"arm64": "ARM 64bit",
"ppc64el": "PowerPC64"
},
"mirrors": {
"country": "IT",
"for": [
"ubuntu"
]
},
"distros": {
"desktop": {
"name": "Ubuntu",
"description": "L'originale, con GNOME"
},
"live-server": {
"name": "Ubuntu Server",
"description": "Tutta la potenza di Ubuntu nel tuo server",
"add_archs": [
"arm64",
"ppc64el"
]
},
"kubuntu": {
"name": "Kubuntu",
"description": "L'esperienza Ubuntu con desktop KDE"
},
"lubuntu": {
"name": "Lubuntu",
"description": "La derivata pi\u00f9 leggera, con LXDE"
},
"ubuntu-budgie": {
"name": "Ubuntu Budgie",
"description": "La potenza di Ubuntu e la leggerezza di Budgie"
},
"ubuntukylin": {
"name": "Ubuntu Kylin",
"description": "La derivata di Ubuntu con desktop UKUI"
},
"ubuntu-mate": {
"name": "Ubuntu MATE",
"description": "Ubuntu si unisce a MATE"
},
"ubuntustudio": {
"name": "Ubuntu Studio",
"description": "La derivata di Ubuntu dedicata alla multimedialit\u00e0"
},
"xubuntu": {
"name": "Xubuntu",
"description": "La derivata di Ubuntu leggera ma personalizzabile, con desktop XFCE"
}
}
}
# Source code of the Ubuntu-it website
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; witout even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import toml
import json
import requests
from lxml import html
from urllib.request import urlopen
from urllib.error import HTTPError
class UbuntuRelease:
def __init__(self, path_url=None, ignore_version=None):
self.path_url = path_url
self.ignore_version = ignore_version
def get_table_rows(self, only_data: bool = True):
"""
:param only_data: to check you want to ignore the table header
"""
# get requests
res = requests.get(self.path_url)
# check requests status code
status = res.status_code
if status != 200:
raise Exception(f"Error requests - status: {status}")
# get all table rows
table_rows = html.fromstring(res.content).xpath('//pre//text()')
# if only_data is True ignore table header
if only_data is True:
return table_rows[8:]
# else get all table
return table_rows
@property
def get_list_version(self) -> list:
# init clean row and row_list
row = []
row_list = []
# get table_rows without header
for val in self.get_table_rows():
val = val.strip()
if val:
if 'Ubuntu ' not in val:
row.append(val.replace("/", ""))
else:
row.append(val)
if len(row) == 2:
if not row[0].split(".")[0].isnumeric():
continue
if 'LTS' in row[1]:
typology = "LTS"
else:
typology = "interim"
name = row[1].split("(")[1].replace(")", "")
short = name.split()[0].lower()
row_list.append((row[0], short, name, typology))
row = []
# the lambda is just out of scruple
return sorted(row_list, key=lambda x: x[0], reverse=True)
class CompileVersion:
"""
:param SUPPORT_5: set distro with 5 years support (else are 3 years)
:param INVERT_RELEASES: set if you want invert releases in template render
:param path_url: url to get version list
:param path_out: path where the toml file will be written
:param constants: constants path
:param ignore_version: list of tags versions to be ignored
"""
SUPPORT_5 = ["desktop", "live-server"]
INVERT_RELEASES = ["live-server"]
def __init__(
self,
path_url: str = "https://releases.ubuntu.com/",
path_out: str = "../../data/downloads.toml",
constants: str = "./assets/costants.json",
ignore_version: list = None,
):
self.path_url = path_url
self.path_out = path_out
self.constants = constants
self.ignore_version = ignore_version if ignore_version else []
self.list_version = UbuntuRelease(
path_url=self.path_url,
ignore_version=self.ignore_version,
).get_list_version
@property
def data_sources(self) -> list:
return [
(
"releases-ubuntu",
"https://releases.ubuntu.com/{codename}/ubuntu-{version}-{distro}-{arch}.iso",
),
(
"cdimages-ubuntu",
"https://cdimages.ubuntu.com/releases/{codename}/release/ubuntu-{version}-{distro}-{arch}.iso",
),
(
"cdimages-derivatives",