Commit 5fe66770 authored by Mattia Rizzolo's avatar Mattia Rizzolo

Merge remote-tracking branch 'origin/dev-shadmod-01-debug-downloads-versions' into develop

MR: https://code.ubuntu-it.org/ubuntu-it-web/www/-/merge_requests/48Signed-off-by: Mattia Rizzolo's avatarMattia Rizzolo <mapreri@ubuntu.com>
parents 8d14fd3e 8bfb2ec0
Pipeline #339 passed with stage
in 0 seconds
# Source code of the Ubuntu-it website # Source code of the Ubuntu-it website
# Copyright (C) 2015-2018 Pietro Albini <pietroalbini@ubuntu.com> # Copyright (C) 2015-2018 Pietro Albini <pietroalbini@ubuntu.com>
# Copyright (C) 2023 shadMod
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published # it under the terms of the GNU Affero General Public License as published
...@@ -29,7 +30,7 @@ from . import redirects ...@@ -29,7 +30,7 @@ from . import redirects
from . import utils from . import utils
def create_app(data_path): def create_app(data_path, debug=False):
"""Create a new instance of the application""" """Create a new instance of the application"""
# Normalize the data path # Normalize the data path
data_path = os.path.expanduser(os.path.abspath(data_path)) data_path = os.path.expanduser(os.path.abspath(data_path))
...@@ -44,8 +45,8 @@ def create_app(data_path): ...@@ -44,8 +45,8 @@ def create_app(data_path):
init_data_directory(data_path) init_data_directory(data_path)
# Load the secret key # Load the secret key
with open(os.path.join(data_path, "secret_key")) as f: with open(os.path.join(data_path, "secret_key")) as fn:
app.secret_key = f.read().strip() app.secret_key = fn.read().strip()
# Initialize the database # Initialize the database
app.db = db.Database(os.path.join(data_path, "database.db")) app.db = db.Database(os.path.join(data_path, "database.db"))
...@@ -79,14 +80,14 @@ def create_app(data_path): ...@@ -79,14 +80,14 @@ def create_app(data_path):
return app return app
def init_data_directory(data_path): def init_data_directory(data_path: str):
"""Initialize the data directory""" """Initialize the data directory"""
src_directory = os.path.dirname(os.path.abspath(__file__)) src_directory = os.path.dirname(os.path.abspath(__file__))
# Create all the directories # Create all the directories
dirs = ["", "cache"] dirs = ["", "cache"]
for dir in dirs: for _dir in dirs:
os.makedirs(os.path.join(data_path, dir), exist_ok=True) os.makedirs(os.path.join(data_path, _dir), exist_ok=True)
# Initialize the cache # Initialize the cache
static_dirs = {"static": "+assets"} static_dirs = {"static": "+assets"}
...@@ -105,6 +106,10 @@ def init_data_directory(data_path): ...@@ -105,6 +106,10 @@ def init_data_directory(data_path):
f.write("%s\n" % utils.random_key(64)) f.write("%s\n" % utils.random_key(64))
os.chmod(secret_key_path, 0o400) os.chmod(secret_key_path, 0o400)
def init_data_downloads(data_path, debug=False):
# Initialize the download files # Initialize the download files
download_inst = download.Downloads(data_path) download_inst = download.Downloads(data_path, debug=debug)
download_inst.store_cache_file() if debug is False:
# mk cache file
download_inst.store_cache_file()
# Configurazione delle release disponibili
# Quando viene rilasciata una release o una point release è solamente
# necessario aggiornare questa sezione.
[releases.latest]
version = "21.10"
codename = "impish"
lts = false
[releases.lts]
version = "22.04.3"
codename = "jammy"
lts = true
# Distribuzioni disponibili per il download
# Tutte le pagine in /download sono generate automaticamente da questa lista.
#
# name: Il nome della distro
# description: Una breve descrizione della distro
# lts-support-years: Il numero di anni per cui la LTS è supportata
# lts-only: Se impostato a `true`, solo le LTS verranno offerte
# releases: Una lista di release da offrire -- viene filtrata da lts-only
# archs: Una tabella di architettura e sorgente delle ISO
#
# Se è necessario sovrascrivere la sorgente delle ISO per una singola coppia
# release-architettura, è possibile aggiungere la chiave:
#
# source-override-RELEASE-ARCH = "SOURCE"
[distros.desktop]
name = "Ubuntu"
description = "L'originale, con GNOME"
lts-support-years = 5
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "releases-ubuntu" }
[distros.live-server]
name = "Ubuntu Server"
description = "Tutta la potenza di Ubuntu nel tuo server"
lts-support-years = 5
lts-only = true
releases = ["lts", "latest"] # Invertiti intenzionalmente
# amd64 è su releases.u.c, mentre arm64 e ppc64el sono su cdimages.u.c
archs = { amd64 = "releases-ubuntu", arm64 = "cdimages-ubuntu", ppc64el = "cdimages-ubuntu" }
[distros.kubuntu]
name = "Kubuntu"
description = "L'esperienza Ubuntu con desktop KDE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.lubuntu]
name = "Lubuntu"
description = "La derivata più leggera, con LXDE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntu-budgie]
name = "Ubuntu Budgie"
description = "La potenza di Ubuntu e la leggerezza di Budgie"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntukylin]
name = "Ubuntu Kylin"
description = "La derivata di Ubuntu con desktop UKUI"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntu-mate]
name = "Ubuntu MATE"
description = "Ubuntu si unisce a MATE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
[distros.ubuntustudio]
name = "Ubuntu Studio"
description = "La derivata di Ubuntu dedicata alla multimedialità"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-studio" }
[distros.xubuntu]
name = "Xubuntu"
description = "La derivata di Ubuntu leggera ma personalizzabile, con desktop XFCE"
lts-support-years = 3
lts-only = true
releases = ["latest", "lts"]
archs = { amd64 = "cdimages-derivatives" }
# Nomi delle architetture
# Vengono usati per scegliere l'architettura da scaricare
[archs]
amd64 = "64bit"
#i386 = "32bit (computer più datati)"
arm64 = "ARM 64bit"
ppc64el = "POWER"
# Tabella delle sorgenti delle ISO
# Le distro possono scegliere da quale sorgente prendere ogni architettura
[sources.releases-ubuntu]
http = "{mirror:ubuntu}/{codename}/ubuntu-{version}-{distro}-{arch}.iso"
torrent = "{mirror:ubuntu}/{codename}/ubuntu-{version}-{distro}-{arch}.iso.torrent"
[sources.cdimages-ubuntu]
http = "http://cdimages.ubuntu.com/releases/{codename}/release/ubuntu-{version}-{distro}-{arch}.iso"
torrent = "http://cdimages.ubuntu.com/releases/{codename}/release/ubuntu-{version}-{distro}-{arch}.iso.torrent"
[sources.cdimages-derivatives]
http = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso"
torrent = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-desktop-{arch}.iso.torrent"
[sources.cdimages-studio]
http = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-dvd-{arch}.iso"
torrent = "http://cdimages.ubuntu.com/{distro}/releases/{codename}/release/{distro}-{version}-dvd-{arch}.iso.torrent"
# Configurazione dei mirror
# I mirror vengono caricati da Launchpad in automatico
[mirrors]
# apparentemente non ci sono più mirror ufficiali in Italia ☹
country = "DE"
for = ["ubuntu"]
# Source code of the Ubuntu-it website # Source code of the Ubuntu-it website
# Copyright (C) 2016 Pietro Albini <pietroalbini@ubuntu.com> # Copyright (C) 2016 Pietro Albini <pietroalbini@ubuntu.com>
# Copyright (C) 2023 shadMod
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published # it under the terms of the GNU Affero General Public License as published
...@@ -29,16 +30,20 @@ from . import cache ...@@ -29,16 +30,20 @@ from . import cache
from . import launchpad from . import launchpad
from .constants import UITWWW_DIR from .constants import UITWWW_DIR
CONFIG_FILE = "/data/downloads.toml" DOWNLOAD_FILE = "downloads.toml"
CACHE_FILE = "download-cache.json" CACHE_FILE = "download-cache.json"
CACHE_FILE_VERSION = 1 CACHE_FILE_VERSION = 1
class Downloads: class Downloads:
def __init__(self, data_path): def __init__(self, data_path, debug: bool = False):
# init _mirrors and _sha256sums
self._mirrors = None
self._sha256sums = None
# Load the configuration # Load the configuration
path = UITWWW_DIR + CONFIG_FILE path = os.path.join(data_path, DOWNLOAD_FILE)
self.config = toml.load( self.config = toml.load(
path, _dict=collections.OrderedDict, path, _dict=collections.OrderedDict,
) )
...@@ -49,7 +54,8 @@ class Downloads: ...@@ -49,7 +54,8 @@ class Downloads:
with open(path, "rb") as raw: with open(path, "rb") as raw:
self._config_hash = "sha1=%s" % hashlib.sha1(raw.read()).hexdigest() self._config_hash = "sha1=%s" % hashlib.sha1(raw.read()).hexdigest()
self._cache_file = os.path.join(data_path, CACHE_FILE) if debug is False:
self._cache_file = os.path.join(data_path, CACHE_FILE)
@property @property
def _cache(self): # Cacheception def _cache(self): # Cacheception
...@@ -61,7 +67,7 @@ class Downloads: ...@@ -61,7 +67,7 @@ class Downloads:
@property @property
def mirrors(self): def mirrors(self):
"""Get a list of CD mirrors needed by the website""" """Get a list of CD mirrors needed by the website"""
if not hasattr(self, "_mirrors"): if not hasattr(self, "_mirrors") or self._mirrors is None:
if self._cache is not None: if self._cache is not None:
self._mirrors = self._cache["mirrors"] self._mirrors = self._cache["mirrors"]
else: else:
...@@ -70,6 +76,8 @@ class Downloads: ...@@ -70,6 +76,8 @@ class Downloads:
found_mirrors = list(sorted(launchpad.get_cdimage_mirrors( found_mirrors = list(sorted(launchpad.get_cdimage_mirrors(
distro, self.config["mirrors"]["country"] distro, self.config["mirrors"]["country"]
))) )))
if "it1.mirror.vhosting-it.com" in found_mirrors[0]:
found_mirrors = ["https://releases.ubuntu.com/"]
if found_mirrors: if found_mirrors:
self._mirrors[distro] = found_mirrors self._mirrors[distro] = found_mirrors
if not self._mirrors: if not self._mirrors:
...@@ -80,14 +88,14 @@ class Downloads: ...@@ -80,14 +88,14 @@ class Downloads:
return self._mirrors return self._mirrors
@property @property
def md5sums(self): def sha256sums(self):
"""Get a list of all the MD5SUMS""" """Get a list of all the SHA256SUMS"""
if not hasattr(self, "_md5sums"): if getattr(self, "_sha256sums") is None:
if self._cache is not None: if self._cache is not None:
self._md5sums = self._cache["md5sums"] self._sha256sums = self._cache["sha256sums"]
else: else:
self._md5sums = self._fetch_md5sums() self._sha256sums = self._fetch_sha256sums()
return self._md5sums return self._sha256sums
def _strip_non_lts_releases(self): def _strip_non_lts_releases(self):
"""Process the lts-only distro configuration""" """Process the lts-only distro configuration"""
...@@ -99,14 +107,14 @@ class Downloads: ...@@ -99,14 +107,14 @@ class Downloads:
if not self.config["releases"][release]["lts"]: if not self.config["releases"][release]["lts"]:
config["releases"].remove(release) config["releases"].remove(release)
def _fetch_md5sums(self): def _fetch_sha256sums(self):
"""Fetch all the needed MD5SUMS""" """Fetch all the needed SHA256SUMS"""
result = {} result = {}
files_content = {} files_content = {}
for distro, config in self.config["distros"].items(): for distro, config in self.config["distros"].items():
for release in config["releases"]: for release in config["releases"]:
if config["lts-only"] and not self.config["releases"][release]["lts"]: if config["lts-only"]:
continue continue
for arch in config["archs"].keys(): for arch in config["archs"].keys():
...@@ -115,7 +123,7 @@ class Downloads: ...@@ -115,7 +123,7 @@ class Downloads:
# Fetch the file from the remote if it wasn't done already # Fetch the file from the remote if it wasn't done already
if path not in files_content: if path not in files_content:
md5s = {} sha_hash = {}
response = requests.get("%s/SHA256SUMS" % path) response = requests.get("%s/SHA256SUMS" % path)
if response.status_code == 404: if response.status_code == 404:
...@@ -127,19 +135,19 @@ class Downloads: ...@@ -127,19 +135,19 @@ class Downloads:
if line.strip() == "": if line.strip() == "":
continue continue
hash, name = line.split(" ", 1) _hash, name = line.split(" ", 1)
if name.startswith("*"): if name.startswith("*"):
name = name[1:] name = name[1:]
md5s[name] = hash sha_hash[name] = _hash
files_content[path] = md5s files_content[path] = sha_hash
# Add the MD5 to the result if it was found # Add the SHA256 to the result if it was found
if file in files_content[path]: if file in files_content[path]:
key = "%s:%s:%s" % (distro, release, arch) key = "%s:%s:%s" % (distro, release, arch)
result[key] = files_content[path][file] result[key] = files_content[path][file]
else: else:
print(files_content[path]) print(files_content[path])
raise RuntimeError("Missing MD5 for {}".format(file)) raise RuntimeError("Missing SHA256SUMS for {}".format(file))
return result return result
...@@ -173,7 +181,7 @@ class Downloads: ...@@ -173,7 +181,7 @@ class Downloads:
"version": CACHE_FILE_VERSION, "version": CACHE_FILE_VERSION,
"content": { "content": {
"mirrors": self.mirrors, "mirrors": self.mirrors,
"md5sums": self.md5sums, "sha256sums": self.sha256sums,
}, },
}, f, separators=(",", ":")) }, f, separators=(",", ":"))
...@@ -183,12 +191,13 @@ class Downloads: ...@@ -183,12 +191,13 @@ class Downloads:
def url_for(self, distro, release, arch, torrent=False, use_random=True): def url_for(self, distro, release, arch, torrent=False, use_random=True):
"""Get the URL of a download file""" """Get the URL of a download file"""
# Build the dict of keys to replace # Build the dict of keys to replace
data_rel = self.config["distros"][distro]["archs"][arch][release]
replaces = { replaces = {
"distro": distro, "distro": distro,
"release": release, "release": release,
"arch": arch, "arch": arch,
"version": self.config["releases"][release]["version"], "version": data_rel["version"],
"codename": self.config["releases"][release]["codename"], "codename": data_rel["codename"],
} }
# Add mirrors to the replaces key # Add mirrors to the replaces key
...@@ -199,14 +208,14 @@ class Downloads: ...@@ -199,14 +208,14 @@ class Downloads:
replaces["mirror:%s" % name] = choices[0] replaces["mirror:%s" % name] = choices[0]
# Get the right source # Get the right source
source_name = self.config["distros"][distro]["archs"][arch] host = data_rel["host"]
source = self.config["sources"][source_name] source = self.config["sources"][host]
# Download URLs are different for torrent and HTTP # Download URLs are different for torrent and HTTP
if torrent: if torrent:
url = source["torrent"] url = source["torrent"]
else: else:
url = source["http"] url = source["https"]
for key, value in replaces.items(): for key, value in replaces.items():
url = url.replace("{%s}" % key, value) url = url.replace("{%s}" % key, value)
...@@ -233,11 +242,12 @@ class Downloads: ...@@ -233,11 +242,12 @@ class Downloads:
if distro not in self.config["distros"]: if distro not in self.config["distros"]:
flask.abort(404) flask.abort(404)
data = self.config["distros"][distro]
return flask.render_template( return flask.render_template(
"download/landing.html", "download/landing.html",
distro_name=distro, distro_name=distro,
distro=self.config["distros"][distro], distro=data,
releases=self.config["releases"], releases=data["archs"]["amd64"],
archs=self.config["archs"], archs=self.config["archs"],
) )
...@@ -261,16 +271,19 @@ class Downloads: ...@@ -261,16 +271,19 @@ class Downloads:
torrent = "torrent" in flask.request.form torrent = "torrent" in flask.request.form
payload = signer.dumps({ data_rel = self.config["distros"][distro]["archs"][arch][release]
"url": self.url_for(distro, release, arch, torrent), payload = signer.dumps(
"md5": self.md5sums["%s:%s:%s" % (distro, release, arch)], {
"name": "%s %s%s" % ( "url": self.url_for(distro, release, arch, torrent),
self.config["distros"][distro]["name"], "sha256": self.sha256sums[f"{distro}:{release}:{arch}"],
self.config["releases"][release]["version"], "name": "%s %s%s" % (
" LTS" if self.config["releases"][release]["lts"] else "", self.config["distros"][distro]["name"],
), data_rel["version"],
"theme": distro, " LTS" if data_rel["lts"] else "",
}) ),
"theme": distro,
}
)
return flask.redirect(flask.url_for(".thanks", payload=payload)) return flask.redirect(flask.url_for(".thanks", payload=payload))
@bp.route("/grazie/<payload>") @bp.route("/grazie/<payload>")
...@@ -288,10 +301,11 @@ class Downloads: ...@@ -288,10 +301,11 @@ class Downloads:
"""Generate the sub-navbar for /download""" """Generate the sub-navbar for /download"""
result = [] result = []
for name, distro in self.config["distros"].items(): for name, distro in self.config["distros"].items():
result.append({ result.append(
"name": distro["name"], {
"endpoint": "download.landing", "name": distro["name"],
"endpoint-args": {"distro": name}, "endpoint": "download.landing",
}) "endpoint-args": {"distro": name},
}
)
return result return result
# Source code of the Ubuntu-it website # Source code of the Ubuntu-it website
# Copyright (C) 2015-2016 Pietro Albini <pietroalbini@ubuntu.com> # Copyright (C) 2015-2016 Pietro Albini <pietroalbini@ubuntu.com>
# Copyright (C) 2023 shadMod
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published # it under the terms of the GNU Affero General Public License as published
...@@ -19,28 +20,31 @@ import os ...@@ -19,28 +20,31 @@ import os
import click import click
import uitwww import uitwww
from scss.compiler import compile_file
from .constants import UITWWW_DIR, BASE_DIR from .constants import UITWWW_DIR, BASE_DIR
from .utils import ReverseProxied, GunicornInstance from .utils import ReverseProxied, GunicornInstance
from scss.compiler import compile_file from .src.download.compile_download import CompileVersion
@click.group() @click.group()
def cli(): def cli():
""" Ubuntu-it website's command line utility """ """Ubuntu-it website's command line utility"""
pass pass
@cli.command("run") @cli.command("run")
@click.argument("data") @click.argument("data")
@click.option("-g", "--gunicorn-config", default=None, help="Path to a" @click.option(
"gunicorn config file") "-g", "--gunicorn-config", default=None, help="Path to a gunicorn config file"
)
@click.option("-p", "--port", default=8000, help="Bind that port") @click.option("-p", "--port", default=8000, help="Bind that port")
@click.option("--public", help="Make available to the public", is_flag=True) @click.option("--public", help="Make available to the public", is_flag=True)
@click.option("-w", "--workers", help="Number of workers to start", default=3) @click.option("-w", "--workers", help="Number of workers to start", default=3)
@click.option("-d", "--debug", help="Enable debug mode", is_flag=True) @click.option("-d", "--debug", help="Enable debug mode", is_flag=True)
def run(data, gunicorn_config, port, public, workers, debug): def run(data, gunicorn_config, port, public, workers, debug):
"""Run the application""" """Run the application"""
app = uitwww.create_app(data) app = uitwww.create_app(data, debug)
app.wsgi_app = ReverseProxied(app.wsgi_app) app.wsgi_app = ReverseProxied(app.wsgi_app)
host = "127.0.0.1" host = "127.0.0.1"
...@@ -76,11 +80,21 @@ def run(data, gunicorn_config, port, public, workers, debug): ...@@ -76,11 +80,21 @@ def run(data, gunicorn_config, port, public, workers, debug):
@cli.command("init") @cli.command("init")
@click.argument("data") @click.argument("data")
def init(data): @click.option("-d", "--debug", help="Enable debug mode", is_flag=True)
def init(data, debug):
"""Initialize the data directory""" """Initialize the data directory"""
print("Initializing data directory:", data)
data_path = os.path.expanduser(os.path.abspath(data)) data_path = os.path.expanduser(os.path.abspath(data))
print("Initializing data directory:", data)
uitwww.init_data_directory(data_path) uitwww.init_data_directory(data_path)
print("Created data directory:", data)
compiler = CompileVersion(
path_out=data_path + "/downloads.toml",
constants=UITWWW_DIR + "/src/download/assets/constants.json",
)
compiler.compile_download()
print("Compile download file")
uitwww.init_data_downloads(data_path, debug)
print("Populate data download:", data)
@cli.command("build_scss") @cli.command("build_scss")
......
# Source code of the Ubuntu-it website
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; witout even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Source code of the Ubuntu-it website
# Copyright (C) 2023 shadMod
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; witout even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
{
"archs": {
"amd64": "64bit",
"arm64": "ARM 64bit",
"ppc64el": "PowerPC64"
},
"mirrors": {
"country": "IT",
"for": [
"ubuntu"
]
},
"distros": {
"desktop": {
"name": "Ubuntu",
"description": "L'originale, con GNOME"
},
"live-server": {
"name": "Ubuntu Server",
"description": "Tutta la potenza di Ubuntu nel tuo server",
"add_archs": [
"arm64",
"ppc64el"
]
},
"kubuntu": {
"name": "Kubuntu",
"description": "L'esperienza Ubuntu con desktop KDE"
},
"lubuntu": {
"name": "Lubuntu",
"description": "La derivata pi\u00f9 leggera, con LXDE"
},
"ubuntu-budgie": {