init, current status

Signed-off-by: Mattia Rizzolo <mapreri@ubuntu.com>

cron.sh

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+
+# Allow this script to be run only by ubuntu-it
+if [[ "$(whoami)" != "ubuntu-it" ]]; then
+    echo "Error: this script must be executed by the ubuntu-it user"
+    exit 1
+fi
+
+chronic_sh (){
+    local pipe pipe_q
+    pipe=$(mktemp -p "${TMPDIR:-/tmp}" chronic_sh.XXXXXXXXXX)
+    printf -v pipe_q '%q' "$pipe"
+    trap 'rm -fv '"$pipe_q" RETURN
+    # one can't open a reading fd and a writing fd without blocking, because
+    # they want to already have something on the other side of the pipe.
+    # the temporary fd 5 will be that something.
+    exec 5<>"$pipe" # hack
+    exec 3>"$pipe"  # writing fd
+    exec 4<"$pipe"  # reading fd
+    exec 5>&-       # end hack
+    rm "$pipe"
+
+    local ret=0
+    "$@" >&3 2>&3 || ret=$?
+    if [ "$ret" -ne 0 ] ; then
+        exec 3>&-
+        cat <&4-
+        return $ret
+    fi
+}
+
+# Try to deploy all the websites
+for script in "${base}/scripts/"*; do
+    if [[ ! -x "${script}" ]]; then
+        continue
+    fi
+
+    # Create a temporary build directory
+    build_dir="$(mktemp -d deploy-XXXXXXXXX -p /tmp)"
+
+    # Switch to the temporary build directory
+    cd "${build_dir}"
+    export HOME="${build_dir}"
+    TMPDIR="$HOME"
+    output="$(mktemp -p "${TMPDIR:-/tmp}" deploy_output.XXXXXXXX)"
+
+    # Execute the deploy script
+    # Capture the output and print it only if the script fails
+    "$script" > "$output"
+    if [[ $? -ne 0 ]]; then
+        echo "Deploy script $(basename "${script}") failed!"
+        echo "Script output"
+        echo "============="
+        cat "${output}"
+        echo
+    fi
+
+    # Restore the old home
+    cd "${base}"
+    export HOME="${base}"
+
+    # Remove the temporary directory
+    rm -rf "${build_dir}"
+
+done

scripts/2013.ducc.it

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export YEAR="2013"
+export NAME="${YEAR}.ducc.it"
+export REPO_URL="https://gitlab.com/ducc-it/websites-archive"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+repo="$(git_clone_repo "${NAME}" "${REPO_URL}")"
+
+if git_needs_update "${NAME}"; then
+    deploy="$(deploy_new "${NAME}")"
+    cp -r "/srv/${NAME}/repo/${YEAR}"/* "${deploy}"
+
+    git_save_revision "${NAME}" "${deploy}"
+    deploy_switch "${NAME}" "${deploy}"
+fi
+

scripts/2014.ducc.it

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export YEAR="2014"
+export NAME="${YEAR}.ducc.it"
+export REPO_URL="https://gitlab.com/ducc-it/websites-archive"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+repo="$(git_clone_repo "${NAME}" "${REPO_URL}")"
+
+if git_needs_update "${NAME}"; then
+    deploy="$(deploy_new "${NAME}")"
+    cp -r "/srv/${NAME}/repo/${YEAR}"/* "${deploy}"
+
+    git_save_revision "${NAME}" "${deploy}"
+    deploy_switch "${NAME}" "${deploy}"
+fi
+

scripts/2015.ducc.it

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export YEAR="2015"
+export NAME="${YEAR}.ducc.it"
+export REPO_URL="https://gitlab.com/ducc-it/websites-archive"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+repo="$(git_clone_repo "${NAME}" "${REPO_URL}")"
+
+if git_needs_update "${NAME}"; then
+    deploy="$(deploy_new "${NAME}")"
+    cp -r "/srv/${NAME}/repo/${YEAR}"/* "${deploy}"
+
+    git_save_revision "${NAME}" "${deploy}"
+    deploy_switch "${NAME}" "${deploy}"
+fi
+

scripts/cerca.ubuntu-it.org

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export NAME="cerca.ubuntu-it.org"
+export REPO_URL="git@code.ubuntu-it.org:ubuntu-it-web/cerca.git"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+macros_static_website_git "${NAME}" "${REPO_URL}"

scripts/chat.ubuntu-it.org

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export NAME="chat.ubuntu-it.org"
+export REPO_URL="git@code.ubuntu-it.org:ubuntu-it-web/chat.git"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+macros_static_website_git "${NAME}" "${REPO_URL}"

scripts/forum.ubuntu-it.org

+#!/bin/sh
+
+set -eu
+
+cd /srv/forum.ubuntu-it.org/www-dev/styles/relight
+
+git pull --ff-only

scripts/help.ubuntu-it.org

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export NAME="help.ubuntu-it.org"
+export REPO_URL="git@code.ubuntu-it.org:ubuntu-it-doc/help.git"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+macros_static_website_git "${NAME}" "${REPO_URL}"

scripts/planet.ubuntu-it.org

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export NAME="planet.ubuntu-it.org"
+export REPO_URL="git@code.ubuntu-it.org:ubuntu-it-web/planet.git"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+# Ensure the latest source code is present
+repo="$(git_clone_repo "${NAME}" "${REPO_URL}")"
+
+# Execute the planet build
+make -C "${repo}" update build
+
+# Create a new deploy
+deploy="$(deploy_new "${NAME}")"
+cp -r "${repo}/build/"* "${deploy}"
+rm -rf "${repo}/build"
+
+deploy_switch "${NAME}" "${deploy}"

scripts/static.ubuntu-it.org

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+set -x
+
+export NAME="static.ubuntu-it.org"
+export REPO_URL="git@code.ubuntu-it.org:ubuntu-it-web/static.git"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+repo="$(git_clone_repo "${NAME}" "${REPO_URL}")"
+
+if git_needs_update "${NAME}"; then
+
+    # Build the repository
+    make -C "$repo" clean
+    make -C "$repo"
+
+    # Create a new deploy
+    deploy="$(deploy_new "${NAME}")"
+    cp -r "$repo"/build/* "${deploy}"
+
+    git_save_revision "${NAME}" "${deploy}"
+    deploy_switch "${NAME}" "${deploy}"
+fi

scripts/stats.ubuntu-it.org

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+set -euo pipefail
+
+NAME="stats.ubuntu-it.org"
+LOGS_DIR="/var/log/apache2"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+today="$(date +"%Y-%m-%d")"
+today_file="${DIR_BASE}/${NAME}/current/.day"
+if [[ -f "${today_file}" ]]; then
+    if [[ "$(cat "${today_file}")" = "${today}" ]]; then
+        echo "Already built stats for today -- skipping."
+        exit
+    fi
+fi
+
+deploy="$(deploy_new "${NAME}")"
+echo "${today}" > "${deploy}/.day"
+echo "Options Indexes FollowSymLinks MultiViews" > "${deploy}/.htaccess"
+echo "IndexOptions FancyIndexing" >> "${deploy}/.htaccess"
+
+for dir in $(ls "${LOGS_DIR}"); do
+    if [[ -d "${LOGS_DIR}/${dir}" ]]; then
+        echo "Generating stats for ${dir}..."
+        # blackhole stderr because that's where the progress counter goes,
+        # filling up the log/screen/terminal
+        zgrep -h "" "${LOGS_DIR}/${dir}"/access* | goaccess - \
+            --log-format=COMBINED --http-protocol=no --ignore-panel=HOSTS \
+            --all-static-files --ignore-crawlers --real-os \
+            > "${deploy}/${dir}.html" 2> /dev/null
+    fi
+done
+
+deploy_switch "${NAME}" "${deploy}"

scripts/wiki.ubuntu-it.org

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export NAME="wiki.ubuntu-it.org"
+export REPO_URL="lp:~ubuntu-it-wiki/wiki-ubuntu-it/wiki-repo"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+repo="$(bzr_clone_repo "${NAME}" "${REPO_URL}")"
+
+if bzr_needs_update "${NAME}"; then
+    deploy="$(deploy_new "${NAME}")"
+
+    cp -r "${repo}/data/plugin" "${deploy}"
+    cp -r "${repo}/htdocs" "${deploy}"
+    cp -r "${repo}/wikiconfig.py" "${deploy}"
+
+    bzr_save_revision "${NAME}" "${deploy}"
+    deploy_switch "${NAME}" "${deploy}"
+fi

scripts/www.ducc.it

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export NAME="www.ducc.it"
+export REPO_URL="https://gitlab.com/ducc-it/website"
+
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+. "${base}/../utils/include.sh"
+
+
+repo="$(git_clone_repo "${NAME}" "${REPO_URL}")"
+
+if git_needs_update "${NAME}"; then
+
+    # Build the repository
+    cp -r "${repo}" repo
+    make -C repo
+
+    # Create a new deploy
+    deploy="$(deploy_new "${NAME}")"
+    cp -r repo/build/html/* "${deploy}"
+
+    git_save_revision "${NAME}" "${deploy}"
+    deploy_switch "${NAME}" "${deploy}"
+fi
+

utils/bzr.sh

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+
+# Clone a bazaar repository in /srv/{name}/repo
+#
+# $ bzr_clone_repo www.ubuntu-it.org lp:ubuntu-it-www
+bzr_clone_repo() {
+    name="$1"
+    url="$2"
+
+    dest="${DIR_BASE}/${name}/repo"
+
+    if [[ ! -d "${dest}" ]]; then
+        bzr branch "${url}" "${dest}" -q
+    else
+        (cd "${dest}"; bzr pull -q)
+    fi
+}
+
+
+# Check if a new deploy for the service is needed
+#
+# $ bzr_needs_update www.ubuntu-it.org
+bzr_needs_update() {
+    name="$1"
+
+    repo="${DIR_BASE}/${name}/repo"
+    current="${DIR_BASE}/${name}/current/.revision"
+
+    if [[ ! -f "${current}" ]]; then
+        return 0
+    fi
+
+    latest="$(cd "${repo}"; bzr revno)"
+
+    if [[ "$(cat "${current}")" != "${latest}" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+
+# Save the revision for this service in the provided build
+#
+# $ bzr_save_revision www.ubuntu-it.org /srv/www.ubuntu-it.org/builds/XXXXXX
+bzr_save_revision() {
+    name="$1"
+    build="$2"
+
+    repo="${DIR_BASE}/${name}/repo"
+    dest="${build}/.revision"
+
+    (cd "${repo}"; bzr revno) > "${dest}"
+}

utils/deploy.sh

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+
+# Create a new deploy for the service, echoing its path
+#
+# $ deploy_new www.ubuntu-it.org
+# /srv/www.ubuntu-it.org/builds/XXXXXX
+deploy_new() {
+    name="$1"
+
+    now="$(date +"%Y-%m-%d--%H-%M-%S")"
+
+    build_dir="${DIR_BASE}/${name}/builds/${now}"
+    mkdir -p "${build_dir}"
+
+    echo "${build_dir}"
+}
+
+
+# Switch to a different deploy, cleaning up old ones
+#
+# $ deploy_switch www.ubuntu-it.org /srv/www.ubuntu-it.org/builds/XXXXXX
+deploy_switch() {
+    name="$1"
+    build="$2"
+
+    base="${DIR_BASE}/${name}"
+    current="${base}/current"
+
+    # Switch the current/ symlink to the new build
+    ln -sfT "${build}" "${current}"
+
+    # Remove the oldest builds
+    directories=($(ls -t -d -1 "${base}/builds"/????-??-??--??-??-?? | awk "NR>${KEEP_BUILDS}"))
+    for dir in "${directories[@]}"; do
+        # Avoid deleting the current build
+        if [[ "${dir}" != "${build}" ]]; then
+            rm -rf "${dir}"
+        fi
+    done
+}

utils/git.sh

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+
+# Clone a git repository in /srv/{name}/repo
+#
+# $ git_clone_repo static.ubuntu-it.org git@code.ubuntu-it.org:ubuntu-it-web/static
+git_clone_repo() {
+    name="$1"
+    url="$2"
+
+    dest="${DIR_BASE}/${name}/repo"
+
+    if [[ ! -d "${dest}" ]]; then
+        git clone "${url}" "${dest}" -q
+    else
+        (cd "${dest}"; git pull -q)
+    fi
+
+    echo "${dest}"
+}
+
+
+# Check if a new deploy for the service is needed
+#
+# $ git_needs_update www.ubuntu-it.org
+git_needs_update() {
+    name="$1"
+
+    repo="${DIR_BASE}/${name}/repo"
+    current="${DIR_BASE}/${name}/current/.commit"
+
+    if [[ ! -f "${current}" ]]; then
+        return 0
+    fi
+
+    latest="$(cd ${repo}; git rev-parse master)"
+
+    if [[ "$(cat "${current}")" != "${latest}" ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+
+# Save the revision for this service in the provided build
+#
+# $ git_save_revision www.ubuntu-it.org /srv/www.ubuntu-it.org/builds/XXXXXX
+git_save_revision() {
+    name="$1"
+    build="$2"
+
+    repo="${DIR_BASE}/${name}/repo"
+    dest="${build}/.commit"
+
+    (cd "${repo}"; git rev-parse master) > "${dest}"
+}

utils/include.sh

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+export DIR_BASE="/srv"
+export KEEP_BUILDS=3
+
+# Bash strict mode
+set -euo pipefail
+IFS=$'\n\t'
+
+# Allow this script to be run only by ubuntu-it
+if [[ "$(whoami)" != "ubuntu-it" ]]; then
+    echo "Error: this script must be executed by the ubuntu-it user"
+    exit 1
+fi
+
+base="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )"
+
+# Include all the utilities from the subfiles
+. "${base}/bzr.sh"
+. "${base}/git.sh"
+. "${base}/deploy.sh"
+. "${base}/macros.sh"
+
+unset base

utils/macros.sh

+#!/bin/bash
+# Made by Pietro Albini <pietroalbini@ubuntu-it.org>
+
+
+# Do everything needed for a static website on bzr
+#
+# $ macros_static_website_bzr chat.ubuntu-it.org lp:~ubuntu-it-www/ubuntu-it-www/chat
+macros_static_website_bzr() {
+    name="$1"
+    repo_url="$2"
+
+    repo="$(bzr_clone_repo "${name}" "${repo_url}")"
+
+    if bzr_needs_update "${name}"; then
+        deploy="$(deploy_new "${name}")"
+        cp -r "${repo}/www"/* "${deploy}"
+
+        bzr_save_revision "${name}" "${deploy}"
+        deploy_switch "${name}" "${deploy}"
+    fi
+}
+
+
+# Do everything needed for a static website on git
+#
+# $ macros_static_website_git chat.ubuntu-it.org git@code.ubuntu-it.org:ubuntu-it-web/chat.git
+macros_static_website_git() {
+    name="$1"
+    repo_url="$2"
+
+    repo="$(git_clone_repo "${name}" "${repo_url}")"
+
+    if git_needs_update "${name}"; then
+        deploy="$(deploy_new "${name}")"
+        cp -r "${repo}/www"/* "${deploy}"
+
+        git_save_revision "${name}" "${deploy}"
+        deploy_switch "${name}" "${deploy}"
+    fi
+}