From fd4c7d2dd302cd64cec57c74536d4dbcada2941d Mon Sep 17 00:00:00 2001 From: Harald Pfeiffer Date: Sun, 23 Mar 2025 07:41:16 +0100 Subject: Rename repo to packages.lirion.de, move aptly stuff to aptly subfolder --- Makefile | 21 ----- aptly/Makefile | 21 +++++ aptly/bin/aptly-lirionde | 213 ++++++++++++++++++++++++++++++++++++++++++ aptly/etc/aptly-lirionde.conf | 18 ++++ bin/aptly-lirionde | 213 ------------------------------------------ etc/aptly-lirionde.conf | 18 ---- 6 files changed, 252 insertions(+), 252 deletions(-) delete mode 100644 Makefile create mode 100644 aptly/Makefile create mode 100755 aptly/bin/aptly-lirionde create mode 100644 aptly/etc/aptly-lirionde.conf delete mode 100755 bin/aptly-lirionde delete mode 100644 etc/aptly-lirionde.conf diff --git a/Makefile b/Makefile deleted file mode 100644 index bd29cbb..0000000 --- a/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -all: - - -install: copy-etc copy-bin - - -copy-etc: mkdir-etc - @if [ ! -e /etc/aptly-lirionde/aptly.conf ]; then \ - install -vpm0644 -oroot -groot -t /etc/lirion/aptly.conf ./etc/aptly-lirionde.conf; \ - else printf '/etc/lirion/aptly.conf existing, skipping.\n'; \ - fi - -mkdir-etc: - @install -dvm0755 -oroot -groot /etc/lirion - - -copy-bin: - @install -vpm0755 -oroot -groot -t /usr/bin bin/aptly-lirionde - - -.PHONY: mkdir-etc copy-etc copy-bin diff --git a/aptly/Makefile b/aptly/Makefile new file mode 100644 index 0000000..bd29cbb --- /dev/null +++ b/aptly/Makefile @@ -0,0 +1,21 @@ +all: + + +install: copy-etc copy-bin + + +copy-etc: mkdir-etc + @if [ ! -e /etc/aptly-lirionde/aptly.conf ]; then \ + install -vpm0644 -oroot -groot -t /etc/lirion/aptly.conf ./etc/aptly-lirionde.conf; \ + else printf '/etc/lirion/aptly.conf existing, skipping.\n'; \ + fi + +mkdir-etc: + @install -dvm0755 -oroot -groot /etc/lirion + + +copy-bin: + @install -vpm0755 -oroot -groot -t /usr/bin bin/aptly-lirionde + + +.PHONY: mkdir-etc copy-etc copy-bin diff --git a/aptly/bin/aptly-lirionde b/aptly/bin/aptly-lirionde new file mode 100755 index 0000000..8933cd3 --- /dev/null +++ b/aptly/bin/aptly-lirionde @@ -0,0 +1,213 @@ +#!/usr/bin/env bash + +# Manages aptly input repositories and publishes a merged output repository. +# Structure is as follows: +# repo1 ───> snapshot "repo1-%Y-%m-%d" ─┬─> snapshot "%Y-%m-%d" ───> publish inside $TBASE/public +# │ +# repo2 ───> snapshot "repo2-%Y-%m-%d" ─┘ +# +# Furthermore, our PACKAGE input folder is /tmp/aptly with repository names as subfolder, so if +# you want a new package added inside repo1, you would place a file inside /tmp/aptly/repo1/. +# The package would then be added and its file inside /tmp be removed. +# +# Why bash? +# 1. This script is INTERACTIVE, it asks for your GPG passphrase before signing packages +# 2. It still uses a lot of syscalls like aptly +# 3. The steps we do are still not that resource-hungry, so the downsides while using bash are rather low +# The combination of these three led to bash in the first iteration. +# +# What doesn't this script do? +# 1. It does not create the source repos or the aptly config. +# 2. It does not repair if one of the parts of the structure above is missing; in that case, +# we assume an extraordinary failure and will fail ourselves. Exception: We skip failing +# on removal if a published repo or a snapshot does not exist. +# 3. It does not manage multiple snapshots yet. Nor does it cleanup snapshot remainders in case the script +# is interrupted. Both can be considered a TODO. + + +MYCONF="/etc/lirion/aptly.conf" +if [ ! -r "$MYCONF" ];then + printf '%b cannot be read, exiting!\n' "$MYCONF" >&2 + exit 254 +else + # shellcheck disable=SC1091,SC1090 + source "$MYCONF" || exit 254 +fi + +MALFORMED=0 +[ -z "$MYREPS" ] && MALFORMED=1 +[ -z "$GPGKEY" ] && MALFORMED=1 +[ -z "$GPGTESTKEY" ] && GPGKEY="$GPGTESTKEY" +[ -z "$PBASE" ] && MALFORMED=1 +[ -z "$TBASE" ] && MALFORMED=1 + +[ "$MALFORMED" -eq 1 ] && printf '%b malformed, exiting.\n' "$MYCONF" >&2 && exit 253 + + +printf -v repjoined "%s-$(date -I) " "${MYREPS[@]}" + +# shellcheck disable=SC1091 +source /usr/lib/lirion/ln-initfunctions || exit 10 + +SNDATE="$(aptly snapshot list -raw | head -n1)" +if ! printf '%b' "$SNDATE" | grep -P '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' > /dev/null; then + SNDATE="$(date -I)" +fi +printf -v repoldjoined "%s-$SNDATE " "${MYREPS[@]}" +printf 'Snapshot suffix that will be deleted: %b\n' "$SNDATE" || exit 11 + +printf 'Have you added all packages? :)\n' +printf 'Starting snapshots and publication\033[s in ' +for ((i=5;i>0;--i)); do + printf '\r\033[u\033[K in %b...' "$i" + sleep 1 +done +printf '\r\033[u\033[K:\n' + +for rep in "${MYREPS[@]}"; do + if ! aptly repo list -raw 2>/dev/null | grep -P "^${rep}$" >/dev/null; then + lnfail "repository ${rep} does not exist!" + fi + lnbegin "Adding packages to repo $rep" + if [ ! -d "${PBASE}/$rep" ]; then + lnskip "source directory not existing" + continue + fi + readarray -t debfiles < <( + find "${PBASE}/$rep" -type l -name "*deb" 2>/dev/null + find "${PBASE}/$rep" -type f -name "*deb" 2>/dev/null + ) + case "${#debfiles[@]}" in + 0) + lnskip "no files in ${PBASE}/$rep" + ;; + *) + for debfile in "${debfiles[@]}"; do + lnprog "$(basename "$debfile")" + sleep 0.271828 + if ! aptly repo add "$rep" "$debfile" >/dev/null 2>&1; then + lnfail "adding file failed" + exit 100 + fi + if ! rm -f "$debfile" >/dev/null 2>&1; then + lnfail "source file removal failed" + exit 101 + fi + done + lnok + ;; + esac +done + +lnbegin "Unpublishing repo \"all\"" +if [ "$(aptly publish list | grep -cv '^No snapshots/local repos')" -lt 1 ]; then + lnskip "No snapshots / published repos" +else + if ! aptly publish drop all >/dev/null 2>&1; then + lnfail + exit 110 + fi +fi +lnok + +lnbegin "Dropping snapshots" +for ss in "$SNDATE" ${repoldjoined%,};do + lnprog "$ss" + sleep 0.271828 + if ! aptly snapshot list -raw | grep "^$ss\$" > /dev/null; then + lnprog "skipping $ss, not present" + sleep 0.314159 + else + if ! aptly snapshot drop "$ss" >/dev/null 2>&1; then + lnfail + exit 111 + fi + fi +done +lnok + +lnbegin "Creating fresh snapshots" +for rep in "${MYREPS[@]}"; do + lnprog "$rep" + sleep 0.271828 + if ! faketime "$(date -I) 13:37:08" aptly snapshot create \ + "${rep}-$(date -I)" from repo "$rep" >/dev/null 2>&1; then + lnfail + exit 120 + fi +done +lnok + +lnbegin "Merging snapshots" +printf -v repjoined "%s-$(date -I) " "${MYREPS[@]}" +# shellcheck disable=SC2086 +if ! aptly snapshot merge "$(date -I)" ${repjoined%,} >/dev/null 2>&1; then + lnfail + exit 121 +fi +lnok + +printf 'GPG pseudo operation...\n' +MYLEL="$(mktemp --tmpdir lel.XXX)" +printf 'lel\n' > "$MYLEL" || exit 122 +gpg -eu "$GPGKEY" -r "$GPGTESTKEY" "$MYLEL" || exit 122 +gpg -qd "${MYLEL}.gpg" > /dev/null || exit 122 +rm "$MYLEL" "${MYLEL}.gpg" || exit 122 +printf '...done.\n' + +lnbegin "Publishing snapshot result" +if ! faketime "$(date -I) 13:37:11" aptly publish snapshot \ + -gpg-key="$GPGKEY" -distribution='all' "$(date -I)" >/dev/null 2>&1; then + lnfail + exit 123 +fi +lnok + +lnbegin "Creating aptly graphs" +for layout in horizontal vertical; do + lnprog "$layout" + if ! aptly graph -layout="$layout" \ + -output="${TBASE}/public/aptly-graph-${layout}.png" >/dev/null 2>&1; then + lnfail + exit 130 + fi +done +lnok + +lnbegin "Creating sha256 checksums" +lnprog "directories" +if ! mkdir -p "${TBASE}/public/dists/all/utils"/binary-{amd64,arm64}/by-hash/SHA{256,512} 2>/dev/null; then + lnfail "directory creation failed" + exit 140 +fi +lnprog "old hash removal" +for dir in "${TBASE}/public/dists/all/utils"/binary-{amd64,arm64}/by-hash/SHA{256,512}; do + if ! find "$dir" -type l -exec rm -f '{}' \; ; then + lnfail "hash file removal failed in $dir" + exit 141 + fi +done +lnprog "hash creation" +for file in "${TBASE}/public/dists/all/utils"/binary-{amd64,arm64}/{Packages,Release}*;do + S256="$(sha256sum "$file"|awk '{print $1}')" + S512="$(sha512sum "$file"|awk '{print $1}')" + PROGSTR="SHA256 hash for $(dirname "$file")/$(basename "$file")" + PROGSTR="$(printf '%b' "$PROGSTR" | sed 's#.*\(/dists/\)#...\1#')" + lnprog "$PROGSTR" + unset PROGSTR + sleep 0.271828 + if ! ln -fs "../../$(basename "$file")" "$(dirname "$file")/by-hash/SHA256/$S256" 2>/dev/null; then + lnfail "hash failed for $(dirname "$file")/$(basename "$file")" + exit 142 + fi + PROGSTR="SHA512 hash for $(dirname "$file")/$(basename "$file")" + PROGSTR="$(printf '%b' "$PROGSTR" | sed 's#.*\(/dists/\)#...\1#')" + lnprog "$PROGSTR" + unset PROGSTR + sleep 0.271828 + if ! ln -fs "../../$(basename "$file")" "$(dirname "$file")/by-hash/SHA512/$S512" 2>/dev/null; then + lnfail "hash failed for $(dirname "$file")/$(basename "$file")" + exit 142 + fi +done +lnok diff --git a/aptly/etc/aptly-lirionde.conf b/aptly/etc/aptly-lirionde.conf new file mode 100644 index 0000000..34f8ac3 --- /dev/null +++ b/aptly/etc/aptly-lirionde.conf @@ -0,0 +1,18 @@ +# vim:syntax=sh +# Repos that should be snapshotted and added to the publication: +MYREPS=( + "repo1" + "repo2" +) +# The GPG key with which we will sign the repositories. +GPGKEY='1A7EDEADBEEFBADC0FFEE0123456789ABCDEF012' +# A test public key to which we will encrypt a temporary string to check +# whether GPG works at all: +GPGTESTKEY='1A7EDEADBEEFBADC0FFEE0123456789ABCDEF012' +# Packages base folder. Structure is: +# BASEFOLDER/reponame/package.deb +PBASE="/tmp/aptly" +# Base folder where packages are being published. Assumption is that +# aptly is configured to use $TBASE for all operations except for the +# resulting repo which will be managed and live in $TBASE/public. +TBASE="/var/cache/repos/aptly" diff --git a/bin/aptly-lirionde b/bin/aptly-lirionde deleted file mode 100755 index 8933cd3..0000000 --- a/bin/aptly-lirionde +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash - -# Manages aptly input repositories and publishes a merged output repository. -# Structure is as follows: -# repo1 ───> snapshot "repo1-%Y-%m-%d" ─┬─> snapshot "%Y-%m-%d" ───> publish inside $TBASE/public -# │ -# repo2 ───> snapshot "repo2-%Y-%m-%d" ─┘ -# -# Furthermore, our PACKAGE input folder is /tmp/aptly with repository names as subfolder, so if -# you want a new package added inside repo1, you would place a file inside /tmp/aptly/repo1/. -# The package would then be added and its file inside /tmp be removed. -# -# Why bash? -# 1. This script is INTERACTIVE, it asks for your GPG passphrase before signing packages -# 2. It still uses a lot of syscalls like aptly -# 3. The steps we do are still not that resource-hungry, so the downsides while using bash are rather low -# The combination of these three led to bash in the first iteration. -# -# What doesn't this script do? -# 1. It does not create the source repos or the aptly config. -# 2. It does not repair if one of the parts of the structure above is missing; in that case, -# we assume an extraordinary failure and will fail ourselves. Exception: We skip failing -# on removal if a published repo or a snapshot does not exist. -# 3. It does not manage multiple snapshots yet. Nor does it cleanup snapshot remainders in case the script -# is interrupted. Both can be considered a TODO. - - -MYCONF="/etc/lirion/aptly.conf" -if [ ! -r "$MYCONF" ];then - printf '%b cannot be read, exiting!\n' "$MYCONF" >&2 - exit 254 -else - # shellcheck disable=SC1091,SC1090 - source "$MYCONF" || exit 254 -fi - -MALFORMED=0 -[ -z "$MYREPS" ] && MALFORMED=1 -[ -z "$GPGKEY" ] && MALFORMED=1 -[ -z "$GPGTESTKEY" ] && GPGKEY="$GPGTESTKEY" -[ -z "$PBASE" ] && MALFORMED=1 -[ -z "$TBASE" ] && MALFORMED=1 - -[ "$MALFORMED" -eq 1 ] && printf '%b malformed, exiting.\n' "$MYCONF" >&2 && exit 253 - - -printf -v repjoined "%s-$(date -I) " "${MYREPS[@]}" - -# shellcheck disable=SC1091 -source /usr/lib/lirion/ln-initfunctions || exit 10 - -SNDATE="$(aptly snapshot list -raw | head -n1)" -if ! printf '%b' "$SNDATE" | grep -P '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' > /dev/null; then - SNDATE="$(date -I)" -fi -printf -v repoldjoined "%s-$SNDATE " "${MYREPS[@]}" -printf 'Snapshot suffix that will be deleted: %b\n' "$SNDATE" || exit 11 - -printf 'Have you added all packages? :)\n' -printf 'Starting snapshots and publication\033[s in ' -for ((i=5;i>0;--i)); do - printf '\r\033[u\033[K in %b...' "$i" - sleep 1 -done -printf '\r\033[u\033[K:\n' - -for rep in "${MYREPS[@]}"; do - if ! aptly repo list -raw 2>/dev/null | grep -P "^${rep}$" >/dev/null; then - lnfail "repository ${rep} does not exist!" - fi - lnbegin "Adding packages to repo $rep" - if [ ! -d "${PBASE}/$rep" ]; then - lnskip "source directory not existing" - continue - fi - readarray -t debfiles < <( - find "${PBASE}/$rep" -type l -name "*deb" 2>/dev/null - find "${PBASE}/$rep" -type f -name "*deb" 2>/dev/null - ) - case "${#debfiles[@]}" in - 0) - lnskip "no files in ${PBASE}/$rep" - ;; - *) - for debfile in "${debfiles[@]}"; do - lnprog "$(basename "$debfile")" - sleep 0.271828 - if ! aptly repo add "$rep" "$debfile" >/dev/null 2>&1; then - lnfail "adding file failed" - exit 100 - fi - if ! rm -f "$debfile" >/dev/null 2>&1; then - lnfail "source file removal failed" - exit 101 - fi - done - lnok - ;; - esac -done - -lnbegin "Unpublishing repo \"all\"" -if [ "$(aptly publish list | grep -cv '^No snapshots/local repos')" -lt 1 ]; then - lnskip "No snapshots / published repos" -else - if ! aptly publish drop all >/dev/null 2>&1; then - lnfail - exit 110 - fi -fi -lnok - -lnbegin "Dropping snapshots" -for ss in "$SNDATE" ${repoldjoined%,};do - lnprog "$ss" - sleep 0.271828 - if ! aptly snapshot list -raw | grep "^$ss\$" > /dev/null; then - lnprog "skipping $ss, not present" - sleep 0.314159 - else - if ! aptly snapshot drop "$ss" >/dev/null 2>&1; then - lnfail - exit 111 - fi - fi -done -lnok - -lnbegin "Creating fresh snapshots" -for rep in "${MYREPS[@]}"; do - lnprog "$rep" - sleep 0.271828 - if ! faketime "$(date -I) 13:37:08" aptly snapshot create \ - "${rep}-$(date -I)" from repo "$rep" >/dev/null 2>&1; then - lnfail - exit 120 - fi -done -lnok - -lnbegin "Merging snapshots" -printf -v repjoined "%s-$(date -I) " "${MYREPS[@]}" -# shellcheck disable=SC2086 -if ! aptly snapshot merge "$(date -I)" ${repjoined%,} >/dev/null 2>&1; then - lnfail - exit 121 -fi -lnok - -printf 'GPG pseudo operation...\n' -MYLEL="$(mktemp --tmpdir lel.XXX)" -printf 'lel\n' > "$MYLEL" || exit 122 -gpg -eu "$GPGKEY" -r "$GPGTESTKEY" "$MYLEL" || exit 122 -gpg -qd "${MYLEL}.gpg" > /dev/null || exit 122 -rm "$MYLEL" "${MYLEL}.gpg" || exit 122 -printf '...done.\n' - -lnbegin "Publishing snapshot result" -if ! faketime "$(date -I) 13:37:11" aptly publish snapshot \ - -gpg-key="$GPGKEY" -distribution='all' "$(date -I)" >/dev/null 2>&1; then - lnfail - exit 123 -fi -lnok - -lnbegin "Creating aptly graphs" -for layout in horizontal vertical; do - lnprog "$layout" - if ! aptly graph -layout="$layout" \ - -output="${TBASE}/public/aptly-graph-${layout}.png" >/dev/null 2>&1; then - lnfail - exit 130 - fi -done -lnok - -lnbegin "Creating sha256 checksums" -lnprog "directories" -if ! mkdir -p "${TBASE}/public/dists/all/utils"/binary-{amd64,arm64}/by-hash/SHA{256,512} 2>/dev/null; then - lnfail "directory creation failed" - exit 140 -fi -lnprog "old hash removal" -for dir in "${TBASE}/public/dists/all/utils"/binary-{amd64,arm64}/by-hash/SHA{256,512}; do - if ! find "$dir" -type l -exec rm -f '{}' \; ; then - lnfail "hash file removal failed in $dir" - exit 141 - fi -done -lnprog "hash creation" -for file in "${TBASE}/public/dists/all/utils"/binary-{amd64,arm64}/{Packages,Release}*;do - S256="$(sha256sum "$file"|awk '{print $1}')" - S512="$(sha512sum "$file"|awk '{print $1}')" - PROGSTR="SHA256 hash for $(dirname "$file")/$(basename "$file")" - PROGSTR="$(printf '%b' "$PROGSTR" | sed 's#.*\(/dists/\)#...\1#')" - lnprog "$PROGSTR" - unset PROGSTR - sleep 0.271828 - if ! ln -fs "../../$(basename "$file")" "$(dirname "$file")/by-hash/SHA256/$S256" 2>/dev/null; then - lnfail "hash failed for $(dirname "$file")/$(basename "$file")" - exit 142 - fi - PROGSTR="SHA512 hash for $(dirname "$file")/$(basename "$file")" - PROGSTR="$(printf '%b' "$PROGSTR" | sed 's#.*\(/dists/\)#...\1#')" - lnprog "$PROGSTR" - unset PROGSTR - sleep 0.271828 - if ! ln -fs "../../$(basename "$file")" "$(dirname "$file")/by-hash/SHA512/$S512" 2>/dev/null; then - lnfail "hash failed for $(dirname "$file")/$(basename "$file")" - exit 142 - fi -done -lnok diff --git a/etc/aptly-lirionde.conf b/etc/aptly-lirionde.conf deleted file mode 100644 index 34f8ac3..0000000 --- a/etc/aptly-lirionde.conf +++ /dev/null @@ -1,18 +0,0 @@ -# vim:syntax=sh -# Repos that should be snapshotted and added to the publication: -MYREPS=( - "repo1" - "repo2" -) -# The GPG key with which we will sign the repositories. -GPGKEY='1A7EDEADBEEFBADC0FFEE0123456789ABCDEF012' -# A test public key to which we will encrypt a temporary string to check -# whether GPG works at all: -GPGTESTKEY='1A7EDEADBEEFBADC0FFEE0123456789ABCDEF012' -# Packages base folder. Structure is: -# BASEFOLDER/reponame/package.deb -PBASE="/tmp/aptly" -# Base folder where packages are being published. Assumption is that -# aptly is configured to use $TBASE for all operations except for the -# resulting repo which will be managed and live in $TBASE/public. -TBASE="/var/cache/repos/aptly" -- cgit v1.2.3