diff --git a/scripts/simple_builder.sh b/scripts/simple_builder.sh index a4f1c2e..2bdc48c 100644 --- a/scripts/simple_builder.sh +++ b/scripts/simple_builder.sh @@ -2,14 +2,13 @@ set -euo pipefail # gen_simple_inplace.sh -# Generate PEP 503 "simple" indexes in-place for existing directories: -# //index.html -# Optionally generates: -# /index.html +# Moves wheels from a pool to their corresponding PEP 503 directory and generates the index.html files. +# Standardizes web permissions (755 dir, 644 files) die() { echo "ERROR: $*" >&2; exit 1; } -SIMPLE_DIR="" +FINAL_REPO="" +W_POOL="" ONLY_PKG="" NO_TOP=0 DO_VERIFY=0 @@ -17,7 +16,8 @@ VERIFY_ONLY=0 while [ $# -gt 0 ]; do case "$1" in - --simple-dir) SIMPLE_DIR="${2:-}"; shift 2 ;; + --final-repo|--simple-dir) FINAL_REPO="${2:-}"; shift 2 ;; + --w-pool) W_POOL="${2:-}"; shift 2 ;; --pkg) ONLY_PKG="${2:-}"; shift 2 ;; --no-top) NO_TOP=1; shift ;; --verify) DO_VERIFY=1; shift ;; @@ -25,14 +25,17 @@ while [ $# -gt 0 ]; do -h|--help) cat <<'EOF' Usage: - ./gen_simple_inplace.sh --simple-dir /var/www/.../simple - ./gen_simple_inplace.sh --simple-dir /var/www/.../simple --pkg cffi --no-top - ./gen_simple_inplace.sh --simple-dir /var/www/.../simple --verify - ./gen_simple_inplace.sh --simple-dir /var/www/.../simple --verify-only + ./simple_builder.sh --final-repo ~/simple/ --w-pool ~/wheel_pool/ + ./simple_builder.sh --final-repo ~/simple/ --pkg cffi + ./simple_builder.sh --final-repo ~/simple/ --verify + Options: - --no-top Don't rewrite /simple/index.html - --verify Verify that each /index.html href exists and sha256 matches - --verify-only Verify only (do not regenerate any index.html) + --final-repo Path to the repository's root directory (e.g., ~/simple/) + --w-pool Path to the folder containing new wheels to accommodate (e.g., ~/wheel_pool/) + --pkg Update/Verify only a specific package + --no-top Do not regenerate the main /simple/index.html file + --verify Verify that the href attributes exist and the SHA256 attributes match + --verify-only Only verify (does not regenerate any index.html files or move wheels) EOF exit 0 ;; @@ -40,16 +43,40 @@ EOF esac done -[ -n "$SIMPLE_DIR" ] || die "--simple-dir is required" -[ -d "$SIMPLE_DIR" ] || die "Not a directory: $SIMPLE_DIR" +[ -n "$FINAL_REPO" ] || die "--final-repo is required" +[ -d "$FINAL_REPO" ] || die "It is not a valid directory: $FINAL_REPO" -is_artifact() { - case "$1" in - *.whl|*.tar.gz|*.zip|*.tgz) return 0 ;; - *) return 1 ;; - esac +# Strict PEP 503 normalization +normalize_pkg_name() { + echo "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[-_.]+/-/g' } +# 1. Process the Pool of Wheels +if [ "$VERIFY_ONLY" -eq 0 ] && [ -n "$W_POOL" ]; then + [ -d "$W_POOL" ] || die "The pool is not a valid directory: $W_POOL" + echo "=> Scanning pool of wheels in: $W_POOL" + while IFS= read -r -d '' whl; do + filename="$(basename "$whl")" + + # The distribution name is everything that comes before the first hyphen (-) + raw_dist="${filename%%-*}" + norm_pkg="$(normalize_pkg_name "$raw_dist")" + dest_dir="$FINAL_REPO/$norm_pkg" + + # Create the folder and ensure its web permissions + mkdir -p "$dest_dir" + chmod 755 "$dest_dir" + + echo " -> Moving: $filename to the directory /$norm_pkg/" + mv -f "$whl" "$dest_dir/" + + # Ensure that the newly moved file has web permissions + chmod 644 "$dest_dir/$filename" + + done < <(find "$W_POOL" -maxdepth 1 -type f -name '*.whl' -print0) + echo "=> Moving complete." +fi + verify_pkg_index() { local pkgdir="$1" local pkgname="$2" @@ -57,19 +84,17 @@ verify_pkg_index() { local errs=0 if [ ! -f "$idx" ]; then - echo "VERIFY FAIL [$pkgname]: missing index.html" >&2 + echo "VERIFY FAIL [$pkgname]: index.html is missing" >&2 return 1 fi - # Extract href targets like: filename#sha256=.... - # We assume filenames don't contain quotes/spaces (true for wheels/sdists typically). while IFS= read -r href; do [ -n "$href" ] || continue local file="${href%%#sha256=*}" local want="${href##*#sha256=}" if [ ! -f "$pkgdir/$file" ]; then - echo "VERIFY FAIL [$pkgname]: missing file: $file" >&2 + echo "VERIFY FAIL [$pkgname]: file is missing: $file" >&2 errs=$((errs+1)) continue fi @@ -78,12 +103,11 @@ verify_pkg_index() { got="$(sha256sum "$pkgdir/$file" | awk '{print $1}')" if [ "$got" != "$want" ]; then echo "VERIFY FAIL [$pkgname]: sha256 mismatch for $file" >&2 - echo " want: $want" >&2 - echo " got : $got" >&2 + echo " expected: $want" >&2 + echo " got: $got" >&2 errs=$((errs+1)) fi done < <( - # Pull href="..."; keep only those containing #sha256= grep -oE 'href="[^"]+"' "$idx" \ | sed -E 's/^href="(.*)"$/\1/' \ | grep -E '#sha256=' || true @@ -103,61 +127,72 @@ write_pkg_index() { local pkgname="$2" local idx="$pkgdir/index.html" - # Collect artifacts in that pkg dir - mapfile -t files < <(find "$pkgdir" -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' -o -name '*.zip' -o -name '*.tgz' \) -printf '%f\n' | sort) + # Ensure directory and existing wheels permissions prior to this script + chmod 755 "$pkgdir" + find "$pkgdir" -maxdepth 1 -type f -name '*.whl' -exec chmod 644 {} + - # If no artifacts, skip (or you can still write an empty index if you want) + mapfile -t files < <(find "$pkgdir" -maxdepth 1 -type f -name '*.whl' -printf '%f\n' | sort) [ "${#files[@]}" -gt 0 ] || return 0 { echo "" echo "${pkgname}" for bn in "${files[@]}"; do - # hash the file in place sha="$(sha256sum "$pkgdir/$bn" | awk '{print $1}')" printf '%s
\n' "$bn" "$sha" "$bn" done echo "" } > "$idx" + + # Ensure generated index permissions + chmod 644 "$idx" } -# Determine package dirs +# Determine package directories pkg_dirs=() if [ -n "$ONLY_PKG" ]; then - [ -d "$SIMPLE_DIR/$ONLY_PKG" ] || die "Package dir not found: $SIMPLE_DIR/$ONLY_PKG" - pkg_dirs+=("$SIMPLE_DIR/$ONLY_PKG") + ONLY_PKG="$(normalize_pkg_name "$ONLY_PKG")" + [ -d "$FINAL_REPO/$ONLY_PKG" ] || die "Package directory not found: $FINAL_REPO/$ONLY_PKG" + pkg_dirs+=("$FINAL_REPO/$ONLY_PKG") else - # All subdirs except hidden ones while IFS= read -r d; do pkg_dirs+=("$d") - done < <(find "$SIMPLE_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.*' | sort) + done < <(find "$FINAL_REPO" -mindepth 1 -maxdepth 1 -type d ! -name '.*' | sort) fi if [ "$VERIFY_ONLY" -eq 0 ]; then - # Generate per-package indexes + echo "=> Generating indexes (index.html) and standardizing permissions..." + + # Ensure root directory permissions + chmod 755 "$FINAL_REPO" + for d in "${pkg_dirs[@]}"; do pkg="$(basename "$d")" write_pkg_index "$d" "$pkg" done - # Top index (optional) + # Main Index if [ "$NO_TOP" -eq 0 ] && [ -z "$ONLY_PKG" ]; then - top="$SIMPLE_DIR/index.html" + top="$FINAL_REPO/index.html" { echo "" echo "Simple Index" for d in "${pkg_dirs[@]}"; do pkg="$(basename "$d")" - if find "$d" -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' -o -name '*.zip' -o -name '*.tgz' \) | grep -q .; then + if find "$d" -maxdepth 1 -type f -name '*.whl' | grep -q .; then printf '%s
\n' "$pkg" "$pkg" fi done echo "" } > "$top" + + # Main index permissions + chmod 644 "$top" fi fi if [ "$DO_VERIFY" -eq 1 ]; then + echo "=> Verifying integrity..." vfail=0 for d in "${pkg_dirs[@]}"; do pkg="$(basename "$d")" @@ -166,12 +201,12 @@ if [ "$DO_VERIFY" -eq 1 ]; then fi done if [ "$vfail" -ne 0 ]; then - die "Verification failed" + die "Verification failed for one or more packages." fi fi if [ "$VERIFY_ONLY" -eq 1 ]; then - echo "OK: verified indexes under: $SIMPLE_DIR" + echo "=> OK: indexes verified in: $FINAL_REPO" else - echo "OK: indexes generated under: $SIMPLE_DIR" + echo "=> OK: process finished successfully in: $FINAL_REPO" fi