diff --git a/multipass/cloud-init-rpios-like-arm64.yaml b/multipass/cloud-init-rpios-like-arm64.yaml new file mode 100644 index 0000000..0e06770 --- /dev/null +++ b/multipass/cloud-init-rpios-like-arm64.yaml @@ -0,0 +1,62 @@ +#cloud-config +hostname: rpios-like-arm64 +manage_etc_hosts: true + +write_files: + - path: /usr/local/sbin/setup-raspi-repo-and-sys-mods-arm64.sh + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + arch="$(dpkg --print-architecture)" + if [[ "$arch" != "arm64" ]]; then + echo "arm64-only; detected '$arch' -> skipping" >&2 + exit 0 + fi + + export DEBIAN_FRONTEND=noninteractive + + apt-get update + apt-get install -y --no-install-recommends ca-certificates curl gnupg util-linux + + install -d -m 0755 /usr/share/keyrings /etc/apt/preferences.d + + if [[ ! -s /usr/share/keyrings/raspberrypi-archive-keyring.gpg ]]; then + curl -fsSL https://archive.raspberrypi.org/debian/raspberrypi.gpg.key \ + | gpg --dearmor -o /usr/share/keyrings/raspberrypi-archive-keyring.gpg + chmod 0644 /usr/share/keyrings/raspberrypi-archive-keyring.gpg + fi + + #cat >/etc/apt/preferences.d/raspi.pref <<'EOF' + #Package: * + #Pin: origin "archive.raspberrypi.org" + #Pin-Priority: 100 + #EOF + + if [[ ! -f /etc/apt/sources.list.d/raspi.sources ]]; then + cat >/etc/apt/sources.list.d/raspi.sources <<'EOF' + Types: deb + URIs: https://archive.raspberrypi.org/debian/ + Suites: trixie + Components: main + Signed-By: /usr/share/keyrings/raspberrypi-archive-keyring.gpg + EOF + fi + + apt-get update + + # Add custom /etc/rpi-issue for "rpios-like" + if [[ ! -s /etc/rpi-issue ]]; then + { + echo "Raspberry Pi reference trixie-latest $(date -R)" + echo "Emulated using debian-13-genericcloud" + } >/etc/rpi-issue + fi + + echo "---- /etc/rpi-issue ----" + cat /etc/rpi-issue || true + echo "------------------------" + +runcmd: + - [ bash, -lc, "/usr/local/sbin/setup-raspi-repo-and-sys-mods-arm64.sh" ] diff --git a/multipass/run_parallel_iiab_test.sh b/multipass/run_parallel_iiab_test.sh index 8e0472d..8a39dd4 100644 --- a/multipass/run_parallel_iiab_test.sh +++ b/multipass/run_parallel_iiab_test.sh @@ -7,7 +7,6 @@ # --clean Stop+delete+purge target VMs (by BASE-, regardless of COUNT) # Distro selection: # --debian-13 Debian 13 only (sets IMAGE=$DEBIAN13_IMAGE_URL and BASE=deb13) -# --debian-13 Debian 13 only (sets IMAGE=$DEBIAN13_IMAGE_URL and BASE=deb13) # --both-distros Run Ubuntu + Debian 13 in parallel: COUNT=N => 2N VMs (default order: interleaved) # --first-ubuntu (with --both-distros) order: all Ubuntu first, then all Debian # --first-debian (with --both-distros) order: all Debian first, then all Ubuntu @@ -16,6 +15,11 @@ # --pr 4122 (repeatable) add PR numbers passed to install.txt # --run-pr 4122 same as --pr but also forces --run (alias: --test-pr) # +# Cloud-init: +# --cloud-init FILE Apply cloud-init FILE to all VMs (Ubuntu + Debian) +# --cloud-init-rpios Apply RaspberryPiOS-style cloud-init only to Debian side +# (alias of --debian-13, but sets Debian BASE=rpios-d13) +# # Env vars: # IIAB_PR="4122 4191" Space-separated PRs # (compat) IIAB_INSTALL_ARGS If set, used as fallback for IIAB_PR @@ -33,14 +37,25 @@ set -euo pipefail +DPKG_ARCH="$(dpkg --print-architecture)" + +# Resolve paths relative to this script (so cloud-init files work from any cwd) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + # Debian 13 (Trixie) official cloud image (qcow2). Multipass can launch from URL/file:// on Linux. # Source: Debian cloud images live under cloud.debian.org/images/cloud/ ('genericcloud' includes cloud-init). -DEBIAN13_IMAGE_URL="${DEBIAN13_IMAGE_URL:-https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2}" +DEBIAN13_IMAGE_URL="${DEBIAN13_IMAGE_URL:-https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-${DPKG_ARCH}.qcow2}" IMAGE="${IMAGE:-24.04}" BASE="${BASE:-ubu2404}" +# cloud-init controls +CLOUD_INIT_FILE="${CLOUD_INIT_FILE:-}" +RPIOS_CLOUD_INIT_FILE="${RPIOS_CLOUD_INIT_FILE:-$SCRIPT_DIR/cloud-init-rpios-like-arm64.yaml}" +DEB_CLOUD_INIT_FILE="${DEB_CLOUD_INIT_FILE:-}" +RPIOS_MODE=0 COUNT="${COUNT:-1}" -CPUS="${CPUS:-3}" +[ "$DPKG_ARCH" = "arm64" ] && CPUS="${CPUS:-2}" # SBC don't have spare CPUs. +[ "$DPKG_ARCH" = "amd64" ] && CPUS="${CPUS:-3}" MEM="${MEM:-4G}" DISK="${DISK:-20G}" @@ -52,7 +67,9 @@ LOCAL_VARS_URL="${LOCAL_VARS_URL:-https://raw.githubusercontent.com/iiab/iiab/re WAIT_TRIES="${WAIT_TRIES:-60}" # used ONLY for the first auto-resume WAIT_SLEEP="${WAIT_SLEEP:-5}" -STAGGER="${STAGGER:-15}" +STAGGER="${STAGGER:-30}" +RESUME_TRIES="${RESUME_TRIES:-3}" +RESUME_RETRY_SLEEP="${RESUME_RETRY_SLEEP:-8}" ACTION="run" modules=() @@ -87,12 +104,16 @@ Image shortcuts: --first-ubuntu With --both-distros: run all Ubuntu first, then Debian --first-debian With --both-distros: run all Debian first, then Ubuntu +Cloud-init: + --cloud-init FILE Apply FILE to all VMs + --cloud-init-rpios Apply rpios cloud-init only to Debian side; BASE=rpios-d13 (Debian) + PR options: --pr N Add PR number (repeatable) --run-pr N Add PR number and force --run Env: - IMAGE BASE COUNT CPUS MEM DISK IIAB_PR IIAB_FAST LOCAL_VARS_URL WAIT_TRIES WAIT_SLEEP STAGGER + IMAGE BASE COUNT CPUS MEM DISK IIAB_PR IIAB_FAST LOCAL_VARS_URL WAIT_TRIES WAIT_SLEEP STAGGER RESUME_TRIES RESUME_RETRY_SLEEP EOF } @@ -107,12 +128,25 @@ while [[ $# -gt 0 ]]; do --first-ubuntu) FIRST_UBUNTU=1; shift ;; --first-debian) FIRST_DEBIAN=1; shift ;; + --cloud-init) + [[ $# -lt 2 ]] && { echo "[ERROR] --cloud-init needs a file path"; exit 2; } + CLOUD_INIT_FILE="$2"; shift 2 ;; + --debian-13) DEBIAN13_ONLY=1 IMAGE="$DEBIAN13_IMAGE_URL" BASE="deb13" shift ;; + --cloud-init-rpios) + # Debian-only rpios flavor: also implies Debian 13 selection + DEBIAN13_ONLY=1 + RPIOS_MODE=1 + IMAGE="$DEBIAN13_IMAGE_URL" + BASE="rpios-d13" + DEB_CLOUD_INIT_FILE="$RPIOS_CLOUD_INIT_FILE" + shift ;; + --module) [[ $# -lt 2 ]] && { echo "[ERROR] --module needs a value"; exit 2; } modules+=("$2"); shift 2 ;; @@ -135,7 +169,7 @@ while [[ $# -gt 0 ]]; do esac done -# ---- Incoherency checks (fail fast) ---- +# Incoherency checks (fail fast) if [[ "$FIRST_UBUNTU" == "1" && "$FIRST_DEBIAN" == "1" ]]; then echo "[ERROR] Incoherent options: --first-ubuntu and --first-debian cannot be used together." exit 2 @@ -145,11 +179,35 @@ if [[ ( "$FIRST_UBUNTU" == "1" || "$FIRST_DEBIAN" == "1" ) && "$BOTH_DISTROS" != echo "[ERROR] --first-ubuntu/--first-debian requires --both-distros." exit 2 fi -# ---- + +if ! [[ "${COUNT}" =~ ^[0-9]+$ ]] || (( COUNT < 1 )); then + echo "[ERROR] COUNT must be an integer >= 1 (got: '${COUNT}')" >&2 + exit 2 +fi +# --- + +# Fail-fast validation for cloud-init files +if [[ -n "${CLOUD_INIT_FILE:-}" && ! -f "$CLOUD_INIT_FILE" ]]; then + echo "[ERROR] --cloud-init file not found on host: $CLOUD_INIT_FILE" >&2 + exit 2 +fi + +if [[ -n "${DEB_CLOUD_INIT_FILE:-}" && ! -f "$DEB_CLOUD_INIT_FILE" ]]; then + echo "[ERROR] Debian cloud-init file not found on host: $DEB_CLOUD_INIT_FILE" >&2 + exit 2 +fi + +# RPiOS flavor is meant for arm64 (rpios-like). Fail fast to avoid surprising runs. +if [[ "$RPIOS_MODE" == "1" && "$DPKG_ARCH" != "arm64" ]]; then + echo "[ERROR] --cloud-init-rpios is intended for arm64 hosts (detected: $DPKG_ARCH)." >&2 + echo " If you really want to run it on non-arm64, pass a normal --cloud-init FILE instead." >&2 + exit 2 +fi +# --- # Default module if [[ "${#modules[@]}" -eq 0 ]]; then - modules=("code") + modules=("") fi # If no --pr provided, take from env IIAB_PR (space-separated) @@ -165,6 +223,7 @@ if [[ "$BOTH_DISTROS" == "1" ]]; then UBU_BASE="$UBU_BASE_ORIG" DEB_IMAGE="$DEBIAN13_IMAGE_URL" DEB_BASE="deb13" + [[ "$RPIOS_MODE" == "1" ]] && DEB_BASE="rpios-d13" else UBU_IMAGE="$IMAGE" UBU_BASE="$BASE" @@ -174,6 +233,7 @@ fi LOGROOT="${LOGROOT:-iiab_multipass_runs_$(date +%Y%m%d)}" mkdir -p "$LOGROOT" +echo "[INFO] Logging files stored at: $LOGROOT" stamp() { date +%H%M%S; } @@ -209,12 +269,13 @@ wait_all() { # Escape BASE for regex usage re_escape() { printf '%s' "$1" | sed -e 's/[].[^$*+?(){}|\\]/\\&/g'; } -declare -A VM_IMAGE +declare -A VM_IMAGE VM_CLOUD_INIT names=() build_vm_lists() { names=() VM_IMAGE=() + VM_CLOUD_INIT=() if [[ "$BOTH_DISTROS" == "1" ]]; then if [[ "$FIRST_UBUNTU" == "1" ]]; then @@ -223,11 +284,13 @@ build_vm_lists() { local u="${UBU_BASE}-${n}" names+=("$u") VM_IMAGE["$u"]="$UBU_IMAGE" + VM_CLOUD_INIT["$u"]="$CLOUD_INIT_FILE" done for n in $(seq 0 $((COUNT-1))); do local d="${DEB_BASE}-${n}" names+=("$d") VM_IMAGE["$d"]="$DEB_IMAGE" + VM_CLOUD_INIT["$d"]="${DEB_CLOUD_INIT_FILE:-$CLOUD_INIT_FILE}" done elif [[ "$FIRST_DEBIAN" == "1" ]]; then @@ -236,11 +299,13 @@ build_vm_lists() { local d="${DEB_BASE}-${n}" names+=("$d") VM_IMAGE["$d"]="$DEB_IMAGE" + VM_CLOUD_INIT["$d"]="${DEB_CLOUD_INIT_FILE:-$CLOUD_INIT_FILE}" done for n in $(seq 0 $((COUNT-1))); do local u="${UBU_BASE}-${n}" names+=("$u") VM_IMAGE["$u"]="$UBU_IMAGE" + VM_CLOUD_INIT["$u"]="$CLOUD_INIT_FILE" done else @@ -251,6 +316,8 @@ build_vm_lists() { names+=("$u" "$d") VM_IMAGE["$u"]="$UBU_IMAGE" VM_IMAGE["$d"]="$DEB_IMAGE" + VM_CLOUD_INIT["$u"]="$CLOUD_INIT_FILE" + VM_CLOUD_INIT["$d"]="${DEB_CLOUD_INIT_FILE:-$CLOUD_INIT_FILE}" done fi else @@ -258,6 +325,11 @@ build_vm_lists() { local vm="${BASE}-${n}" names+=("$vm") VM_IMAGE["$vm"]="$IMAGE" + if [[ "$DEBIAN13_ONLY" == "1" ]]; then + VM_CLOUD_INIT["$vm"]="${DEB_CLOUD_INIT_FILE:-$CLOUD_INIT_FILE}" + else + VM_CLOUD_INIT["$vm"]="$CLOUD_INIT_FILE" + fi done fi } @@ -315,14 +387,20 @@ cleanup_vms() { launch_one() { local vm="$1" local img="${VM_IMAGE[$vm]:-}" - [[ -z "$img" ]] && { echo "[ERROR] No image mapping for VM '$vm'"; return 2; } + [[ -n "$img" ]] || { echo "[ERROR] No image mapping for VM '$vm'"; return 2; } + + # Cloud-init file to apply (already validated earlier in fail-fast checks) + local ci="${VM_CLOUD_INIT[$vm]:-}" + local -a ci_args=() + [[ -n "${ci:-}" ]] && ci_args=(--cloud-init "$ci") if vm_exists "$vm"; then echo "[INFO] VM already exists: $vm" return 0 fi + echo "[INFO] Launching $vm ..." - multipass launch "$img" -n "$vm" -c "$CPUS" -m "$MEM" -d "$DISK" >/dev/null + multipass launch "$img" -n "$vm" -c "$CPUS" -m "$MEM" -d "$DISK" "${ci_args[@]}" >/dev/null } run_install_txt() { @@ -332,7 +410,17 @@ run_install_txt() { log="$LOGROOT/${vm}.install.${t}.log" rc="$LOGROOT/${vm}.install.${t}.rc" - echo "[INFO] Logging files stored at: $LOGROOT" + echo "[INFO] Waiting for VM readiness before install: $vm" + if ! wait_for_vm "$vm"; then + { + echo "[INFO] Waiting for VM readiness before install: $vm" + echo "[ERROR] VM did not become ready in time (install phase): $vm" + } >"$log" + echo "88" >"$rc" + set_latest_links "$vm" "install" "$log" "$rc" + return 88 + fi + echo "[INFO] install.txt in $vm (log $(basename "$log")) ..." local modules_str pr_str @@ -392,8 +480,22 @@ run_install_txt() { fi echo "--- install.txt ---" - echo "curl -fsSL https://iiab.io/install.txt | bash -s -- ${install_args[*]}" - curl -fsSL https://iiab.io/install.txt | bash -s -- "${install_args[@]}" + INSTALL_PRIMARY="https://iiab.io/install.txt" + INSTALL_FALLBACK="https://raw.githubusercontent.com/iiab/iiab-factory/refs/heads/master/install.txt" + CURL="curl -fsSL --retry 5 --retry-delay 2 --connect-timeout 15" + + tmp_install="$(mktemp)" + if $CURL "$INSTALL_PRIMARY" -o "$tmp_install"; then + : + else + echo "Warning: failed to fetch $INSTALL_PRIMARY" + echo "Falling back to $INSTALL_FALLBACK" + $CURL "$INSTALL_FALLBACK" -o "$tmp_install" + fi + + echo "bash -s -- ${install_args[*]} < \"$tmp_install\"" + bash -s -- "${install_args[@]}" < "$tmp_install" + rm -f "$tmp_install" echo "--- install done ---" ' >"$log" 2>&1 @@ -405,7 +507,7 @@ run_install_txt() { resume_iiab() { local vm="$1" - local do_long_wait="$2" # 1 => wait_for_vm (only for first auto-resume), 0 => no long wait + local do_long_wait="$2" local t log rc t="$(stamp)" log="$LOGROOT/${vm}.resume.${t}.log" @@ -414,7 +516,7 @@ resume_iiab() { echo "[INFO] resume (iiab -f) in $vm (log $(basename "$log")) ..." set +e - { + ( multipass start "$vm" >/dev/null 2>&1 || true if [[ "$do_long_wait" == "1" ]]; then @@ -425,18 +527,56 @@ resume_iiab() { fi fi - multipass exec "$vm" -- bash -lc ' - set -euo pipefail - echo "--- resume: sudo iiab -f ---" - if command -v iiab >/dev/null 2>&1; then - sudo iiab -f - else - echo "[ERROR] iiab command not found; install likely not finished." - exit 89 + # Retry on rc=255 (SSH connection dropped; often reboot/network restart during apt/upgrade) + # Also neutralize apt-listchanges/pagers to avoid "Waiting for data..." stalls. + r=1 + for attempt in $(seq 1 "$RESUME_TRIES"); do + echo "[INFO] Resume attempt ${attempt}/${RESUME_TRIES} on $vm" + + multipass exec "$vm" -- bash -lc ' + set -euo pipefail + echo "--- resume: sudo /usr/sbin/iiab -f ---" + + if ! sudo test -x /usr/sbin/iiab; then + echo "[ERROR] /usr/sbin/iiab not found/executable; install likely not finished." + exit 89 + fi + + # Avoid interactive/pager behaviour during apt actions that IIAB may trigger. + # (apt-listchanges commonly causes: "Waiting for data... (interrupt to abort)") + sudo env \ + DEBIAN_FRONTEND=noninteractive \ + APT_LISTCHANGES_FRONTEND=none \ + NEEDRESTART_MODE=a \ + TERM=dumb \ + PAGER=cat \ + /usr/sbin/iiab -f + + echo "--- resume done ---" + ' + + r=$? + if [[ "$r" -eq 0 ]]; then + break fi - echo "--- resume done ---" - ' - } >"$log" 2>&1 + + if [[ "$r" -eq 255 ]]; then + echo "[WARN] multipass exec rc=255 (connection dropped; likely reboot). Waiting for VM and retrying..." + if ! wait_for_vm "$vm"; then + echo "[ERROR] VM did not become ready after reconnect wait: $vm" + r=88 + break + fi + sleep "$RESUME_RETRY_SLEEP" + continue + fi + + # Any other non-zero: don't loop forever. + break + done + + exit "$r" + ) >"$log" 2>&1 echo "$?" >"$rc" set -e @@ -453,9 +593,7 @@ summary() { [[ -f "$LOGROOT/latest.${vm}.install.rc" ]] && ir="$(cat "$LOGROOT/latest.${vm}.install.rc" 2>/dev/null || echo n/a)" [[ -f "$LOGROOT/latest.${vm}.resume.rc" ]] && rr="$(cat "$LOGROOT/latest.${vm}.resume.rc" 2>/dev/null || echo n/a)" - printf "%-12s %-8s %-8s %s\n" \ - "$vm" "$ir" "$rr" \ - "latest.${vm}.install.log / latest.${vm}.resume.log" + printf "%-12s %-8s %-8s %s\n" "$vm" "$ir" "$rr" "latest.${vm}.install.log / latest.${vm}.resume.log" done echo @@ -491,6 +629,33 @@ phase_parallel_stagger() { set -e } +pipeline_parallel_stagger() { + # Per-VM pipeline: each VM does launch->install->resume independently. + # This avoids "install barrier" where slow Debian blocks Ubuntu's resume. + local pids=() + + for i in "${!names[@]}"; do + local vm="${names[$i]}" + ( + sleep $((i * STAGGER)) + launch_one "$vm" + run_install_txt "$vm" + # Only Ubuntu tends to reboot during install; Debian often doesn't. + local waitflag=1 + if [[ "$vm" == deb13-* ]]; then + waitflag=0 + fi + resume_iiab "$vm" "$waitflag" + ) & + pids+=("$!") + done + + # Don't abort the whole script if one VM fails; we still want logs + summary. + set +e + wait_all "${pids[@]}" + set -e +} + # ---- Main ---- build_vm_lists @@ -505,9 +670,7 @@ fi case "$ACTION" in run) - phase_parallel_stagger launch - phase_parallel_stagger install - phase_parallel_stagger resume 1 + pipeline_parallel_stagger summary ;; continue) diff --git a/scripts/tests/cotg_curl_stats.sh b/scripts/tests/cotg_curl_stats.sh new file mode 100644 index 0000000..39b8f83 --- /dev/null +++ b/scripts/tests/cotg_curl_stats.sh @@ -0,0 +1,397 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ----------------------------------------------------------------------------- +# cotg_curl_stats.ansiblelike.sh +# +# Stress-test helper that mimics the Ansible playbook logic: +# - "uri" to fetch the APK index with retries/delay/until +# - optional fallback "uri" to a WP JSON endpoint (best-effort) +# - (optional) "get_url"-like APK download with retries, without *.apk.1 files +# +# Positional args (kept compatible with your original script): +# 1: N (default: 300) +# 2: SLEEP_SEC (default: 0.5) +# +# Environment variables: +# UA=ansible-httpget +# RETRIES=5 +# DELAY=5 +# JSON_URL=... # like code_fetch_apk_url_json (optional) +# DOWNLOAD_BASE=... # like code_download_url (optional, used if DO_DOWNLOAD=1) +# DO_DOWNLOAD=0|1 # if 1, download the latest armv8a APK each loop +# DOWNLOAD_DIR=... # where to store downloaded APKs (default: tmpdir/apk) +# DOWNLOAD_TIMEOUT=60 # curl max-time +# CURL_HTTP=--http1.1 # force HTTP version (Ansible's urllib is typically HTTP/1.1) +# CURL_EXTRA_ARGS="" # extra curl args (space-separated) +# ----------------------------------------------------------------------------- + +URL="https://www.appdevforall.org/codeonthego/" +N="${1:-500}" +SLEEP_SEC="${2:-0.5}" + +UA="${UA:-ansible-httpget}" +RETRIES="${RETRIES:-5}" +DELAY="${DELAY:-5}" +JSON_URL="${JSON_URL:-https://www.appdevforall.org/wp-json/wp/v2/pages/2223}" +DOWNLOAD_BASE="${DOWNLOAD_BASE:-}" +DO_DOWNLOAD="${DO_DOWNLOAD:-0}" +DOWNLOAD_TIMEOUT="${DOWNLOAD_TIMEOUT:-60}" +CURL_HTTP="${CURL_HTTP:---http1.1}" +CURL_EXTRA_ARGS="${CURL_EXTRA_ARGS:-}" + +# Patterns (defaults aligned to the playbook vars) +APK_RE="${APK_RE:-CodeOnTheGo-.*?armv8a\\.apk}" +CF_RE='(cloudflare|cf-chl|just a moment|attention required)' + +# "uri" logic in your playbook treats these as terminal statuses for the index fetch +ALLOWED_INDEX_STATUS=(200 403 404) + +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +DOWNLOAD_DIR="${DOWNLOAD_DIR:-$tmpdir/apk}" +mkdir -p "$DOWNLOAD_DIR" + +# Stats +declare -A status_count=() +declare -A apk_count=() +declare -A cf_count=() +declare -A blocked_count=() +declare -A retry_hist_index=() +declare -A retry_hist_json=() +declare -A retry_hist_apk=() + +min_size="" +max_size="" +sum_size=0 + +note_size() { + local sz="$1" + [[ -z "$min_size" || "$sz" -lt "$min_size" ]] && min_size="$sz" + [[ -z "$max_size" || "$sz" -gt "$max_size" ]] && max_size="$sz" + sum_size=$((sum_size + sz)) +} + +in_list() { + local x="$1"; shift + local v + for v in "$@"; do + [[ "$x" == "$v" ]] && return 0 + done + return 1 +} + +# curl -> files, return status code (0 means "undefined" like Ansible when status missing) +# Writes headers/body even if status is 0. +curl_fetch() { + local url="$1" body="$2" hdr="$3" accept_header="${4:-}" + local rc status + local -a extra + + # Allow tuning curl behavior to get closer to what Ansible's urllib does. + # (Example: force HTTP/1.1 instead of opportunistic HTTP/2.) + extra=("$CURL_HTTP") + if [[ -n "$CURL_EXTRA_ARGS" ]]; then + # shellcheck disable=SC2206 + extra+=( $CURL_EXTRA_ARGS ) + fi + + : >"$hdr" + : >"$body" + + # Disable -e locally to collect rc + still return 0 status on failures + set +e + if [[ -n "$accept_header" ]]; then + curl -sS -L -A "$UA" -m "$DOWNLOAD_TIMEOUT" "${extra[@]}" -H "Accept: $accept_header" -D "$hdr" -o "$body" "$url" + else + curl -sS -L -A "$UA" -m "$DOWNLOAD_TIMEOUT" "${extra[@]}" -D "$hdr" -o "$body" "$url" + fi + rc=$? + set -e + + # status = last HTTP status in the chain + status="$(awk '/^HTTP\//{code=$2} END{print code+0}' "$hdr" 2>/dev/null)" + + # If curl failed hard, treat as "status undefined" + if [[ $rc -ne 0 || -z "$status" || "$status" -eq 0 ]]; then + echo 0 + return 0 + fi + + echo "$status" +} + +# Mimic: +# retries: RETRIES +# delay: DELAY +# until: status is defined and status in [200,403,404] +# failed_when: status is not defined or status not in [200,403,404] +# Returns: " " where attempts is how many tries were used. +fetch_index_like_ansible() { + local url="$1" body="$2" hdr="$3" + local attempt status + + for ((attempt=1; attempt<=RETRIES; attempt++)); do + status="$(curl_fetch "$url" "$body" "$hdr")" + + # until: status defined AND status in allowed + if [[ "$status" -ne 0 ]] && in_list "$status" "${ALLOWED_INDEX_STATUS[@]}"; then + echo "$status $attempt" + return 0 + fi + + [[ $attempt -lt $RETRIES ]] && sleep "$DELAY" + done + + # exhausted retries; status may be 0 or non-allowed + echo "$status $RETRIES" +} + +# Mimic your JSON fallback: +# retries: RETRIES +# delay: DELAY +# until: status is defined +# failed_when: false (best-effort) +# Returns: " " with status 0 if never got any HTTP status. +fetch_json_best_effort() { + local url="$1" body="$2" hdr="$3" + local attempt status + + for ((attempt=1; attempt<=RETRIES; attempt++)); do + status="$(curl_fetch "$url" "$body" "$hdr" 'application/json')" + + if [[ "$status" -ne 0 ]]; then + echo "$status $attempt" + return 0 + fi + + [[ $attempt -lt $RETRIES ]] && sleep "$DELAY" + done + + echo "0 $RETRIES" +} + +# Extract content.rendered from a WP JSON response (similar to Ansible's parsed json) +# Prints rendered HTML to stdout, or nothing on failure. +extract_wp_rendered() { + local json_file="$1" + + if command -v python3 >/dev/null 2>&1; then + python3 - "$json_file" <<'PY' +import json, sys +p = sys.argv[1] +try: + with open(p, 'r', encoding='utf-8', errors='replace') as f: + data = json.load(f) + rendered = data.get('content', {}).get('rendered', '') + if isinstance(rendered, str): + sys.stdout.write(rendered) +except Exception: + pass +PY + else + # Very rough fallback (not JSON-safe); prefer python3. + sed -n 's/.*"rendered"[[:space:]]*:[[:space:]]*"\(.*\)".*/\1/p' "$json_file" | head -n 1 + fi +} + +# get_url-like download: +# retries: RETRIES +# delay: DELAY +# until: succeeded +# Ensures final file name is exactly dest (downloads to temp then atomic mv). +# Returns: " " +download_apk_like_ansible() { + local url="$1" dest="$2" + local attempt status tmp_part hdr + + hdr="$tmpdir/apk.hdr" + + for ((attempt=1; attempt<=RETRIES; attempt++)); do + tmp_part="${dest}.part.$$" + + status="$(curl_fetch "$url" "$tmp_part" "$hdr")" + + if [[ "$status" -eq 200 && -s "$tmp_part" ]]; then + mv -f "$tmp_part" "$dest" + echo "1 $attempt $status" + return 0 + fi + + rm -f "$tmp_part" + [[ $attempt -lt $RETRIES ]] && sleep "$DELAY" + done + + echo "0 $RETRIES ${status:-0}" +} + +echo "URL: $URL" +echo "Iterations: $N Sleep: ${SLEEP_SEC}s UA: $UA" +echo "RETRIES: $RETRIES DELAY: ${DELAY}s" +[[ -n "$JSON_URL" ]] && echo "JSON_URL: $JSON_URL" +[[ -n "$DOWNLOAD_BASE" ]] && echo "DOWNLOAD_BASE: $DOWNLOAD_BASE" +[[ "$DO_DOWNLOAD" == "1" ]] && echo "DO_DOWNLOAD: 1 (dir: $DOWNLOAD_DIR)" +echo + +for ((i=1; i<=N; i++)); do + body="$tmpdir/body.$i.html" + hdr="$tmpdir/hdr.$i.txt" + json_body="$tmpdir/json.$i.json" + json_hdr="$tmpdir/jsonhdr.$i.txt" + rendered_body="$tmpdir/rendered.$i.html" + + # 1) Fetch index like Ansible 'uri' task + read -r idx_status idx_attempts < <(fetch_index_like_ansible "$URL" "$body" "$hdr") + idx_retries_used=$((idx_attempts - 1)) + + # classify status for stats + if [[ "$idx_status" -eq 0 ]]; then + status_key="undefined" # like "status is undefined" + else + status_key="$idx_status" + fi + status_count["$status_key"]=$(( ${status_count["$status_key"]:-0} + 1 )) + retry_hist_index["$idx_retries_used"]=$(( ${retry_hist_index["$idx_retries_used"]:-0} + 1 )) + + size="$(wc -c < "$body" | tr -d ' ')" + note_size "$size" + + # 2) Detect whether APK links exist + has_apk=0 + if grep -Eqo "$APK_RE" "$body"; then + has_apk=1 + fi + + # 3) Optional JSON fallback if no APK link found + json_status=0 + json_attempts=0 + if [[ "$has_apk" -eq 0 && -n "$JSON_URL" ]]; then + read -r json_status json_attempts < <(fetch_json_best_effort "$JSON_URL" "$json_body" "$json_hdr") + json_retries_used=$((json_attempts - 1)) + retry_hist_json["$json_retries_used"]=$(( ${retry_hist_json["$json_retries_used"]:-0} + 1 )) + + # If 200 and we can extract rendered HTML, replace the body (mimic set_fact update) + if [[ "$json_status" -eq 200 ]]; then + extracted="$(extract_wp_rendered "$json_body" || true)" + if [[ -n "$extracted" ]]; then + printf '%s' "$extracted" > "$rendered_body" + body_to_parse="$rendered_body" + else + body_to_parse="$body" + fi + else + body_to_parse="$body" + fi + + # re-check APK links after fallback + if grep -Eqo "$APK_RE" "$body_to_parse"; then + has_apk=1 + fi + else + body_to_parse="$body" + fi + + # 4) Mimic code_blocked_by_cdn condition: + # blocked if 403 OR (apk_missing even after fallback) + blocked=0 + if [[ "$idx_status" -eq 403 || "$has_apk" -eq 0 ]]; then + blocked=1 + fi + blocked_count["$blocked"]=$(( ${blocked_count["$blocked"]:-0} + 1 )) + + # Additional signal (not in Ansible, but useful) + if grep -Eqi "$CF_RE" "$body_to_parse"; then + cf_count["cf_like"]=$(( ${cf_count["cf_like"]:-0} + 1 )) + else + cf_count["no_cf"]=$(( ${cf_count["no_cf"]:-0} + 1 )) + fi + + if [[ "$has_apk" -eq 1 ]]; then + apk_count["apk_found"]=$(( ${apk_count["apk_found"]:-0} + 1 )) + else + apk_count["apk_missing"]=$(( ${apk_count["apk_missing"]:-0} + 1 )) + fi + + # 5) Optional APK download like Ansible get_url + dl_ok=0 + dl_attempts=0 + dl_status=0 + if [[ "$DO_DOWNLOAD" == "1" && "$blocked" -eq 0 ]]; then + # Match playbook behavior: Jinja2 `sort` is lexicographic. + apk_name="$(grep -Eo "$APK_RE" "$body_to_parse" | LC_ALL=C sort | tail -n 1 || true)" + if [[ -n "$apk_name" ]]; then + # Match Ansible playbook behavior: download comes from a separate base domain. + # If DOWNLOAD_BASE is not set, fall back to the index URL's directory. + if [[ -n "$DOWNLOAD_BASE" ]]; then + apk_url="${DOWNLOAD_BASE%/}/$apk_name" + else + apk_url="${URL%/}/$apk_name" + fi + apk_dest="$DOWNLOAD_DIR/$apk_name" + read -r dl_ok dl_attempts dl_status < <(download_apk_like_ansible "$apk_url" "$apk_dest") + dl_retries_used=$((dl_attempts - 1)) + retry_hist_apk["$dl_retries_used"]=$(( ${retry_hist_apk["$dl_retries_used"]:-0} + 1 )) + fi + fi + + # progress line + printf "[%03d/%03d] idx=%s(attempts=%s) apk=%s blocked=%s json=%s(attempts=%s)\r" \ + "$i" "$N" \ + "${idx_status:-0}" "$idx_attempts" \ + "$([[ "$has_apk" -eq 1 ]] && echo Y || echo N)" \ + "$blocked" \ + "${json_status:-0}" "${json_attempts:-0}" + + sleep "$SLEEP_SEC" +done + +echo -e "\n\n==== SUMMARY (Ansible-like) ====" + +echo "Index status counts (0 means status undefined):" +for k in "${!status_count[@]}"; do + printf " %-10s %d\n" "$k" "${status_count[$k]}" +done | sort -k2,2nr + +echo +echo "APK armv8a links:" +for k in "${!apk_count[@]}"; do + printf " %-12s %d\n" "$k" "${apk_count[$k]}" +done | sort -k2,2nr + +echo +echo "Blocked (Ansible condition: idx==403 OR apk_missing):" +printf " blocked=1 %d\n" "${blocked_count[1]:-0}" +printf " blocked=0 %d\n" "${blocked_count[0]:-0}" + +echo +echo "Retry histogram (retries used):" +echo " Index fetch (retries used -> count):" +for k in "${!retry_hist_index[@]}"; do + printf " %s -> %d\n" "$k" "${retry_hist_index[$k]}" +done | sort -n -k1,1 + +if [[ -n "$JSON_URL" ]]; then + echo " JSON fallback (retries used -> count):" + for k in "${!retry_hist_json[@]}"; do + printf " %s -> %d\n" "$k" "${retry_hist_json[$k]}" + done | sort -n -k1,1 +fi + +if [[ "$DO_DOWNLOAD" == "1" ]]; then + echo " APK download (retries used -> count):" + for k in "${!retry_hist_apk[@]}"; do + printf " %s -> %d\n" "$k" "${retry_hist_apk[$k]}" + done | sort -n -k1,1 +fi + +echo +avg_size=$((sum_size / N)) +echo "Body size bytes: min=$min_size max=$max_size avg=$avg_size" + +echo +if [[ "$DO_DOWNLOAD" == "1" ]]; then + echo "Downloaded APKs are in: $DOWNLOAD_DIR" +fi + +echo "Note: files are saved under $tmpdir and will be auto-removed at exit (trap)."