ci/android: Fix and improve caching. Fix and improve android-commands.sh

.github/workflows/android.yml

@@ -35,7 +35,8 @@ jobs:
           ~/.android/avd/*
           ~/.android/avd/*/snapshots/*
           ~/.android/adb*
-        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest
+          ~/__rustc_hash__
+        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash
     - name: Create and cache emulator image
       if: steps.avd-cache.outputs.cache-hit != 'true'
       uses: reactivecircus/android-emulator-runner@v2
@@ -44,16 +45,11 @@ jobs:
         target: ${{ matrix.target }}
         arch: ${{ matrix.arch }}
         ram-size: 2048M
-        disk-size: 5120M
+        disk-size: 7GB
         force-avd-creation: true
         emulator-options: -no-snapshot-load -noaudio -no-boot-anim -camera-back none
         script: |
-          set -e
+          util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}"
-          wget https://github.com/termux/termux-app/releases/download/${{ env.TERMUX }}/termux-app_${{ env.TERMUX }}+github-debug_${{ matrix.arch }}.apk
-          util/android-commands.sh snapshot termux-app_${{ env.TERMUX }}+github-debug_${{ matrix.arch }}.apk
-          adb -s emulator-5554 emu avd snapshot save ${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}
-          echo "Emulator image created."
-          pkill -9 qemu-system-x86_64
     - name: Save AVD cache
       if: steps.avd-cache.outputs.cache-hit != 'true'
       uses: actions/cache/save@v3
@@ -62,7 +58,22 @@ jobs:
           ~/.android/avd/*
           ~/.android/avd/*/snapshots/*
           ~/.android/adb*
-        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest
+          ~/__rustc_hash__
+        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash
+    - uses: juliangruber/read-file-action@v1
+      id: read_rustc_hash
+      with:
+        # ~ expansion didn't work
+        path: /Users/runner/__rustc_hash__
+        trim: true
+    - name: Restore rust cache
+      id: rust-cache
+      uses: actions/cache/restore@v3
+      with:
+        path: ~/__rust_cache__
+        # The version vX at the end of the key is just a development version to avoid conflicts in
+        # the github cache during the development of this workflow
+        key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3
     - name: Build and Test
       uses: reactivecircus/android-emulator-runner@v2
       with:
@@ -70,10 +81,20 @@ jobs:
         target: ${{ matrix.target }}
         arch: ${{ matrix.arch }}
         ram-size: 2048M
-        disk-size: 5120M
+        disk-size: 7GB
         force-avd-creation: false
         emulator-options: -no-snapshot-save -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -snapshot ${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}
+        # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If
+        # one of the lines returns with error the whole script is failed (like running a script with
+        # set -e) and in consequences the other lines (shells) are not executed.
         script: |
-          util/android-commands.sh sync
+          util/android-commands.sh sync_host
           util/android-commands.sh build
           util/android-commands.sh tests
+          if [[ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]]; then util/android-commands.sh sync_image; fi; exit 0
+    - name: Save rust cache
+      if: steps.rust-cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v3
+      with:
+        path: ~/__rust_cache__
+        key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3

util/android-commands.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest
+# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest logfile
 
 # There are three shells: the host's, adb, and termux. Only adb lets us run
 # commands directly on the emulated device, only termux provides a GNU
@@ -8,29 +8,36 @@
 # This means that the commands sent to termux are first parsed as arguments in
 # this shell, then as arguments in the adb shell, before finally being used as
 # text inputs to the app. Hence, the "'wrapping'" on those commands.
-# There's no way to get any feedback from termux, so every time we run a
+# There's no way to get any direct feedback from termux, so every time we run a
-# command on it, we make sure it ends by creating a unique *.probe file at the
+# command on it, we make sure it creates a unique *.probe file which is polled
-# end of the command. The contents of the file are used as a return code: 0 on
+# every 30 seconds together with the current output of the command in a *.log file.
-# success, some other number for errors (an empty file is basically the same as
+# The contents of the probe file are used as a return code: 0 on success, some
-# 0). Note that the return codes are text, not raw bytes.
+# other number for errors (an empty file is basically the same as 0). Note that
+# the return codes are text, not raw bytes.
 
 this_repo="$(dirname "$(dirname -- "$(readlink -- "${0}")")")"
+cache_dir_name="__rust_cache__"
 
 help() {
     echo \
         "Usage: $0 COMMAND [ARG]
 
 where COMMAND is one of:
+  init          download termux and initialize the emulator image
   snapshot APK  install APK and dependencies on an emulator to prep a snapshot
                 (you can, but probably don't want to, run this for physical
                 devices -- just set up termux and the dependencies yourself)
-  sync [REPO]   push the repo at REPO to the device, deleting and restoring all
+  sync_host [REPO]
-                symlinks (locally) in the process; by default, REPO is:
+                push the repo at REPO to the device, deleting and restoring all symlinks (locally)
+                in the process; The cached rust directories are restored, too; by default, REPO is:
                 $this_repo
+  sync_image [REPO]
+                copy the repo/target and the HOME/.cargo directories from the device back to the
+                host; by default, REPO is: $this_repo
   build         run \`cargo build --features feat_os_unix_android\` on the
-                device, then pull the output as build.log
+                device
   tests         run \`cargo test --features feat_os_unix_android\` on the
-                device, then pull the output as tests.log
+                device
 
 If you have multiple devices, use the ANDROID_SERIAL environment variable to
 specify which to connect to."
@@ -40,6 +47,10 @@ hit_enter() {
     adb shell input keyevent 66
 }
 
+exit_termux() {
+    adb shell input text "exit" && hit_enter && hit_enter
+}
+
 launch_termux() {
     echo "launching termux"
     if ! adb shell 'am start -n com.termux/.HomeActivity'; then
@@ -56,127 +67,295 @@ launch_termux() {
     adb shell 'rm /sdcard/launch.probe' && echo "removed launch.probe"
 }
 
+# Usage: run_termux_command
+#
+# Runs the command specified in $1 in a termux shell, polling for the probe specified in $2 (and the
+# current output). If polling the probe succeeded the command is considered to have finished. This
+# method prints the current stdout and stderr of the command every SLEEP_INTERVAL seconds and
+# finishes a command run with a summary. It returns with the exit code of the probe if specified as
+# file content of the probe.
+#
+# Positional arguments
+# $1                The command to execute in the termux shell
+# $2                The path to the probe. The file name must end with `.probe`
+#
+# It's possible to overwrite settings by specifying the setting the variable before calling this
+# method (Default in parentheses):
+# keep_log  0|1     Keeps the logs after running the command if set to 1. The log file name is
+#                   derived from the probe file name (the last component of the path) and
+#                   `.probe` replaced with `.log. (0)
+# debug     0|1     Adds additional debugging output to the log file if set to 1. (1)
+# timeout   SECONDS The timeout in full SECONDS for the command to complete before giving up. (3600)
+# retries   RETRIES The number of retries for trying to fix possible issues when we're not receiving
+#                   any progress from the emulator. (3)
+# sleep_interval
+#           SECONDS The time interval in full SECONDS between polls for the probe and the current
+#           output. (5)
 run_termux_command() {
-    command="$1" # text of the escaped command, including creating the probe!
+    # shellcheck disable=SC2155
-    probe="$2"   # unique file that indicates the command is complete
+    local command="$(echo "$1" | sed -E "s/^['](.*)[']$/\1/")" # text of the escaped command, including creating the probe!
+    local probe="$2"                                           # unique file that indicates the command is complete
+    local keep_log=${keep_log:-0}
+    local debug=${debug:-1}
+
+    log_name="$(basename -s .probe "${probe}").log" # probe name must have suffix .probe
+    log_file="/sdcard/${log_name}"
+    log_read="${log_name}.read"
+    echo 0 >"${log_read}"
+    if [[ $debug -eq 1 ]]; then
+        shell_command="'set -x; { ${command}; } &> ${log_file}; set +x'"
+    else
+        shell_command="'{ ${command}; } &> ${log_file}'"
+    fi
+
     launch_termux
-    adb shell input text "$command" && hit_enter
+    echo "Running command: ${command}"
+    start=$(date +%s)
+    adb shell input text "$shell_command" && sleep 3 && hit_enter
+    # just for safety wait a little bit before polling for the probe and the log file
+    sleep 5
+
+    local timeout=${timeout:-3600}
+    local retries=${retries:-3}
+    local sleep_interval=${sleep_interval:-5}
+    try_fix=3
     while ! adb shell "ls $probe" 2>/dev/null; do
-        echo "waiting for $probe"
+        echo -n "Waiting for $probe: "
-        sleep 30
+
+        if [[ -e "$log_name" ]]; then
+            rm "$log_name"
+        fi
+
+        adb pull "$log_file" . || try_fix=$((try_fix - 1))
+        if [[ -e "$log_name" ]]; then
+            tail -n +"$(<"$log_read")" "$log_name"
+            echo
+            wc -l <"${log_name}" | tr -d "[:space:]" >"$log_read"
+        fi
+
+        if [[ retries -le 0 ]]; then
+            echo "Maximum retries reached running command. Aborting ..."
+            return 1
+        elif [[ try_fix -le 0 ]]; then
+            retries=$((retries - 1))
+            try_fix=3
+            # Since there is no output, there is no way to know what is happening inside. See if
+            # hitting the enter key solves the issue, sometimes the github runner is just a little
+            # bit slow.
+            echo "No output received. Trying to fix the issue ... (${retries} retries left)"
+            hit_enter
+        fi
+
+        sleep "$sleep_interval"
+        timeout=$((timeout - sleep_interval))
+
+        if [[ $timeout -le 0 ]]; then
+            echo "Timeout reached running command. Aborting ..."
+            return 1
+        fi
     done
-    return_code=$(adb shell "cat $probe")
+    end=$(date +%s)
-    adb shell "rm $probe"
+
-    echo "return code: $return_code"
+    return_code=$(adb shell "cat $probe") || return_code=0
-    return "$return_code"
+    adb shell "rm ${probe}"
+
+    adb pull "$log_file" .
+    echo "==================================== SUMMARY ==================================="
+    echo "Command: ${command}"
+    echo "Finished in $((end - start)) seconds."
+    echo "Output was:"
+    cat "$log_name"
+    echo "Return code: $return_code"
+    echo "================================================================================"
+
+    adb shell "rm ${log_file}"
+    [[ $keep_log -ne 1 ]] && rm -f "$log_name"
+    rm -f "$log_read" "$probe"
+
+    # shellcheck disable=SC2086
+    return $return_code
+}
+
+init() {
+    arch="$1"
+    api_level="$2"
+    termux="$3"
+
+    # shellcheck disable=SC2015
+    wget "https://github.com/termux/termux-app/releases/download/${termux}/termux-app_${termux}+github-debug_${arch}.apk" &&
+        snapshot "termux-app_${termux}+github-debug_${arch}.apk" &&
+        hash_rustc &&
+        exit_termux &&
+        adb -s emulator-5554 emu avd snapshot save "${api_level}-${arch}+termux-${termux}" &&
+        echo "Emulator image created." || {
+        pkill -9 qemu-system-x86_64
+        return 1
+    }
+    pkill -9 qemu-system-x86_64 || true
 }
 
 snapshot() {
     apk="$1"
-    echo "running snapshot"
+    echo "Running snapshot"
     adb install -g "$apk"
 
     echo "Prepare and install system packages"
     probe='/sdcard/pkg.probe'
-    log='/sdcard/pkg.log'
+    command="'mkdir -vp ~/.cargo/bin; yes | pkg install rust binutils openssl tar -y; echo \$? > $probe'"
-    command="'{ mkdir -vp ~/.cargo/bin; yes | pkg install rust binutils openssl -y; echo \$? > $probe; } &> $log'"
+    run_termux_command "$command" "$probe" || return
-    run_termux_command "$command" "$probe"
-    return_code=$?
-
-    adb pull "$log" .
-    cat "$(basename "$log")"
-
-    if [[ $return_code -ne 0 ]]; then return $return_code; fi
 
     echo "Installing cargo-nextest"
     probe='/sdcard/nextest.probe'
-    log='/sdcard/nextest.log'
     # We need to install nextest via cargo currently, since there is no pre-built binary for android x86
-    command="'cargo install cargo-nextest &> $log; touch $probe'"
+    command="'\
+export CARGO_TERM_COLOR=always; \
+cargo install cargo-nextest; \
+echo \$? > $probe'"
     run_termux_command "$command" "$probe"
-
+    return_code=$?
-    adb pull "$log" .
-    cat "$(basename "$log")"
 
     echo "Info about cargo and rust"
     probe='/sdcard/info.probe'
-    log='/sdcard/info.log'
+    command="'echo \$HOME; \
-    command="'{ \
+PATH=\$HOME/.cargo/bin:\$PATH; \
-        set -x; \
+export PATH; \
-        echo \$HOME; \
+echo \$PATH; \
-        PATH=\$HOME/.cargo/bin:\$PATH; \
+pwd; \
-        export PATH; \
+command -v rustc && rustc -Vv; \
-        echo \$PATH; \
+ls -la ~/.cargo/bin; \
-        pwd; \
+cargo --list; \
-        command -v rustc && rustc --version; \
+cargo nextest --version; \
-        ls -la ~/.cargo/bin; \
+touch $probe'"
-        cargo --list; \
-        cargo nextest --version; \
-        set +x; \
-        } &> $log; touch $probe'"
     run_termux_command "$command" "$probe"
 
-    adb pull "$log" .
+    echo "Snapshot complete"
-    cat "$(basename "$log")"
+    # shellcheck disable=SC2086
-
+    return $return_code
-    echo "snapshot complete"
-    adb shell input text "exit" && hit_enter && hit_enter
 }
 
-sync() {
+sync_host() {
     repo="$1"
-    echo "running sync $1"
+    cache_home="${HOME}/${cache_dir_name}"
+    cache_dest="/sdcard/${cache_dir_name}"
+
+    echo "Running sync host -> image: ${repo}"
+
     # android doesn't allow symlinks on shared dirs, and adb can't selectively push files
     symlinks=$(find "$repo" -type l)
     # dash doesn't support process substitution :(
     echo "$symlinks" | sort >symlinks
+
     git -C "$repo" diff --name-status | cut -f 2 >modified
     modified_links=$(join symlinks modified)
     if [ -n "$modified_links" ]; then
         echo "You have modified symlinks. Either stash or commit them, then try again: $modified_links"
         exit 1
     fi
-    if ! git ls-files --error-unmatch "$symlinks" >/dev/null; then
+    #shellcheck disable=SC2086
+    if ! git ls-files --error-unmatch $symlinks >/dev/null; then
         echo "You have untracked symlinks. Either remove or commit them, then try again."
         exit 1
     fi
-    rm "$symlinks"
+
+    #shellcheck disable=SC2086
+    rm $symlinks
     # adb's shell user only has access to shared dirs...
-    adb push "$repo" /sdcard/coreutils
+    adb push -a "$repo" /sdcard/coreutils
-    git -C "$repo" checkout "$symlinks"
+    [[ -e "$cache_home" ]] && adb push -a "$cache_home" "$cache_dest"
+
+    #shellcheck disable=SC2086
+    git -C "$repo" checkout $symlinks
+
     # ...but shared dirs can't build, so move it home as termux
-    probe='/sdcard/mv.probe'
+    probe='/sdcard/sync.probe'
-    command="'cp -r /sdcard/coreutils ~/; touch $probe'"
+    command="'mv /sdcard/coreutils ~/; \
-    run_termux_command "$command" "$probe"
+cd ~/coreutils; \
+if [[ -e ${cache_dest} ]]; then \
+rm -rf ~/.cargo ./target; \
+tar xzf ${cache_dest}/cargo.tgz -C ~/; \
+ls -la ~/.cargo; \
+tar xzf ${cache_dest}/target.tgz; \
+ls -la ./target; \
+rm -rf ${cache_dest}; \
+fi; \
+touch $probe'"
+    run_termux_command "$command" "$probe" || return
+
+    echo "Finished sync host -> image: ${repo}"
+}
+
+sync_image() {
+    repo="$1"
+    cache_home="${HOME}/${cache_dir_name}"
+    cache_dest="/sdcard/${cache_dir_name}"
+
+    echo "Running sync image -> host: ${repo}"
+
+    probe='/sdcard/cache.probe'
+    command="'rm -rf /sdcard/coreutils ${cache_dest}; \
+mkdir -p ${cache_dest}; \
+cd ${cache_dest}; \
+tar czf cargo.tgz -C ~/ .cargo; \
+tar czf target.tgz -C ~/coreutils target; \
+ls -la ${cache_dest}; \
+echo \$? > ${probe}'"
+    run_termux_command "$command" "$probe" || return
+
+    rm -rf "$cache_home"
+    adb pull -a "$cache_dest" "$cache_home" || return
+
+    echo "Finished sync image -> host: ${repo}"
 }
 
 build() {
+    echo "Running build"
+
     probe='/sdcard/build.probe'
-    command="'cd ~/coreutils && cargo build --features feat_os_unix_android 2>/sdcard/build.log; echo \$? >$probe'"
+    command="'export CARGO_TERM_COLOR=always; \
-    echo "running build"
+export CARGO_INCREMENTAL=0; \
-    run_termux_command "$command" "$probe"
+cd ~/coreutils && cargo build --features feat_os_unix_android; \
-    return_code=$?
+echo \$? >$probe'"
-    adb pull /sdcard/build.log .
+    run_termux_command "$command" "$probe" || return
-    cat build.log
+
-    return $return_code
+    echo "Finished build"
 }
 
 tests() {
+    echo "Running tests"
+
     probe='/sdcard/tests.probe'
-    command="'\
+    command="'export PATH=\$HOME/.cargo/bin:\$PATH; \
-        export PATH=\$HOME/.cargo/bin:\$PATH; \
+export RUST_BACKTRACE=1; \
-        export RUST_BACKTRACE=1; \
+export CARGO_TERM_COLOR=always; \
-        export CARGO_TERM_COLOR=always; \
+export CARGO_INCREMENTAL=0; \
-        cd ~/coreutils || { echo 1 > $probe; exit; }; \
+cd ~/coreutils; \
-        timeout --preserve-status --verbose -k 1m 60m \
+timeout --preserve-status --verbose -k 1m 60m \
-            cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android \
+    cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android; \
-            &>/sdcard/tests.log; \
+echo \$? >$probe'"
-        echo \$? >$probe'"
+    run_termux_command "$command" "$probe" || return
-    run_termux_command "$command" "$probe"
+
-    return_code=$?
+    echo "Finished tests"
-    adb pull /sdcard/tests.log .
+}
-    cat tests.log
+
-    return $return_code
+hash_rustc() {
+    probe='/sdcard/rustc.probe'
+    tmp_hash="__rustc_hash__.tmp"
+    hash="__rustc_hash__"
+
+    echo "Hashing rustc version: ${HOME}/${hash}"
+
+    command="'rustc -Vv; echo \$? > ${probe}'"
+    keep_log=1
+    debug=0
+    run_termux_command "$command" "$probe" || return
+    rm -f "$tmp_hash"
+    mv "rustc.log" "$tmp_hash" || return
+    # sha256sum is not available. shasum is the macos native program.
+    shasum -a 256 "$tmp_hash" | cut -f 1 -d ' ' | tr -d '[:space:]' >"${HOME}/${hash}" || return
+
+    rm -f "$tmp_hash"
+
+    echo "Finished hashing rustc version: ${HOME}/${hash}"
 }
 
 #adb logcat &
@@ -184,8 +363,12 @@ exit_code=0
 
 if [ $# -eq 1 ]; then
     case "$1" in
-        sync)
+        sync_host)
-            sync "$this_repo"
+            sync_host "$this_repo"
+            exit_code=$?
+            ;;
+        sync_image)
+            sync_image "$this_repo"
             exit_code=$?
             ;;
         build)
@@ -204,8 +387,24 @@ elif [ $# -eq 2 ]; then
             snapshot "$2"
             exit_code=$?
             ;;
-        sync)
+        sync_host)
-            sync "$2"
+            sync_host "$2"
+            exit_code=$?
+            ;;
+        sync_image)
+            sync_image "$2"
+            exit_code=$?
+            ;;
+        *)
+            help
+            exit 1
+            ;;
+    esac
+elif [ $# -eq 4 ]; then
+    case "$1" in
+        init)
+            shift
+            init "$@"
             exit_code=$?
             ;;
         *)