Merge pull request #5939 from cre4ture/fix/stabilize_improve_android_build

`ci:android` stabilize and improve android github actions

.github/workflows/android.yml

@@ -17,20 +17,55 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
+env:
+  TERMUX: v0.118.0
+  KEY_POSTFIX: nextest+rustc-hash+adb+sshd+upgrade+XGB+inc15
+  COMMON_EMULATOR_OPTIONS: -no-window -noaudio -no-boot-anim -camera-back none -gpu swiftshader_indirect
+  EMULATOR_DISK_SIZE: 12GB
+  EMULATOR_HEAP_SIZE: 2048M
+  EMULATOR_BOOT_TIMEOUT: 1200 # 20min
+
 jobs:
   test_android:
     name: Test builds
-    runs-on: macos-latest
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu-latest] # , macos-latest
+        cores: [4] # , 6
+        ram: [4096, 8192]
         api-level: [28]
         target: [default]
         arch: [x86, x86_64] # , arm64-v8a
+        exclude:
+          - ram: 8192
+            arch: x86
+          - ram: 4096
+            arch: x86_64
+    runs-on: ${{ matrix.os }}
     env:
-      TERMUX: v0.118.0
+      EMULATOR_RAM_SIZE: ${{ matrix.ram }}
+      EMULATOR_CORES: ${{ matrix.cores }}
+      RUNNER_OS: ${{ matrix.os }}
+      AVD_CACHE_KEY: "set later due to limitations of github actions not able to concatenate env variables"
     steps:
+    - name: Concatenate values to environment file
+      run: |
+        echo "AVD_CACHE_KEY=${{ matrix.os }}-${{ matrix.cores }}-${{ matrix.ram }}-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+${{ env.KEY_POSTFIX }}" >> $GITHUB_ENV
+    - name: Collect information about runner
+      run: |
+        hostname
+        uname -a
+        free -mh
+        df -h
+        cat /proc/cpuinfo
+    - name: Enable KVM group perms (linux hardware acceleration)
+      if: ${{ runner.os == 'Linux' }}
+      run: |
+        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+        sudo udevadm control --reload-rules
+        sudo udevadm trigger --name-match=kvm
     - uses: actions/checkout@v4
     - name: Restore AVD cache
       uses: actions/cache/restore@v4
@@ -41,7 +76,7 @@ jobs:
           ~/.android/avd/*/snapshots/*
           ~/.android/adb*
           ~/__rustc_hash__
-        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash
+        key: avd-${{ env.AVD_CACHE_KEY }}
     - name: Delete AVD Lockfile when run from cache
       if: steps.avd-cache.outputs.cache-hit == 'true'
       run: |
@@ -50,15 +85,18 @@ jobs:
           ~/.android/avd/*/*.lock              
     - name: Create and cache emulator image
       if: steps.avd-cache.outputs.cache-hit != 'true'
-      uses: reactivecircus/android-emulator-runner@v2
+      uses: reactivecircus/android-emulator-runner@v2.30.1
       with:
         api-level: ${{ matrix.api-level }}
         target: ${{ matrix.target }}
         arch: ${{ matrix.arch }}
-        ram-size: 2048M
+        ram-size: ${{ env.EMULATOR_RAM_SIZE }}
-        disk-size: 7GB
+        heap-size: ${{ env.EMULATOR_HEAP_SIZE }}
+        disk-size: ${{ env.EMULATOR_DISK_SIZE }}
+        cores: ${{ env.EMULATOR_CORES }}
         force-avd-creation: true
-        emulator-options: -no-window -no-snapshot-load -noaudio -no-boot-anim -camera-back none
+        emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-load
+        emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }}
         script: |
           util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}"
     - name: Save AVD cache
@@ -70,12 +108,12 @@ jobs:
           ~/.android/avd/*/snapshots/*
           ~/.android/adb*
           ~/__rustc_hash__
-        key: avd-${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+nextest+rustc-hash
+        key: avd-${{ env.AVD_CACHE_KEY }}
     - uses: juliangruber/read-file-action@v1
       id: read_rustc_hash
       with:
         # ~ expansion didn't work
-        path: /Users/runner/__rustc_hash__
+        path: ${{ runner.os == 'Linux' && '/home/runner/__rustc_hash__' || '/Users/runner/__rustc_hash__' }}
         trim: true
     - name: Restore rust cache
       id: rust-cache
@@ -86,15 +124,18 @@ jobs:
         # the github cache during the development of this workflow
         key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3
     - name: Build and Test
-      uses: reactivecircus/android-emulator-runner@v2
+      uses: reactivecircus/android-emulator-runner@v2.30.1
       with:
         api-level: ${{ matrix.api-level }}
         target: ${{ matrix.target }}
         arch: ${{ matrix.arch }}
-        ram-size: 2048M
+        ram-size: ${{ env.EMULATOR_RAM_SIZE }}
-        disk-size: 7GB
+        heap-size: ${{ env.EMULATOR_HEAP_SIZE }}
+        disk-size: ${{ env.EMULATOR_DISK_SIZE }}
+        cores: ${{ env.EMULATOR_CORES }}
         force-avd-creation: false
-        emulator-options: -no-window -no-snapshot-save -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -snapshot ${{ matrix.api-level }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}
+        emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-save -snapshot ${{ env.AVD_CACHE_KEY }}
+        emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }}
         # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If
         # one of the lines returns with error the whole script is failed (like running a script with
         # set -e) and in consequences the other lines (shells) are not executed.
@@ -109,3 +150,9 @@ jobs:
       with:
         path: ~/__rust_cache__
         key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3
+    - name: archive any output (error screenshots)
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test_output_${{ env.AVD_CACHE_KEY }}
+        path: output

src/uucore/src/lib/features/fsext.rs

@@ -664,7 +664,7 @@ impl FsMeta for StatFs {
             any(
                 target_arch = "s390x",
                 target_vendor = "apple",
-                target_os = "android",
+                all(target_os = "android", target_pointer_width = "32"),
                 target_os = "openbsd",
                 not(target_pointer_width = "64")
             )
@@ -675,7 +675,8 @@ impl FsMeta for StatFs {
             target_os = "freebsd",
             target_os = "illumos",
             target_os = "solaris",
-            target_os = "redox"
+            target_os = "redox",
+            all(target_os = "android", target_pointer_width = "64"),
         ))]
         return self.f_bsize.try_into().unwrap();
     }
@@ -741,14 +742,17 @@ impl FsMeta for StatFs {
             not(target_env = "musl"),
             any(
                 target_vendor = "apple",
-                target_os = "android",
+                all(target_os = "android", target_pointer_width = "32"),
                 target_os = "freebsd",
                 target_arch = "s390x",
                 not(target_pointer_width = "64")
             )
         ))]
         return self.f_type.into();
-        #[cfg(target_env = "musl")]
+        #[cfg(any(
+            target_env = "musl",
+            all(target_os = "android", target_pointer_width = "64"),
+        ))]
         return self.f_type.try_into().unwrap();
     }
     #[cfg(not(any(

util/android-commands.sh

@@ -1,22 +1,72 @@
 #!/usr/bin/env bash
 # spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest logfile
+# spell-checker:ignore screencap reinit PIPESTATUS keygen sourceslist
 
-# There are three shells: the host's, adb, and termux. Only adb lets us run
+# There are four shells: the host's, adb, termux and termux via ssh.
-# commands directly on the emulated device, only termux provides a GNU
+# But only termux and termux via ssh provides a GNU environment on the
-# environment on the emulated device (to e.g. run cargo). So we use adb to
+# emulated device (to e.g. run cargo).
-# launch termux, then to send keystrokes to it while it's running.
+# Initially, only adb lets us run commands directly on the emulated device.
-# This means that the commands sent to termux are first parsed as arguments in
+# Thus we first establish a ssh connection which then can be used to access
+# the termux shell directly, getting output and return code as usual.
+# So we use adb to launch termux, then to send keystrokes to it while it's running.
+# This way we install sshd and a public key from the host. After that we can
+# use ssh to directly run commands in termux environment.
+
+# Before ssh, we need to consider some inconvenient, limiting specialties:
+# The commands sent to termux via adb keystrokes are first parsed as arguments in
 # this shell, then as arguments in the adb shell, before finally being used as
 # text inputs to the app. Hence, the "'wrapping'" on those commands.
-# There's no way to get any direct feedback from termux, so every time we run a
+# Using this approach there's no way to get any direct feedback from termux,
-# command on it, we make sure it creates a unique *.probe file which is polled
+# so every time we run a command on it, we make sure it creates a unique *.probe file
-# every 30 seconds together with the current output of the command in a *.log file.
+# which is polled every 30 seconds together with the current output of the
-# The contents of the probe file are used as a return code: 0 on success, some
+# command in a *.log file. The contents of the probe file are used as a return code:
-# other number for errors (an empty file is basically the same as 0). Note that
+# 0 on success, some other number for errors (an empty file is basically the same as 0).
-# the return codes are text, not raw bytes.
+# Note that the return codes are text, not raw bytes.
+
+# Additionally, we can use adb screenshot functionality to investigate issues
+# when there is no feedback arriving from the android device.
 
 this_repo="$(dirname "$(dirname -- "$(readlink -- "${0}")")")"
 cache_dir_name="__rust_cache__"
+dev_probe_dir=/sdcard
+dev_home_dir=/data/data/com.termux/files/home
+
+# This is a list of termux package mirrors approved to be used.
+# The default mirror list contains entries that do not function properly anymore.
+# To avoid failures due to broken mirrors, we use our own list.
+# Choose only reliable mirrors here:
+repo_url_list=(
+    "deb https://packages-cf.termux.org/apt/termux-main/ stable main"
+    "deb https://packages-cf.termux.dev/apt/termux-main/ stable main"
+#    "deb https://grimler.se/termux/termux-main stable main"  # slow
+    "deb https://ftp.fau.de/termux/termux-main stable main"
+)
+number_repo_urls=${#repo_url_list[@]}
+repo_url_round_robin=$RANDOM
+
+move_to_next_repo_url() {
+    repo_url_round_robin=$(((repo_url_round_robin + 1) % number_repo_urls))
+    echo "next round robin repo_url: $repo_url_round_robin"
+}
+move_to_next_repo_url # first call needed for modulo
+
+get_current_repo_url() {
+    echo "${repo_url_list[$repo_url_round_robin]}"
+}
+
+# dump some information about the runners system for debugging purposes:
+echo "====== runner information ======"
+echo "hostname: $(hostname)"
+echo "uname -a: $(uname -a)"
+echo "pwd: $(pwd)"
+echo "\$*: $*"
+echo "\$0: $0"
+# shellcheck disable=SC2140
+echo "\$(readlink -- "\$\{0\}"): $(readlink -- "${0}")"
+echo "\$this_repo: $this_repo"
+echo "readlink -f \$this_repo: $(readlink -f "$this_repo")"
+this_repo=$(readlink -f "$this_repo")
+echo "====== runner info end ========="
 
 help() {
     echo \
@@ -48,23 +98,82 @@ hit_enter() {
 }
 
 exit_termux() {
-    adb shell input text "exit" && hit_enter && hit_enter
+    adb shell input text \"exit\" && hit_enter && hit_enter
+}
+
+timestamp() {
+  date +"%H%M%S%Z"
+}
+
+add_timestamp_to_lines() {
+    while IFS= read -r line; do printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$line"; done
+}
+
+# takes a screenshot with given name from the android device. Filename is prefixed with timestamp.
+# screenshots are collected at the end of the github workflow and provided as download link.
+take_screen_shot() {
+    filename_prefix="$1"
+    filename="$this_repo/output/$(timestamp)_${filename_prefix}_screen.png"
+    echo "take screenshot: $filename"
+    mkdir -p "$this_repo/output"
+    adb exec-out screencap -p > "$filename"
 }
 
 launch_termux() {
     echo "launching termux"
+    take_screen_shot "launch_termux_enter"
+
+    adb shell input tap 120 380  # close potential dialog "System UI isn't responding" with "wait".
+                                 # should not cause side effects when dialog is not there as there are
+                                 # no relevant GUI elements at this position otherwise.
+
     if ! adb shell 'am start -n com.termux/.HomeActivity'; then
         echo "failed to launch termux"
         exit 1
     fi
+
+    take_screen_shot "launch_termux_after_start_activity"
+
     # the emulator can sometimes be a little slow to launch the app
-    while ! adb shell 'ls /sdcard/launch.probe' 2>/dev/null; do
+    loop_count=0
-        echo "waiting for launch.probe"
+    while ! adb shell "dumpsys window windows" | \
-        sleep 5
+            grep -E "imeInputTarget in display# 0 Window{[^}]+com.termux\/com\.termux\.HomeActivity}"
-        adb shell input text 'touch\ /sdcard/launch.probe' && hit_enter
+    do
+        sleep 1
+        loop_count=$((loop_count + 1))
+        if [[ loop_count -ge 20 ]]; then
+            break
+        fi
+    done
+
+    take_screen_shot "launch_termux_after_wait_activity"
+
+    touch_cmd() {
+        adb shell input text "\"touch $dev_probe_dir/launch.probe\"" && hit_enter
+        sleep 1
+    }
+
+    local timeout_start=120
+    local timeout=$timeout_start
+    touch_cmd
+    while ! adb shell "ls $dev_probe_dir/launch.probe" 2>/dev/null
+    do
+        echo "waiting for launch.probe - ($timeout / $timeout_start seconds)"
+        take_screen_shot "launch_termux_touch_probe"
+        sleep 4
+        touch_cmd
+
+        timeout=$((timeout - 4))
+        if [[ timeout -le 0 ]]; then
+            take_screen_shot "error_launch_termux"
+            echo "timeout waiting for termux to start up"
+            return 1
+        fi
+
     done
     echo "found launch.probe"
-    adb shell 'rm /sdcard/launch.probe' && echo "removed launch.probe"
+    take_screen_shot "launch_termux_found_probe"
+    adb shell "rm $dev_probe_dir/launch.probe" && echo "removed launch.probe"
 }
 
 # Usage: run_termux_command
@@ -99,7 +208,7 @@ run_termux_command() {
     local debug=${debug:-1}
 
     log_name="$(basename -s .probe "${probe}").log" # probe name must have suffix .probe
-    log_file="/sdcard/${log_name}"
+    log_file="$dev_probe_dir/${log_name}"
     log_read="${log_name}.read"
     echo 0 >"${log_read}"
     if [[ $debug -eq 1 ]]; then
@@ -108,12 +217,22 @@ run_termux_command() {
         shell_command="'{ ${command}; } &> ${log_file}'"
     fi
 
-    launch_termux
+    launch_termux || return
+
+    take_screen_shot "run_termux_command_before_input_of_shell_command"
+
+    # remove artificial quoting
+    shell_command="${shell_command%\'}"
+    shell_command="${shell_command#\'}"
+
     echo "Running command: ${command}"
+    echo "Running shell-command: ${shell_command}"
     start=$(date +%s)
-    adb shell input text "$shell_command" && sleep 3 && hit_enter
+    adb_input_text_long "$shell_command" && sleep 1 && hit_enter
     # just for safety wait a little bit before polling for the probe and the log file
-    sleep 5
+    sleep 1
+
+    take_screen_shot "run_termux_command_after_input_of_shell_command"
 
     local timeout=${timeout:-3600}
     local retries=${retries:-10}
@@ -136,6 +255,7 @@ run_termux_command() {
 
         if [[ retries -le 0 ]]; then
             echo "Maximum retries reached running command. Aborting ..."
+            take_screen_shot "run_termux_command_maximum_tries_reached"
             return 1
         elif [[ try_fix -le 0 ]]; then
             retries=$((retries - 1))
@@ -144,7 +264,10 @@ run_termux_command() {
             # hitting the enter key solves the issue, sometimes the github runner is just a little
             # bit slow.
             echo "No output received. Trying to fix the issue ... (${retries} retries left)"
+            take_screen_shot "run_termux_command_before_trying_to_fix"
             hit_enter
+            sleep 1
+            take_screen_shot "run_termux_command_after_trying_to_fix"
         fi
 
         sleep "$sleep_interval"
@@ -152,6 +275,7 @@ run_termux_command() {
 
         if [[ $timeout -le 0 ]]; then
             echo "Timeout reached running command. Aborting ..."
+            take_screen_shot "run_termux_command_timeout_reached"
             return 1
         fi
     done
@@ -160,7 +284,7 @@ run_termux_command() {
     return_code=$(adb shell "cat $probe") || return_code=0
     adb shell "rm ${probe}"
 
-    adb pull "$log_file" .
+    adb shell "cat $log_file" > "$log_name"
     echo "==================================== SUMMARY ==================================="
     echo "Command: ${command}"
     echo "Finished in $((end - start)) seconds."
@@ -173,63 +297,214 @@ run_termux_command() {
     [[ $keep_log -ne 1 ]] && rm -f "$log_name"
     rm -f "$log_read" "$probe"
 
+    take_screen_shot "run_termux_command_finished_normally"
+
     # shellcheck disable=SC2086
     return $return_code
 }
 
 init() {
     arch="$1"
+    # shellcheck disable=SC2034
     api_level="$2"
     termux="$3"
 
+    snapshot_name="${AVD_CACHE_KEY}"
+
     # shellcheck disable=SC2015
     wget -nv "https://github.com/termux/termux-app/releases/download/${termux}/termux-app_${termux}+github-debug_${arch}.apk" &&
         snapshot "termux-app_${termux}+github-debug_${arch}.apk" &&
         hash_rustc &&
         exit_termux &&
-        adb -s emulator-5554 emu avd snapshot save "${api_level}-${arch}+termux-${termux}" &&
+        adb -s emulator-5554 emu avd snapshot save "$snapshot_name" &&
-        echo "Emulator image created." || {
+        echo "Emulator image created. Name: $snapshot_name" || {
         pkill -9 qemu-system-x86_64
         return 1
     }
     pkill -9 qemu-system-x86_64 || true
 }
 
+reinit_ssh_connection() {
+    setup_ssh_forwarding
+    test_ssh_connection && return
+
+    start_sshd_via_adb_shell && (
+        test_ssh_connection && return
+        generate_and_install_public_key && test_ssh_connection && return
+    ) || (
+        install_packages_via_adb_shell openssh openssl
+        generate_and_install_public_key
+        start_sshd_via_adb_shell
+        test_ssh_connection && return
+    ) || (
+        echo "failed to setup ssh connection"
+        return 1
+    )
+}
+
+start_sshd_via_adb_shell() {
+    echo "start sshd via adb shell"
+    probe="$dev_probe_dir/sshd.probe"
+    command="'sshd; echo \$? > $probe'"
+    run_termux_command "$command" "$probe"
+}
+
+setup_ssh_forwarding() {
+    echo "setup ssh forwarding"
+    adb forward tcp:9022 tcp:8022
+}
+
+copy_file_or_dir_to_device_via_ssh() {
+    scp -r "$1" "scp://termux@127.0.0.1:9022/$2"
+}
+
+copy_file_or_dir_from_device_via_ssh() {
+    scp -r "scp://termux@127.0.0.1:9022/$1" "$2"
+}
+
+# runs the in args provided command on android side via ssh. forwards return code.
+# adds a timestamp to every line to be able to see where delays are
+run_command_via_ssh() {
+    ssh -p 9022 termux:@127.0.0.1 -o StrictHostKeyChecking=accept-new "$@" 2>&1 | add_timestamp_to_lines
+    return "${PIPESTATUS[0]}"
+}
+
+test_ssh_connection() {
+    run_command_via_ssh echo ssh connection is working
+    run_command_via_ssh free -mh
+}
+
+# takes a local (on runner side) script file and runs it via ssh on the virtual android device. forwards return code.
+# adds a timestamp to every line to be able to see where delays are
+run_script_file_via_ssh() {
+    ssh -p 9022 termux:@127.0.0.1 -o StrictHostKeyChecking=accept-new "bash -s" < "$1" 2>&1 | add_timestamp_to_lines
+    return "${PIPESTATUS[0]}"
+}
+
+# Experiments showed that the adb shell input text functionality has a limitation for the input length.
+# If input length is too big, the input is not fully provided to the android device.
+# To avoid this, we divide large inputs into smaller chunks and put them one-by-one.
+adb_input_text_long() {
+    string=$1
+    length=${#string}
+    step=20
+    p=0
+    for ((i = 0; i < length-step; i = i + step)); do
+        chunk="${string:i:$step}"
+        adb shell input text "'$chunk'"
+        p=$((i+step))
+    done
+
+    remaining="${string:p}"
+    adb shell input text "'$remaining'"
+}
+
+generate_rsa_key_local() {
+    yes "" | ssh-keygen -t rsa -b 4096 -C "Github Action" -N ""
+}
+
+install_rsa_pub() {
+
+    run_command_via_ssh "echo hello" && return  # if this works, we are already fine. Skipping
+
+    # remove old host identity:
+    ssh-keygen -f ~/.ssh/known_hosts -R "[127.0.0.1]:9022"
+
+    rsa_pub_key=$(cat ~/.ssh/id_rsa.pub)
+    echo "====================================="
+    echo "$rsa_pub_key"
+    echo "====================================="
+
+    adb shell input text \"echo \"
+
+    adb_input_text_long "$rsa_pub_key"
+
+    adb shell input text "\" >> ~/.ssh/authorized_keys\"" && hit_enter
+    sleep 1
+}
+
+install_packages_via_adb_shell() {
+    install_package_list="$*"
+
+    install_packages_via_adb_shell_using_apt "$install_package_list"
+    if [[ $? -ne 0 ]]; then
+        echo "apt failed. Now try install with pkg as fallback."
+        probe="$dev_probe_dir/pkg.probe"
+        command="'mkdir -vp ~/.cargo/bin; yes | pkg install $install_package_list -y; echo \$? > $probe'"
+        run_termux_command "$command" "$probe" || return 1
+    fi
+
+    return 0
+}
+
+# We use apt to install the packages as pkg doesn't respect any pre-defined mirror list.
+# Its important to have a defined mirror list to avoid issues with broken mirrors.
+install_packages_via_adb_shell_using_apt() {
+    install_package_list="$*"
+
+    repo_url=$(get_current_repo_url)
+    move_to_next_repo_url
+    echo "set apt repository url: $repo_url"
+    probe="$dev_probe_dir/sourceslist.probe"
+    command="'echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list; echo \$? > $probe'"
+    run_termux_command "$command" "$probe"
+
+    probe="$dev_probe_dir/adb_install.probe"
+    command="'mkdir -vp ~/.cargo/bin; apt update; yes | apt install $install_package_list -y; echo \$? > $probe'"
+    run_termux_command "$command" "$probe"
+}
+
+install_packages_via_ssh_using_apt() {
+    install_package_list="$*"
+
+    repo_url=$(get_current_repo_url)
+    move_to_next_repo_url
+    echo "set apt repository url: $repo_url"
+    run_command_via_ssh "echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list"
+
+    run_command_via_ssh "apt update; yes | apt install $install_package_list -y"
+}
+
+apt_upgrade_all_packages() {
+    repo_url=$(get_current_repo_url)
+    move_to_next_repo_url
+    echo "set apt repository url: $repo_url"
+    run_command_via_ssh "echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list"
+
+    run_command_via_ssh "apt update; yes | apt upgrade -y"
+}
+
+generate_and_install_public_key() {
+    echo "generate local public private key pair"
+    generate_rsa_key_local
+    echo "install public key via 'adb shell input'"
+    install_rsa_pub
+    echo "installed ssh public key on device"
+}
+
 snapshot() {
     apk="$1"
     echo "Running snapshot"
     adb install -g "$apk"
 
     echo "Prepare and install system packages"
-    probe='/sdcard/pkg.probe'
+
-    # as of https://github.com/termux/termux-tools/blob/5b30fbf3b0306c9f3dcd67b68755d990e83f1263/packages/termux-tools/pkg there is one
+    reinit_ssh_connection || return 1
-    # broken mirror, which is not properly detected. thus skipping mirror detection altogether
+
-    command="'mkdir -vp ~/.cargo/bin; export TERMUX_PKG_NO_MIRROR_SELECT=y; yes | pkg install rust binutils openssl tar -y; echo \$? > $probe'"
+    apt_upgrade_all_packages
-    run_termux_command "$command" "$probe" || return
+
+    install_packages_via_ssh_using_apt "rust binutils openssl tar"
+
+    echo "Read /proc/cpuinfo"
+    run_command_via_ssh "cat /proc/cpuinfo"
 
     echo "Installing cargo-nextest"
-    probe='/sdcard/nextest.probe'
     # We need to install nextest via cargo currently, since there is no pre-built binary for android x86
-    command="'\
+    run_command_via_ssh "export CARGO_TERM_COLOR=always && cargo install cargo-nextest"
-export CARGO_TERM_COLOR=always; \
-cargo install cargo-nextest; \
-echo \$? > $probe'"
-    run_termux_command "$command" "$probe"
     return_code=$?
 
-    echo "Info about cargo and rust"
+    echo "Info about cargo and rust - via SSH Script"
-    probe='/sdcard/info.probe'
+    run_script_file_via_ssh "$this_repo/util/android-scripts/collect-info.sh"
-    command="'echo \$HOME; \
-PATH=\$HOME/.cargo/bin:\$PATH; \
-export PATH; \
-echo \$PATH; \
-pwd; \
-command -v rustc && rustc -Vv; \
-ls -la ~/.cargo/bin; \
-cargo --list; \
-cargo nextest --version; \
-touch $probe'"
-    run_termux_command "$command" "$probe"
 
     echo "Snapshot complete"
     # shellcheck disable=SC2086
@@ -239,50 +514,16 @@ touch $probe'"
 sync_host() {
     repo="$1"
     cache_home="${HOME}/${cache_dir_name}"
-    cache_dest="/sdcard/${cache_dir_name}"
+    cache_dest="$dev_home_dir/${cache_dir_name}"
+
+    reinit_ssh_connection
 
     echo "Running sync host -> image: ${repo}"
 
-    # android doesn't allow symlinks on shared dirs, and adb can't selectively push files
+    # run_command_via_ssh "mkdir $dev_home_dir/coreutils"
-    symlinks=$(find "$repo" -type l)
-    # dash doesn't support process substitution :(
-    echo "$symlinks" | sort >symlinks
 
-    git -C "$repo" diff --name-status | cut -f 2 >modified
+    copy_file_or_dir_to_device_via_ssh "$repo" "$dev_home_dir"
-    modified_links=$(join symlinks modified)
+    [[ -e "$cache_home" ]] && copy_file_or_dir_to_device_via_ssh "$cache_home" "$cache_dest"
-    if [ -n "$modified_links" ]; then
-        echo "You have modified symlinks. Either stash or commit them, then try again: $modified_links"
-        exit 1
-    fi
-    #shellcheck disable=SC2086
-    if ! git ls-files --error-unmatch $symlinks >/dev/null; then
-        echo "You have untracked symlinks. Either remove or commit them, then try again."
-        exit 1
-    fi
-
-    #shellcheck disable=SC2086
-    rm $symlinks
-    # adb's shell user only has access to shared dirs...
-    adb push -a "$repo" /sdcard/coreutils
-    [[ -e "$cache_home" ]] && adb push -a "$cache_home" "$cache_dest"
-
-    #shellcheck disable=SC2086
-    git -C "$repo" checkout $symlinks
-
-    # ...but shared dirs can't build, so move it home as termux
-    probe='/sdcard/sync.probe'
-    command="'mv /sdcard/coreutils ~/; \
-cd ~/coreutils; \
-if [[ -e ${cache_dest} ]]; then \
-rm -rf ~/.cargo ./target; \
-tar xzf ${cache_dest}/cargo.tgz -C ~/; \
-ls -la ~/.cargo; \
-tar xzf ${cache_dest}/target.tgz; \
-ls -la ./target; \
-rm -rf ${cache_dest}; \
-fi; \
-touch $probe'"
-    run_termux_command "$command" "$probe" || return
 
     echo "Finished sync host -> image: ${repo}"
 }
@@ -290,22 +531,22 @@ touch $probe'"
 sync_image() {
     repo="$1"
     cache_home="${HOME}/${cache_dir_name}"
-    cache_dest="/sdcard/${cache_dir_name}"
+    cache_dest="$dev_probe_dir/${cache_dir_name}"
+
+    reinit_ssh_connection
 
     echo "Running sync image -> host: ${repo}"
 
-    probe='/sdcard/cache.probe'
+    command="rm -rf $dev_probe_dir/coreutils ${cache_dest}; \
-    command="'rm -rf /sdcard/coreutils ${cache_dest}; \
 mkdir -p ${cache_dest}; \
 cd ${cache_dest}; \
 tar czf cargo.tgz -C ~/ .cargo; \
 tar czf target.tgz -C ~/coreutils target; \
-ls -la ${cache_dest}; \
+ls -la ${cache_dest}"
-echo \$? > ${probe}'"
+    run_command_via_ssh "$command" || return
-    run_termux_command "$command" "$probe" || return
 
     rm -rf "$cache_home"
-    adb pull -a "$cache_dest" "$cache_home" || return
+    copy_file_or_dir_from_device_via_ssh "$cache_dest" "$cache_home" || return
 
     echo "Finished sync image -> host: ${repo}"
 }
@@ -313,12 +554,15 @@ echo \$? > ${probe}'"
 build() {
     echo "Running build"
 
-    probe='/sdcard/build.probe'
+    reinit_ssh_connection
-    command="'export CARGO_TERM_COLOR=always; \
+
-export CARGO_INCREMENTAL=0; \
+    echo "Read /proc/cpuinfo"
-cd ~/coreutils && cargo build --features feat_os_unix_android; \
+    run_command_via_ssh "cat /proc/cpuinfo"
-echo \$? >$probe'"
+
-    run_termux_command "$command" "$probe" || return
+    command="export CARGO_TERM_COLOR=always;
+             export CARGO_INCREMENTAL=0; \
+             cd ~/coreutils && cargo build --features feat_os_unix_android"
+    run_command_via_ssh "$command" || return
 
     echo "Finished build"
 }
@@ -326,31 +570,25 @@ echo \$? >$probe'"
 tests() {
     echo "Running tests"
 
-    probe='/sdcard/tests.probe'
+    reinit_ssh_connection
-    command="'export PATH=\$HOME/.cargo/bin:\$PATH; \
+
-export RUST_BACKTRACE=1; \
+    echo "Read /proc/cpuinfo"
-export CARGO_TERM_COLOR=always; \
+    run_command_via_ssh "cat /proc/cpuinfo"
-export CARGO_INCREMENTAL=0; \
+
-cd ~/coreutils; \
+    run_script_file_via_ssh "$this_repo/util/android-scripts/run-tests.sh" || return
-timeout --preserve-status --verbose -k 1m 60m \
-cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android; \
-echo \$? >$probe'"
-    run_termux_command "$command" "$probe" || return
 
     echo "Finished tests"
 }
 
 hash_rustc() {
-    probe='/sdcard/rustc.probe'
     tmp_hash="__rustc_hash__.tmp"
     hash="__rustc_hash__"
 
+    reinit_ssh_connection
+
     echo "Hashing rustc version: ${HOME}/${hash}"
 
-    command="'rustc -Vv; echo \$? > ${probe}'"
+    run_command_via_ssh "rustc -Vv" > rustc.log || return
-    keep_log=1
-    debug=0
-    run_termux_command "$command" "$probe" || return
     rm -f "$tmp_hash"
     mv "rustc.log" "$tmp_hash" || return
     # sha256sum is not available. shasum is the macos native program.

util/android-scripts/collect-info.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# spell-checker:ignore nextest watchplus PIPESTATUS
+
+echo "$HOME"
+PATH=$HOME/.cargo/bin:$PATH
+export PATH
+echo "$PATH"
+pwd
+command -v rustc && rustc -Vv
+ls -la ~/.cargo/bin
+cargo --list
+cargo nextest --version

util/android-scripts/run-tests.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# spell-checker:ignore nextest watchplus PIPESTATUS
+
+echo "PATH: $PATH"
+
+export PATH=$HOME/.cargo/bin:$PATH
+export RUST_BACKTRACE=1
+export CARGO_TERM_COLOR=always
+export CARGO_INCREMENTAL=0
+
+echo "PATH: $PATH"
+
+run_tests_in_subprocess() (
+
+    # limit virtual memory to 3GB to avoid that OS kills sshd
+    ulimit -v $((1024 * 1024 * 3))
+
+    watchplus() {
+        # call: watchplus <interval> <command>
+        while true; do
+            "${@:2}"
+            sleep "$1"
+        done
+    }
+
+    kill_all_background_jobs() {
+        jobs -p | xargs -I{} kill -- {}
+    }
+
+    # observe (log) every 2 seconds the system resource usage to judge if we are at a limit
+    watchplus 2 df -h &
+    watchplus 2 free -hm &
+
+    # run tests
+    cd ~/coreutils && \
+        timeout --preserve-status --verbose -k 1m 60m \
+            cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android
+
+    result=$?
+
+    kill_all_background_jobs
+
+    return $result
+)
+
+# run sub-shell to be able to use ulimit without affecting the sshd
+run_tests_in_subprocess