RGBCube/uutils-coreutils
1
Fork 0
mirror of https://github.com/RGBCube/uutils-coreutils synced 2025-07-27 11:07:44 +00:00
Code Issues Activity

squash commit

Browse source
This commit is contained in:
Ulrich Hornung 2024-02-03 17:08:39 +01:00
parent b485a480cf
commit 93d922f075
No known key found for this signature in database
GPG key ID: 64EA3BAAF1BC0603
5 changed files with 483 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

470
util/android-commands.sh
View file

@ -1,22 +1,72 @@
#!/usr/bin/env bash
# spell-checker:ignore termux keyevent sdcard binutils unmatch adb's dumpsys logcat pkill nextest logfile
# spell-checker:ignore screencap reinit PIPESTATUS keygen sourceslist
# There are three shells: the host's, adb, and termux. Only adb lets us run
# commands directly on the emulated device, only termux provides a GNU
# environment on the emulated device (to e.g. run cargo). So we use adb to
# launch termux, then to send keystrokes to it while it's running.
# This means that the commands sent to termux are first parsed as arguments in
# There are four shells: the host's, adb, termux and termux via ssh.
# But only termux and termux via ssh provides a GNU environment on the
# emulated device (to e.g. run cargo).
# Initially, only adb lets us run commands directly on the emulated device.
# Thus we first establish a ssh connection which then can be used to access
# the termux shell directly, getting output and return code as usual.
# So we use adb to launch termux, then to send keystrokes to it while it's running.
# This way we install sshd and a public key from the host. After that we can
# use ssh to directly run commands in termux environment.
# Before ssh, we need to consider some inconvenient, limiting specialties:
# The commands sent to termux via adb keystrokes are first parsed as arguments in
# this shell, then as arguments in the adb shell, before finally being used as
# text inputs to the app. Hence, the "'wrapping'" on those commands.
# There's no way to get any direct feedback from termux, so every time we run a
# command on it, we make sure it creates a unique *.probe file which is polled
# every 30 seconds together with the current output of the command in a *.log file.
# The contents of the probe file are used as a return code: 0 on success, some
# other number for errors (an empty file is basically the same as 0). Note that
# the return codes are text, not raw bytes.
# Using this approach there's no way to get any direct feedback from termux,
# so every time we run a command on it, we make sure it creates a unique *.probe file
# which is polled every 30 seconds together with the current output of the
# command in a *.log file. The contents of the probe file are used as a return code:
# 0 on success, some other number for errors (an empty file is basically the same as 0).
# Note that the return codes are text, not raw bytes.
# Additionally, we can use adb screenshot functionality to investigate issues
# when there is no feedback arriving from the android device.
this_repo="$(dirname "$(dirname -- "$(readlink -- "${0}")")")"
cache_dir_name="__rust_cache__"
dev_probe_dir=/sdcard
dev_home_dir=/data/data/com.termux/files/home
# This is a list of termux package mirrors approved to be used.
# The default mirror list contains entries that do not function properly anymore.
# To avoid failures due to broken mirrors, we use our own list.
# Choose only reliable mirrors here:
repo_url_list=(
"deb https://packages-cf.termux.org/apt/termux-main/ stable main"
"deb https://packages-cf.termux.dev/apt/termux-main/ stable main"
# "deb https://grimler.se/termux/termux-main stable main" # slow
"deb https://ftp.fau.de/termux/termux-main stable main"
)
number_repo_urls=${#repo_url_list[@]}
repo_url_round_robin=$RANDOM
move_to_next_repo_url() {
repo_url_round_robin=$(((repo_url_round_robin + 1) % number_repo_urls))
echo "next round robin repo_url: $repo_url_round_robin"
}
move_to_next_repo_url # first call needed for modulo
get_current_repo_url() {
echo "${repo_url_list[$repo_url_round_robin]}"
}
# dump some information about the runners system for debugging purposes:
echo "====== runner information ======"
echo "hostname: $(hostname)"
echo "uname -a: $(uname -a)"
echo "pwd: $(pwd)"
echo "\$*: $*"
echo "\$0: $0"
# shellcheck disable=SC2140
echo "\$(readlink -- "\$\{0\}"): $(readlink -- "${0}")"
echo "\$this_repo: $this_repo"
echo "readlink -f \$this_repo: $(readlink -f "$this_repo")"
this_repo=$(readlink -f "$this_repo")
echo "====== runner info end ========="
help() {
echo \
@ -48,23 +98,82 @@ hit_enter() {
}
exit_termux() {
adb shell input text "exit" && hit_enter && hit_enter
adb shell input text \"exit\" && hit_enter && hit_enter
}
timestamp() {
date +"%H%M%S%Z"
}
add_timestamp_to_lines() {
while IFS= read -r line; do printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$line"; done
}
# takes a screenshot with given name from the android device. Filename is prefixed with timestamp.
# screenshots are collected at the end of the github workflow and provided as download link.
take_screen_shot() {
filename_prefix="$1"
filename="$this_repo/output/$(timestamp)_${filename_prefix}_screen.png"
echo "take screenshot: $filename"
mkdir -p "$this_repo/output"
adb exec-out screencap -p > "$filename"
}
launch_termux() {
echo "launching termux"
take_screen_shot "launch_termux_enter"
adb shell input tap 120 380 # close potential dialog "System UI isn't responding" with "wait".
# should not cause side effects when dialog is not there as there are
# no relevant GUI elements at this position otherwise.
if ! adb shell 'am start -n com.termux/.HomeActivity'; then
echo "failed to launch termux"
exit 1
fi
take_screen_shot "launch_termux_after_start_activity"
# the emulator can sometimes be a little slow to launch the app
while ! adb shell 'ls /sdcard/launch.probe' 2>/dev/null; do
echo "waiting for launch.probe"
sleep 5
adb shell input text 'touch\ /sdcard/launch.probe' && hit_enter
loop_count=0
while ! adb shell "dumpsys window windows" | \
grep -E "imeInputTarget in display# 0 Window{[^}]+com.termux\/com\.termux\.HomeActivity}"
do
sleep 1
loop_count=$((loop_count + 1))
if [[ loop_count -ge 20 ]]; then
break
fi
done
take_screen_shot "launch_termux_after_wait_activity"
touch_cmd() {
adb shell input text "\"touch $dev_probe_dir/launch.probe\"" && hit_enter
sleep 1
}
local timeout_start=120
local timeout=$timeout_start
touch_cmd
while ! adb shell "ls $dev_probe_dir/launch.probe" 2>/dev/null
do
echo "waiting for launch.probe - ($timeout / $timeout_start seconds)"
take_screen_shot "launch_termux_touch_probe"
sleep 4
touch_cmd
timeout=$((timeout - 4))
if [[ timeout -le 0 ]]; then
take_screen_shot "error_launch_termux"
echo "timeout waiting for termux to start up"
return 1
fi
done
echo "found launch.probe"
adb shell 'rm /sdcard/launch.probe' && echo "removed launch.probe"
take_screen_shot "launch_termux_found_probe"
adb shell "rm $dev_probe_dir/launch.probe" && echo "removed launch.probe"
}
# Usage: run_termux_command
@ -99,7 +208,7 @@ run_termux_command() {
local debug=${debug:-1}
log_name="$(basename -s .probe "${probe}").log" # probe name must have suffix .probe
log_file="/sdcard/${log_name}"
log_file="$dev_probe_dir/${log_name}"
log_read="${log_name}.read"
echo 0 >"${log_read}"
if [[ $debug -eq 1 ]]; then
@ -108,12 +217,22 @@ run_termux_command() {
shell_command="'{ ${command}; } &> ${log_file}'"
fi
launch_termux
launch_termux || return
take_screen_shot "run_termux_command_before_input_of_shell_command"
# remove artificial quoting
shell_command="${shell_command%\'}"
shell_command="${shell_command#\'}"
echo "Running command: ${command}"
echo "Running shell-command: ${shell_command}"
start=$(date +%s)
adb shell input text "$shell_command" && sleep 3 && hit_enter
adb_input_text_long "$shell_command" && sleep 1 && hit_enter
# just for safety wait a little bit before polling for the probe and the log file
sleep 5
sleep 1
take_screen_shot "run_termux_command_after_input_of_shell_command"
local timeout=${timeout:-3600}
local retries=${retries:-10}
@ -136,6 +255,7 @@ run_termux_command() {
if [[ retries -le 0 ]]; then
echo "Maximum retries reached running command. Aborting ..."
take_screen_shot "run_termux_command_maximum_tries_reached"
return 1
elif [[ try_fix -le 0 ]]; then
retries=$((retries - 1))
@ -144,7 +264,10 @@ run_termux_command() {
# hitting the enter key solves the issue, sometimes the github runner is just a little
# bit slow.
echo "No output received. Trying to fix the issue ... (${retries} retries left)"
take_screen_shot "run_termux_command_before_trying_to_fix"
hit_enter
sleep 1
take_screen_shot "run_termux_command_after_trying_to_fix"
fi
sleep "$sleep_interval"
@ -152,6 +275,7 @@ run_termux_command() {
if [[ $timeout -le 0 ]]; then
echo "Timeout reached running command. Aborting ..."
take_screen_shot "run_termux_command_timeout_reached"
return 1
fi
done
@ -160,7 +284,7 @@ run_termux_command() {
return_code=$(adb shell "cat $probe") || return_code=0
adb shell "rm ${probe}"
adb pull "$log_file" .
adb shell "cat $log_file" > "$log_name"
echo "==================================== SUMMARY ==================================="
echo "Command: ${command}"
echo "Finished in $((end - start)) seconds."
@ -173,63 +297,214 @@ run_termux_command() {
[[ $keep_log -ne 1 ]] && rm -f "$log_name"
rm -f "$log_read" "$probe"
take_screen_shot "run_termux_command_finished_normally"
# shellcheck disable=SC2086
return $return_code
}
init() {
arch="$1"
# shellcheck disable=SC2034
api_level="$2"
termux="$3"
snapshot_name="${AVD_CACHE_KEY}"
# shellcheck disable=SC2015
wget -nv "https://github.com/termux/termux-app/releases/download/${termux}/termux-app_${termux}+github-debug_${arch}.apk" &&
snapshot "termux-app_${termux}+github-debug_${arch}.apk" &&
hash_rustc &&
exit_termux &&
adb -s emulator-5554 emu avd snapshot save "${api_level}-${arch}+termux-${termux}" &&
echo "Emulator image created." || {
adb -s emulator-5554 emu avd snapshot save "$snapshot_name" &&
echo "Emulator image created. Name: $snapshot_name" || {
pkill -9 qemu-system-x86_64
return 1
}
pkill -9 qemu-system-x86_64 || true
}
reinit_ssh_connection() {
setup_ssh_forwarding
test_ssh_connection && return
start_sshd_via_adb_shell && (
test_ssh_connection && return
generate_and_install_public_key && test_ssh_connection && return
) || (
install_packages_via_adb_shell openssh openssl
generate_and_install_public_key
start_sshd_via_adb_shell
test_ssh_connection && return
) || (
echo "failed to setup ssh connection"
return 1
)
}
start_sshd_via_adb_shell() {
echo "start sshd via adb shell"
probe="$dev_probe_dir/sshd.probe"
command="'sshd; echo \$? > $probe'"
run_termux_command "$command" "$probe"
}
setup_ssh_forwarding() {
echo "setup ssh forwarding"
adb forward tcp:9022 tcp:8022
}
copy_file_or_dir_to_device_via_ssh() {
scp -r "$1" "scp://termux@127.0.0.1:9022/$2"
}
copy_file_or_dir_from_device_via_ssh() {
scp -r "scp://termux@127.0.0.1:9022/$1" "$2"
}
# runs the in args provided command on android side via ssh. forwards return code.
# adds a timestamp to every line to be able to see where delays are
run_command_via_ssh() {
ssh -p 9022 termux:@127.0.0.1 -o StrictHostKeyChecking=accept-new "$@" 2>&1 | add_timestamp_to_lines
return "${PIPESTATUS[0]}"
}
test_ssh_connection() {
run_command_via_ssh echo ssh connection is working
run_command_via_ssh free -mh
}
# takes a local (on runner side) script file and runs it via ssh on the virtual android device. forwards return code.
# adds a timestamp to every line to be able to see where delays are
run_script_file_via_ssh() {
ssh -p 9022 termux:@127.0.0.1 -o StrictHostKeyChecking=accept-new "bash -s" < "$1" 2>&1 | add_timestamp_to_lines
return "${PIPESTATUS[0]}"
}
# Experiments showed that the adb shell input text functionality has a limitation for the input length.
# If input length is too big, the input is not fully provided to the android device.
# To avoid this, we divide large inputs into smaller chunks and put them one-by-one.
adb_input_text_long() {
string=$1
length=${#string}
step=20
p=0
for ((i = 0; i < length-step; i = i + step)); do
chunk="${string:i:$step}"
adb shell input text "'$chunk'"
p=$((i+step))
done
remaining="${string:p}"
adb shell input text "'$remaining'"
}
generate_rsa_key_local() {
yes "" | ssh-keygen -t rsa -b 4096 -C "Github Action" -N ""
}
install_rsa_pub() {
run_command_via_ssh "echo hello" && return # if this works, we are already fine. Skipping
# remove old host identity:
ssh-keygen -f ~/.ssh/known_hosts -R "[127.0.0.1]:9022"
rsa_pub_key=$(cat ~/.ssh/id_rsa.pub)
echo "====================================="
echo "$rsa_pub_key"
echo "====================================="
adb shell input text \"echo \"
adb_input_text_long "$rsa_pub_key"
adb shell input text "\" >> ~/.ssh/authorized_keys\"" && hit_enter
sleep 1
}
install_packages_via_adb_shell() {
install_package_list="$*"
install_packages_via_adb_shell_using_apt "$install_package_list"
if [[ $? -ne 0 ]]; then
echo "apt failed. Now try install with pkg as fallback."
probe="$dev_probe_dir/pkg.probe"
command="'mkdir -vp ~/.cargo/bin; yes | pkg install $install_package_list -y; echo \$? > $probe'"
run_termux_command "$command" "$probe" || return 1
fi
return 0
}
# We use apt to install the packages as pkg doesn't respect any pre-defined mirror list.
# Its important to have a defined mirror list to avoid issues with broken mirrors.
install_packages_via_adb_shell_using_apt() {
install_package_list="$*"
repo_url=$(get_current_repo_url)
move_to_next_repo_url
echo "set apt repository url: $repo_url"
probe="$dev_probe_dir/sourceslist.probe"
command="'echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list; echo \$? > $probe'"
run_termux_command "$command" "$probe"
probe="$dev_probe_dir/adb_install.probe"
command="'mkdir -vp ~/.cargo/bin; apt update; yes | apt install $install_package_list -y; echo \$? > $probe'"
run_termux_command "$command" "$probe"
}
install_packages_via_ssh_using_apt() {
install_package_list="$*"
repo_url=$(get_current_repo_url)
move_to_next_repo_url
echo "set apt repository url: $repo_url"
run_command_via_ssh "echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list"
run_command_via_ssh "apt update; yes | apt install $install_package_list -y"
}
apt_upgrade_all_packages() {
repo_url=$(get_current_repo_url)
move_to_next_repo_url
echo "set apt repository url: $repo_url"
run_command_via_ssh "echo $repo_url | dd of=\$PREFIX/etc/apt/sources.list"
run_command_via_ssh "apt update; yes | apt upgrade -y"
}
generate_and_install_public_key() {
echo "generate local public private key pair"
generate_rsa_key_local
echo "install public key via 'adb shell input'"
install_rsa_pub
echo "installed ssh public key on device"
}
snapshot() {
apk="$1"
echo "Running snapshot"
adb install -g "$apk"
echo "Prepare and install system packages"
probe='/sdcard/pkg.probe'
# as of https://github.com/termux/termux-tools/blob/5b30fbf3b0306c9f3dcd67b68755d990e83f1263/packages/termux-tools/pkg there is one
# broken mirror, which is not properly detected. thus skipping mirror detection altogether
command="'mkdir -vp ~/.cargo/bin; export TERMUX_PKG_NO_MIRROR_SELECT=y; yes | pkg install rust binutils openssl tar -y; echo \$? > $probe'"
run_termux_command "$command" "$probe" || return
reinit_ssh_connection || return 1
apt_upgrade_all_packages
install_packages_via_ssh_using_apt "rust binutils openssl tar"
echo "Read /proc/cpuinfo"
run_command_via_ssh "cat /proc/cpuinfo"
echo "Installing cargo-nextest"
probe='/sdcard/nextest.probe'
# We need to install nextest via cargo currently, since there is no pre-built binary for android x86
command="'\
export CARGO_TERM_COLOR=always; \
cargo install cargo-nextest; \
echo \$? > $probe'"
run_termux_command "$command" "$probe"
run_command_via_ssh "export CARGO_TERM_COLOR=always && cargo install cargo-nextest"
return_code=$?
echo "Info about cargo and rust"
probe='/sdcard/info.probe'
command="'echo \$HOME; \
PATH=\$HOME/.cargo/bin:\$PATH; \
export PATH; \
echo \$PATH; \
pwd; \
command -v rustc && rustc -Vv; \
ls -la ~/.cargo/bin; \
cargo --list; \
cargo nextest --version; \
touch $probe'"
run_termux_command "$command" "$probe"
echo "Info about cargo and rust - via SSH Script"
run_script_file_via_ssh "$this_repo/util/android-scripts/collect-info.sh"
echo "Snapshot complete"
# shellcheck disable=SC2086
@ -239,50 +514,16 @@ touch $probe'"
sync_host() {
repo="$1"
cache_home="${HOME}/${cache_dir_name}"
cache_dest="/sdcard/${cache_dir_name}"
cache_dest="$dev_home_dir/${cache_dir_name}"
reinit_ssh_connection
echo "Running sync host -> image: ${repo}"
# android doesn't allow symlinks on shared dirs, and adb can't selectively push files
symlinks=$(find "$repo" -type l)
# dash doesn't support process substitution :(
echo "$symlinks" | sort >symlinks
# run_command_via_ssh "mkdir $dev_home_dir/coreutils"
git -C "$repo" diff --name-status | cut -f 2 >modified
modified_links=$(join symlinks modified)
if [ -n "$modified_links" ]; then
echo "You have modified symlinks. Either stash or commit them, then try again: $modified_links"
exit 1
fi
#shellcheck disable=SC2086
if ! git ls-files --error-unmatch $symlinks >/dev/null; then
echo "You have untracked symlinks. Either remove or commit them, then try again."
exit 1
fi
#shellcheck disable=SC2086
rm $symlinks
# adb's shell user only has access to shared dirs...
adb push -a "$repo" /sdcard/coreutils
[[ -e "$cache_home" ]] && adb push -a "$cache_home" "$cache_dest"
#shellcheck disable=SC2086
git -C "$repo" checkout $symlinks
# ...but shared dirs can't build, so move it home as termux
probe='/sdcard/sync.probe'
command="'mv /sdcard/coreutils ~/; \
cd ~/coreutils; \
if [[ -e ${cache_dest} ]]; then \
rm -rf ~/.cargo ./target; \
tar xzf ${cache_dest}/cargo.tgz -C ~/; \
ls -la ~/.cargo; \
tar xzf ${cache_dest}/target.tgz; \
ls -la ./target; \
rm -rf ${cache_dest}; \
fi; \
touch $probe'"
run_termux_command "$command" "$probe" || return
copy_file_or_dir_to_device_via_ssh "$repo" "$dev_home_dir"
[[ -e "$cache_home" ]] && copy_file_or_dir_to_device_via_ssh "$cache_home" "$cache_dest"
echo "Finished sync host -> image: ${repo}"
}
@ -290,22 +531,22 @@ touch $probe'"
sync_image() {
repo="$1"
cache_home="${HOME}/${cache_dir_name}"
cache_dest="/sdcard/${cache_dir_name}"
cache_dest="$dev_probe_dir/${cache_dir_name}"
reinit_ssh_connection
echo "Running sync image -> host: ${repo}"
probe='/sdcard/cache.probe'
command="'rm -rf /sdcard/coreutils ${cache_dest}; \
command="rm -rf $dev_probe_dir/coreutils ${cache_dest}; \
mkdir -p ${cache_dest}; \
cd ${cache_dest}; \
tar czf cargo.tgz -C ~/ .cargo; \
tar czf target.tgz -C ~/coreutils target; \
ls -la ${cache_dest}; \
echo \$? > ${probe}'"
run_termux_command "$command" "$probe" || return
ls -la ${cache_dest}"
run_command_via_ssh "$command" || return
rm -rf "$cache_home"
adb pull -a "$cache_dest" "$cache_home" || return
copy_file_or_dir_from_device_via_ssh "$cache_dest" "$cache_home" || return
echo "Finished sync image -> host: ${repo}"
}
@ -313,12 +554,15 @@ echo \$? > ${probe}'"
build() {
echo "Running build"
probe='/sdcard/build.probe'
command="'export CARGO_TERM_COLOR=always; \
export CARGO_INCREMENTAL=0; \
cd ~/coreutils && cargo build --features feat_os_unix_android; \
echo \$? >$probe'"
run_termux_command "$command" "$probe" || return
reinit_ssh_connection
echo "Read /proc/cpuinfo"
run_command_via_ssh "cat /proc/cpuinfo"
command="export CARGO_TERM_COLOR=always;
export CARGO_INCREMENTAL=0; \
cd ~/coreutils && cargo build --features feat_os_unix_android"
run_command_via_ssh "$command" || return
echo "Finished build"
}
@ -326,31 +570,25 @@ echo \$? >$probe'"
tests() {
echo "Running tests"
probe='/sdcard/tests.probe'
command="'export PATH=\$HOME/.cargo/bin:\$PATH; \
export RUST_BACKTRACE=1; \
export CARGO_TERM_COLOR=always; \
export CARGO_INCREMENTAL=0; \
cd ~/coreutils; \
timeout --preserve-status --verbose -k 1m 60m \
cargo nextest run --profile ci --hide-progress-bar --features feat_os_unix_android; \
echo \$? >$probe'"
run_termux_command "$command" "$probe" || return
reinit_ssh_connection
echo "Read /proc/cpuinfo"
run_command_via_ssh "cat /proc/cpuinfo"
run_script_file_via_ssh "$this_repo/util/android-scripts/run-tests.sh" || return
echo "Finished tests"
}
hash_rustc() {
probe='/sdcard/rustc.probe'
tmp_hash="__rustc_hash__.tmp"
hash="__rustc_hash__"
reinit_ssh_connection
echo "Hashing rustc version: ${HOME}/${hash}"
command="'rustc -Vv; echo \$? > ${probe}'"
keep_log=1
debug=0
run_termux_command "$command" "$probe" || return
run_command_via_ssh "rustc -Vv" > rustc.log || return
rm -f "$tmp_hash"
mv "rustc.log" "$tmp_hash" || return
# sha256sum is not available. shasum is the macos native program.