From 611bbc43beed6cce002c5a8bc7651ec261c636be Mon Sep 17 00:00:00 2001 From: Andrew Kaster Date: Wed, 3 Feb 2021 19:04:22 -0700 Subject: [PATCH] Base/CI: Boot serenity in CI in a mode that runs tests on target Build a new version of Serenity in CI that doesn't have all the debug symbols on, or we'd be waiting a very long time to boot. Insert a TestRunner entry into SystemServer.ini that will run a shell script that runs tests in /bin and /usr/Tests and shuts down the system in the new self-test boot mode. Also make sure enough basic services are started in self-test such that the tests will actually run properly. --- .github/workflows/cmake.yml | 171 ++++++++++++------ Base/etc/SystemServer.ini | 14 +- .../home/anon/tests/run-tests-and-shutdown.sh | 30 +++ Meta/lint-shell-scripts.sh | 3 +- 4 files changed, 155 insertions(+), 63 deletions(-) create mode 100755 Base/home/anon/tests/run-tests-and-shutdown.sh diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index ef01991cc3..f2d49b3f50 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -9,20 +9,13 @@ env: SERENITY_ROOT: ${{ github.workspace }} jobs: - build_and_test: + build_and_test_serenity: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - os: ubuntu-20.04 - allow-test-failure: false - # There sure is a lot of logic here, is there a better way? - # TODO: Make IRC notifications its own job/workflow? - should-notify-irc: ${{ github.repository == 'SerenityOS/serenity' && (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master')) }} - - os: macos-10.15 - allow-test-failure: true - should-notify-irc: false + debug-macros: ['ALL_DEBUG', 'NORMAL_DEBUG'] + os: [ubuntu-20.04] steps: - uses: actions/checkout@v2 @@ -37,7 +30,6 @@ jobs: - name: Purge interfering packages # Remove GCC 9 and clang-format 10 (installed by default) run: sudo apt-get purge -y gcc-9 g++-9 libstdc++-9-dev clang-format-10 - if: ${{ runner.os == 'Linux' }} - name: "Install Ubuntu dependencies" # These packages are already part of the ubuntu-20.04 image: # cmake gcc-10 g++-10 shellcheck libgmp-dev @@ -46,17 +38,9 @@ jobs: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main" sudo apt-get update - sudo apt-get install clang-format-11 libstdc++-10-dev libmpfr-dev libmpc-dev ninja-build npm - # If we ever do any qemu-emulation on Github Actions, we should re-enable this: - # e2fsprogs qemu-system-i386 qemu-utils - if: ${{ runner.os == 'Linux' }} + sudo apt-get install clang-format-11 libstdc++-10-dev libmpfr-dev libmpc-dev ninja-build npm e2fsprogs qemu-utils qemu-system-i386 - name: Use GCC 10 instead run: sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 60 --slave /usr/bin/g++ g++ /usr/bin/g++-10 - if: ${{ runner.os == 'Linux' }} - - - name: Install macOS dependencies - run: brew install coreutils ninja - if: ${{ runner.os == 'macOS' }} - name: Install JS dependencies run: sudo npm install -g prettier @@ -72,7 +56,6 @@ jobs: - name: Lint (Phase 1/2) run: ${{ github.workspace }}/Meta/lint-ci.sh - if: ${{ runner.os == 'Linux' }} - name: Toolchain cache uses: actions/cache@v2 with: @@ -86,6 +69,15 @@ jobs: # TODO: ccache # https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ # https://github.com/cristianadam/HelloWorld/blob/master/.github/workflows/build_cmake.yml + - name: Create build environment with debug macros + working-directory: ${{ github.workspace }} + # Build the entire project with debug macros turned on, to prevent code rot. + # However, it is unweildy and slow to run tests with them enabled, so we will build twice. + run: | + mkdir -p Build + cd Build + cmake .. -GNinja -DBUILD_LAGOM=ON -DENABLE_ALL_THE_DEBUG_MACROS=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 + if: ${{ matrix.debug-macros == 'ALL_DEBUG' }} - name: Create build environment working-directory: ${{ github.workspace }} # Note that this needs to run *even if* the Toolchain was built, @@ -93,9 +85,10 @@ jobs: run: | mkdir -p Build cd Build - cmake .. -GNinja -DBUILD_LAGOM=ON -DENABLE_ALL_THE_DEBUG_MACROS=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 + cmake .. -GNinja -DBUILD_LAGOM=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 + if: ${{ matrix.debug-macros == 'NORMAL_DEBUG' }} - # === ACTUALLY BUILD AND TEST === + # === ACTUALLY BUILD === - name: Build Serenity and Tests working-directory: ${{ github.workspace }}/Build @@ -103,16 +96,106 @@ jobs: - name: Lint (Phase 2/2) working-directory: ${{ github.workspace }}/Meta run: ./check-symbols.sh - - name: Run CMake tests + + - name: Create Serenity Rootfs + if: ${{ matrix.debug-macros == 'NORMAL_DEBUG'}} working-directory: ${{ github.workspace }}/Build + run: ninja install && ninja image + + - name: Run On-Target Tests + if: ${{ matrix.debug-macros == 'NORMAL_DEBUG'}} + working-directory: ${{ github.workspace }}/Build + env: + SERENITY_KERNEL_CMDLINE: "boot_mode=self-test" + SERENITY_RUN: "ci" + run: ninja run + timeout-minutes: 10 + - name: Print target logs + if: ${{ !cancelled() && matrix.debug-macros == 'NORMAL_DEBUG'}} + working-directory: ${{ github.workspace }}/Build + run: cat ./debug.log + + build_and_test_lagom: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - with-fuzzers: FUZZ + os: ubuntu-20.04 + allow-test-failure: false + - with-fuzzers: NO_FUZZ + os: ubuntu-20.04 + allow-test-failure: false + - with-fuzzers: NO_FUZZ + os: macos-10.15 + allow-test-failure: true + + steps: + - uses: actions/checkout@v2 + + # === OS SETUP === + # + - name: Install Ubuntu dependencies + run: sudo apt-get install ninja-build + if: ${{ runner.os == 'Linux' }} + - name: Install macOS dependencies + run: brew install ninja + if: ${{ runner.os == 'macOS' }} + - name: Check versions + run: set +e; clang --version; clang++ --version; ninja --version + + # === PREPARE FOR BUILDING === + + # TODO: ccache + # https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ + # https://github.com/cristianadam/HelloWorld/blob/master/.github/workflows/build_cmake.yml + - name: Create build environment (fuzz) + working-directory: ${{ github.workspace }}/Meta/Lagom + run: | + mkdir -p Build + cd Build + cmake -GNinja -DBUILD_LAGOM=ON -DENABLE_FUZZER_SANITIZER=ON -DENABLE_ADDRESS_SANITIZER=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ .. + if: ${{ matrix.with-fuzzers == 'FUZZ' }} + + - name: Create build environment (no fuzz) + working-directory: ${{ github.workspace }}/Meta/Lagom + run: | + mkdir -p Build + cd Build + cmake -GNinja -DBUILD_LAGOM=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 .. + if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }} + + # === ACTUALLY BUILD AND TEST === + + - name: Build Lagom + working-directory: ${{ github.workspace }}/Meta/Lagom/Build + run: cmake --build . + + - name: Run CMake tests + working-directory: ${{ github.workspace }}/Meta/Lagom/Build run: CTEST_OUTPUT_ON_FAILURE=1 ninja test || ${{ matrix.allow-test-failure }} timeout-minutes: 2 + if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }} - name: Run JS tests - working-directory: ${{ github.workspace }}/Build/Meta/Lagom + working-directory: ${{ github.workspace }}/Meta/Lagom/Build + if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }} run: DISABLE_DBG_OUTPUT=1 ./test-js || ${{ matrix.allow-test-failure }} - name: Run LibCompress tests - working-directory: ${{ github.workspace }}/Build/Meta/Lagom + working-directory: ${{ github.workspace }}/Meta/Lagom/Build run: ./test-compress + if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }} + + notify_irc: + needs: [build_and_test_serenity, build_and_test_lagom] + runs-on: ubuntu-20.04 + if: always() && github.repository == 'SerenityOS/serenity' && (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master')) + + steps: + - uses: actions/checkout@v2 + # Sets environment variable env.WORKFLOW_CONCLUSION to one of [neutral, success, skipped, cancelled, timed_out, action_required, failure] + # depending on result of all needs jobs + - uses: technote-space/workflow-conclusion-action@v2 # === NOTIFICATIONS === @@ -122,43 +205,13 @@ jobs: run: | cat <<"EOF" ${{ toJSON(github.event) }} + ${{ toJSON(needs) }} EOF - name: Generate IRC message - if: matrix.should-notify-irc == true && !cancelled() + if: always() run: | ${{ github.workspace }}/Meta/notify_irc.py <<"EOF" - ["${{ github.actor }}", ${{ github.run_id }}, "${{ job.status }}", + ["${{ github.actor }}", ${{ github.run_id }}, "${{ env.WORKFLOW_CONCLUSION }}", ${{ toJSON(github.event) }} ] EOF - - build_lagom_with_fuzzers: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - - # === OS SETUP === - # - - name: Install dependencies - run: sudo apt-get install ninja-build - - name: Check versions - run: set +e; clang --version; clang++ --version; ninja --version - - # === PREPARE FOR BUILDING === - - # TODO: ccache - # https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ - # https://github.com/cristianadam/HelloWorld/blob/master/.github/workflows/build_cmake.yml - - name: Create build environment - working-directory: ${{ github.workspace }}/Meta/Lagom - run: | - mkdir -p Build - cd Build - cmake -GNinja -DBUILD_LAGOM=ON -DENABLE_FUZZER_SANITIZER=ON -DENABLE_ADDRESS_SANITIZER=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ .. - - # === ACTUALLY BUILD === - - - name: Build Lagom with Fuzzers - working-directory: ${{ github.workspace }}/Meta/Lagom/Build - run: cmake --build . diff --git a/Base/etc/SystemServer.ini b/Base/etc/SystemServer.ini index b02567fb9c..8cfc622d5f 100644 --- a/Base/etc/SystemServer.ini +++ b/Base/etc/SystemServer.ini @@ -4,7 +4,7 @@ SocketPermissions=660 Lazy=1 Priority=low User=protocol -BootModes=text,graphical +BootModes=text,graphical,self-test MultiInstance=1 AcceptSocketConnections=1 @@ -39,13 +39,13 @@ Lazy=1 Priority=low KeepAlive=1 User=lookup -BootModes=text,graphical +BootModes=text,graphical,self-test [DHCPClient] Priority=low KeepAlive=1 User=root -BootModes=text,graphical +BootModes=text,graphical,self-test [NotificationServer] Socket=/tmp/portal/notify @@ -175,3 +175,11 @@ AcceptSocketConnections=1 Executable=/bin/CrashDaemon KeepAlive=1 User=anon + +[TestRunner@ttyS0] +Executable=/home/anon/tests/run-tests-and-shutdown.sh +StdIO=/dev/ttyS0 +Environment=DO_SHUTDOWN_AFTER_TESTS=1 TERM=xterm PATH=/bin:/usr/bin:/usr/local/bin +User=anon +WorkingDirectory=/home/anon +BootModes=self-test diff --git a/Base/home/anon/tests/run-tests-and-shutdown.sh b/Base/home/anon/tests/run-tests-and-shutdown.sh new file mode 100755 index 0000000000..4bf46c17fc --- /dev/null +++ b/Base/home/anon/tests/run-tests-and-shutdown.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +echo "==== Running Tests on SerenityOS ====" + +run() { + test_cmd=($*) + echo "Running test -- $test_cmd" + if $test_cmd { + echo "::debug file=$test_cmd:: $test_cmd passed!" + } else { + echo "::error file=$test_cmd:: $test_cmd returned non-zero exit code, check logs!" + } +} + +# TODO: It'd be nice to have a list+list op (as opposed to nest-on-in-another) +# TODO: It'd be nice to have a list.length or enumerate(list) operation to allow terminal progress bar +# TODO: test-web requires the window server +system_tests=(test-js test-pthread test-compress (test-crypto bigint -t)) +# FIXME: Running too much at once is likely to run into #5541. Remove commented out find below when stable +all_tests=($system_tests) #$(find /usr/Tests -type f | grep -v Kernel | grep -v .inc | shuf)) + +for list in $all_tests { + for $list { run $it } +} + +echo "==== Done running tests ====" + +if test $DO_SHUTDOWN_AFTER_TESTS { + shutdown -n +} diff --git a/Meta/lint-shell-scripts.sh b/Meta/lint-shell-scripts.sh index aba9b06b2e..ce59f809b9 100755 --- a/Meta/lint-shell-scripts.sh +++ b/Meta/lint-shell-scripts.sh @@ -11,7 +11,8 @@ if [ "$#" -eq "0" ]; then '*.sh' \ ':!:Toolchain' \ ':!:Ports' \ - ':!:Userland/Shell/Tests' + ':!:Userland/Shell/Tests' \ + ':!:Base/home/anon/tests' ) else files=()