From 44bcd7c7aa9af41b57d44fe532b1047191c383de Mon Sep 17 00:00:00 2001
From: Andrew Kaster <akaster@serenityos.org>
Date: Sat, 19 Mar 2022 16:42:54 -0600
Subject: [PATCH] CI: Add x86_64 Clang Coverage pipeline in Azure

Add a job to the Azure pipelines to run tests with coverage enabled, and
aggregate the test results in a folder of html pages showing the
coverage results overall, and per-file.

Future work is needed to take the published pipeline artifact for the
coverage results and display them somewhere interesting.
---
 Meta/Azure/Caches.yml           |  5 +++--
 Meta/Azure/Serenity.yml         | 27 ++++++++++++++++++++++++++-
 Meta/Azure/nightly-pipeline.yml | 21 +++++++++++++++++++++
 3 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100644 Meta/Azure/nightly-pipeline.yml

diff --git a/Meta/Azure/Caches.yml b/Meta/Azure/Caches.yml
index 8bd5d9cbec..1865918af2 100644
--- a/Meta/Azure/Caches.yml
+++ b/Meta/Azure/Caches.yml
@@ -2,6 +2,7 @@ parameters:
   os: 'Linux'
   arch: 'i686'
   toolchain: 'gcc'
+  coverage: 'OFF'
   build_directory: ''
   ccache_version: 1 # Increment this number if CI has trouble with ccache.
   serenity_ccache_path: ''
@@ -45,9 +46,9 @@ steps:
   - ${{ if ne(parameters.serenity_ccache_path, '') }}:
     - task: Cache@2
       inputs:
-        key: '"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
+        key: '"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.coverage }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
         restoreKeys: |
-          "ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
+          "ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.coverage }}" | "${{ parameters.ccache_version }}"
         path: ${{ parameters.serenity_ccache_path }}
       displayName: 'Serenity Compiler Cache'
 
diff --git a/Meta/Azure/Serenity.yml b/Meta/Azure/Serenity.yml
index 5144ed72b3..748bdf5439 100644
--- a/Meta/Azure/Serenity.yml
+++ b/Meta/Azure/Serenity.yml
@@ -1,8 +1,9 @@
 parameters:
   arch: 'i686'
+  coverage: 'OFF'
 
 jobs:
-  - job: 'Serenity_Clang_${{ parameters.arch }}'
+  - job: 'Serenity_Clang_${{ parameters.arch }}_Coverage_${{ parameters.coverage }}'
     timeoutInMinutes: 0 # Setting to 0 means the maximum allowed timeout is used.
 
     variables:
@@ -24,6 +25,7 @@ jobs:
     - template: Caches.yml
       parameters:
         arch: '${{ parameters.arch }}'
+        coverage: '${{ parameters.coverage }}'
         toolchain: 'clang'
         build_directory: 'Build/${{ parameters.arch }}clang'
         toolchain_ccache_path: '$(LLVM_CCACHE_DIR)'
@@ -44,6 +46,7 @@ jobs:
           -DSERENITY_ARCH=${{ parameters.arch }} \
           -DSERENITY_TOOLCHAIN=Clang \
           -DENABLE_UNDEFINED_SANITIZER=ON \
+          -DENABLE_USERSPACE_COVERAGE_COLLECTION=${{ parameters.coverage }} \
           -DENABLE_PCI_IDS_DOWNLOAD=OFF \
           -DENABLE_USB_IDS_DOWNLOAD=OFF \
           -DCMAKE_C_COMPILER=gcc-11 \
@@ -81,6 +84,7 @@ jobs:
           echo "##[error]:^( Tests failed, failing job"
           exit 1
         fi
+        sudo umount fsmount
       displayName: 'Test'
       workingDirectory: $(Build.SourcesDirectory)/Build/${{ parameters.arch }}clang
       timeoutInMinutes: 60
@@ -95,6 +99,27 @@ jobs:
       workingDirectory: $(Build.SourcesDirectory)/Build/${{ parameters.arch }}clang
       condition: failed()
 
+    - ${{ if eq(parameters.coverage, 'ON') }}:
+      - script: |
+          ./Meta/analyze-qemu-coverage.sh
+        displayName: 'Aggregate Coverage Results'
+        workingDirectory: $(Build.SourcesDirectory)
+        env:
+          SERENITY_TOOLCHAIN: Clang
+          SERENITY_ARCH: ${{ parameters.arch }}
+
+      # FIXME: Deploy the static html pages somewhere
+      # FIXME: Alter script to also (instead?) produce a raw coverage.txt file for ingestion into sonar cloud
+      # Note: tmp_profile_data/Coverage.profdata has the entire combined profile data, but creating the raw txt requires
+      #       all of the instrumented binaries and the profdata file.
+    - ${{ if eq(parameters.coverage, 'ON') }}:
+      - task: PublishPipelineArtifact@1
+        inputs:
+          targetPath: $(Build.SourcesDirectory)/Build/${{ parameters.arch }}clang/reports
+          artifactType: 'pipeline'
+          artifactName: 'Coverage'
+
+
     - script: |
         echo "##[section]Toolchain Cache"
         CCACHE_DIR='$(LLVM_CCACHE_DIR)' ccache -s
diff --git a/Meta/Azure/nightly-pipeline.yml b/Meta/Azure/nightly-pipeline.yml
new file mode 100644
index 0000000000..a3c6bc4877
--- /dev/null
+++ b/Meta/Azure/nightly-pipeline.yml
@@ -0,0 +1,21 @@
+# Pipeline definition for nightly or other scheduled jobs we don't want to run on every PR in Azure
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily midnight build
+  branches:
+    include:
+    - master
+
+stages:
+  - stage: Toolchain
+    dependsOn: []
+    jobs:
+      - template: Meta/Azure/Toolchain.yml
+
+  - stage: SerenityOS (Coverage)
+    dependsOn: Toolchain
+    jobs:
+      - template: Meta/Azure/Serenity.yml
+        parameters:
+          arch: 'x86_64'
+          coverage: 'ON'