akaros/tools/jenkins/launcher.sh
<<
>>
Prefs
   1#!/bin/bash
   2# This script should be called from Jenkins when a new commit has been pushed 
   3# to the repo. 
   4# It analyzes what parts of the codebase have been modified, compiles everything
   5# that is needed, and reports on the results. 
   6
   7#set -e
   8
   9readonly TMP_DIR=tmp
  10readonly DIFF_FILE=$TMP_DIR/changes.txt
  11readonly AKAROS_OUTPUT_FILE=$TMP_DIR/akaros_out.txt
  12readonly TEST_OUTPUT_DIR=output-tests
  13readonly TEST_DIR=tools/jenkins
  14readonly SCR_DIR=tools/jenkins/utils
  15readonly DOWNLOADS_DIR=dl
  16
  17# Config files
  18readonly CONF_DIR=tools/jenkins/config
  19readonly CONF_COMP_COMPONENTS_FILE=$CONF_DIR/compilation_components.json
  20
  21# Utility scripts
  22readonly SCR_WAIT_UNTIL=$SCR_DIR/wait_until.py
  23readonly SCR_GIT_CHANGES=$SCR_DIR/changes.py
  24readonly SCR_GEN_TEST_REPORTS=$SCR_DIR/test_reporter.py
  25
  26# Busybox settings
  27readonly BUSYBOX_VERSION=1.17.3
  28readonly BUSYBOX_DL_URL=http://www.busybox.net/downloads/busybox-1.17.3.tar.bz2
  29readonly BUSYBOX_CONF_FILE=tools/patches/busybox/busybox-1.17.3-config
  30
  31################################################################################
  32###############                 HELPER FUNCTIONS                 ###############
  33################################################################################
  34function last_stable_build() {
  35        curl -s localhost:8080/job/$JOB_NAME/lastStableBuild/api/json?tree=actions%5bbuildsByBranchName%5brevision%5bSHA1%5d%5d%5d | \
  36                python -c 'import sys, json; print json.load(sys.stdin)["actions"][2]["buildsByBranchName"]["'$GIT_BRANCH'"]["revision"]["SHA1"]'
  37}
  38
  39################################################################################
  40###############                   INITIAL SETUP                  ###############
  41################################################################################
  42
  43if [ "$INITIAL_SETUP" == true ]; then
  44        echo -e "\n[INITIAL_SETUP]: Begin"
  45        # Create directory for tests and other temp files.
  46        mkdir -p $TMP_DIR
  47        mkdir -p $TEST_OUTPUT_DIR
  48        mkdir -p $DOWNLOADS_DIR
  49
  50        # Compile QEMU launcher
  51        mkdir -p $WORKSPACE/install/qemu_launcher/
  52        gcc $SCR_DIR/qemu_launcher.c -o install/qemu_launcher/qemu_launcher
  53
  54        echo "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *"
  55        echo "Set up finished succesfully."
  56        echo "Please run sudo chown root:root install/qemu_launcher/qemu_launcher"
  57        echo "Please run sudo chmod 4755 install/qemu_launcher/qemu_launcher"
  58        echo "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *"
  59        echo ""
  60        echo -e "[INITIAL_SETUP]: End\n"
  61        exit 0
  62fi
  63
  64
  65
  66################################################################################
  67###############                 PRE BUILD SETUP                  ###############
  68################################################################################
  69
  70function add_cross_compiler_to_path() {
  71        export PATH=$WORKSPACE/install/riscv-ros-gcc/bin:$PATH
  72        export PATH=$WORKSPACE/install/x86_64-ros-gcc/bin:$PATH
  73}
  74
  75# Clean these two directories
  76rm $TMP_DIR/* $TEST_OUTPUT_DIR/* -f
  77add_cross_compiler_to_path
  78
  79
  80################################################################################
  81###############                    COMPILATION                   ###############
  82################################################################################
  83
  84function build_config() {
  85        echo -e "\n[SET_MAKE_CONFIG]: Begin"
  86
  87        # Begin with default configuration.
  88        case "$COMPILATION_ARCH" in
  89        RISCV)  make ARCH=riscv defconfig
  90            ;;
  91        X86_64)  make ARCH=x86 defconfig
  92            ;;
  93        esac
  94
  95        # Enable tests to run.
  96        # These don't take much to execute so we can run them always and just
  97        # parse results if needed.
  98        echo "CONFIG_KERNEL_TESTING=y" >> .config
  99        echo "CONFIG_USERSPACE_TESTING=y" >> .config
 100        echo "CONFIG_LOUSY_LAPIC_TIMER=y" >> .config
 101        # Set all config variables dependent on the above changes to their
 102        # defaults without prompting
 103        make olddefconfig 
 104
 105        echo -e "[SET_MAKE_CONFIG]: End\n"
 106}
 107
 108function build_cross_compiler() {
 109        declare -A ARCH_SUBDIRS=( ["RISCV"]="riscv-ros-gcc" \
 110                                  ["X86_64"]="x86_64-ros-gcc" )
 111
 112        echo -e "\n[BUILD_CROSS_COMPILER]: Begin"
 113
 114        cd tools/compilers/gcc-glibc
 115
 116        # Clean everything up
 117        # TODO: Possibly down the line try to optimize this to only clean the 
 118        # architecture that we need to rebuild.
 119        make clean
 120
 121        # Define cross compiler Makelocal.
 122        echo "# Number of make jobs to spawn.  
 123MAKE_JOBS := 3
 124RISCV_INSTDIR         := $WORKSPACE/install/${ARCH_SUBDIRS["RISCV"]}/
 125X86_64_INSTDIR        := $WORKSPACE/install/${ARCH_SUBDIRS["X86_64"]}/
 126" > Makelocal
 127
 128        # Create / clean directory where the cross compiler will be installed.
 129        CROSS_COMP_DIR=$WORKSPACE/install/${ARCH_SUBDIRS["$COMPILATION_ARCH"]}/
 130        rm -rf $CROSS_COMP_DIR
 131
 132        # Compile cross compiler.
 133        case "$COMPILATION_ARCH" in
 134        RISCV)  make riscv
 135            ;;
 136        X86_64)  make x86_64
 137            ;;
 138        esac
 139
 140        # Go back to root directory.
 141        cd ../../..
 142        echo -e "[BUILD_CROSS_COMPILER]: End\n"
 143}
 144
 145function build_kernel() {
 146        echo -e "\n[BUILD_KERNEL]: Begin"
 147        make clean
 148        make
 149        echo -e "[BUILD_KERNEL]: End\n"
 150}
 151
 152function build_userspace() {
 153        echo -e "\n[BUILD_USERSPACE]: Begin"
 154        # This is needed because of a bug that won't let tests to be compiled
 155        # unless the following files are present.
 156        cd kern/kfs/bin
 157        touch busybox
 158        touch chmod
 159        cd -
 160
 161        # Build and install user libs.
 162        make userclean
 163        make install-libs
 164
 165        # Compile tests.
 166        make utest
 167
 168        # Fill memory with tests.
 169        make fill-kfs
 170
 171        echo -e "[BUILD_USERSPACE]: End\n"
 172}
 173
 174function build_busybox() {
 175        echo -e "\n[BUILD_BUSYBOX]: Begin"
 176        
 177        BUSYBOX_DIR=busybox-$BUSYBOX_VERSION
 178        
 179        cd $DOWNLOADS_DIR
 180        
 181        # Download busybox if we do not have it yet.
 182        if [[ ! -d "$BUSYBOX_DIR" ]]; then
 183                echo "Trying to download from $BUSYBOX_DL_URL ..."
 184                
 185                wget $BUSYBOX_DL_URL -O busybox-$BUSYBOX_VERSION.tar.bz2
 186                tar -jxvf busybox-$BUSYBOX_VERSION.tar.bz2
 187                rm busybox-$BUSYBOX_VERSION.tar.bz2
 188                cp ../$BUSYBOX_CONF_FILE $BUSYBOX_DIR/.config
 189        fi
 190
 191        # Build busybox and copy it into kfs
 192        echo -e "Build busybox"
 193        cd $BUSYBOX_DIR
 194        make
 195        cp busybox_unstripped ../../kern/kfs/bin/busybox
 196        cd ../../
 197
 198        echo -e "[BUILD_BUSYBOX]: End\n"
 199}
 200
 201function build_all() {
 202        build_config
 203        build_cross_compiler
 204        build_userspace
 205        build_busybox
 206        build_kernel
 207}
 208
 209# TODO: This won't work for RISCV, it must be changed to whatever is used.
 210function run_qemu() {
 211        echo -e "\n[RUN_AKAROS_IN_QEMU]: Begin"
 212
 213        echo "-include $CONF_DIR/Makelocal_qemu" > Makelocal
 214        export PATH=$WORKSPACE/install/qemu_launcher/:$PATH
 215        make qemu > $AKAROS_OUTPUT_FILE &
 216        MAKE_PID=$!
 217
 218        WAIT_RESULT=`$SCR_WAIT_UNTIL $AKAROS_OUTPUT_FILE END_USERSPACE_TESTS \
 219            ${MAX_RUN_TIME:-100}`
 220
 221        # Extract Qemu_launcher PID
 222        QEMU_PID=`ps --ppid $MAKE_PID | grep qemu_launcher | sed -e 's/^\s*//' | \
 223                  cut -d' ' -f1`
 224
 225        # To kill qemu we need to send a USR1 signal to Qemu_launcher.
 226        kill -10 $QEMU_PID
 227
 228        # Then we wait....
 229        wait $MAKE_PID
 230
 231        # clean and cat the output of the file so we can see it in the console
 232        # log
 233        sed -i -e 's/
//g' $AKAROS_OUTPUT_FILE
 234        cat $AKAROS_OUTPUT_FILE
 235
 236        echo -e "[RUN_AKAROS_IN_QEMU]: End\n"
 237
 238        # If the run was terminated via a timeout, then we finish with an error.
 239        if [[ "$WAIT_RESULT" == TIMEOUT ]]; then
 240                echo "AKAROS was terminated after running for $MAX_RUN_TIME seconds."
 241                exit 1
 242        fi
 243}
 244
 245
 246
 247if [ "$COMPILE_ALL" == true ]; then
 248        echo "Building all AKAROS"
 249        build_all
 250        run_qemu
 251
 252        AFFECTED_COMPONENTS="cross-compiler kernel userspace busybox"
 253else
 254        # Save changed files between last tested commit and current one.
 255        git diff --stat $(last_stable_build) $GIT_COMMIT > $DIFF_FILE
 256        if [ $? -ne 0 ]; then
 257                echo "Diff failed, rebuild everything"
 258                build_all
 259                run_qemu
 260
 261                AFFECTED_COMPONENTS="cross-compiler kernel userspace busybox"
 262        else 
 263                # Extract build targets by parsing diff file.
 264                AFFECTED_COMPONENTS=`$SCR_GIT_CHANGES $DIFF_FILE $CONF_COMP_COMPONENTS_FILE`
 265                # Can contain {cross-compiler, kernel, userspace, busybox}
 266
 267                if [[ -n $AFFECTED_COMPONENTS ]]; 
 268                then
 269                        echo "Detected changes in "$AFFECTED_COMPONENTS
 270                        build_config
 271
 272                        if [[ $AFFECTED_COMPONENTS == *cross-compiler* ]]
 273                        then
 274                                build_cross_compiler
 275                                build_userspace
 276                                build_busybox
 277                                build_kernel
 278                        else 
 279                                if [[ $AFFECTED_COMPONENTS == *userspace* ]]
 280                                then
 281                                        build_userspace
 282                                fi
 283
 284                                if [[ $AFFECTED_COMPONENTS == *busybox* ]]
 285                                then
 286                                        build_busybox
 287                                fi
 288
 289                                if [[ $AFFECTED_COMPONENTS == *kernel* ]]
 290                                then
 291                                        build_kernel
 292                                fi
 293                        fi
 294                else
 295                        echo "Skipping build. No changes detected."
 296                fi
 297
 298                run_qemu
 299        fi
 300fi
 301
 302
 303################################################################################
 304###############                  TEST REPORTING                  ###############
 305################################################################################
 306
 307echo -e "\n[TEST_REPORTING]: Begin"
 308
 309# Generate test report
 310$SCR_GEN_TEST_REPORTS $AKAROS_OUTPUT_FILE $TEST_OUTPUT_DIR 
 311echo "Tests generated in $TEST_OUTPUT_DIR"
 312
 313echo -e "[TEST_REPORTING]: End\n"
 314