Jenkins: Rebuild everything if build number wrong
[akaros.git] / tools / jenkins / launcher.sh
1 #!/bin/bash
2 # This script should be called from Jenkins when a new commit has been pushed 
3 # to the repo. 
4 # It analyzes what parts of the codebase have been modified, compiles everything
5 # that is needed, and reports on the results. 
6
7 #set -e
8
9 readonly TMP_DIR=tmp
10 readonly DIFF_FILE=$TMP_DIR/changes.txt
11 readonly AKAROS_OUTPUT_FILE=$TMP_DIR/akaros_out.txt
12 readonly TEST_OUTPUT_DIR=output-tests
13 readonly TEST_DIR=tools/jenkins
14 readonly SCR_DIR=tools/jenkins/utils
15 readonly DOWNLOADS_DIR=dl
16
17 # Config files
18 readonly CONF_DIR=tools/jenkins/config
19 readonly CONF_COMP_COMPONENTS_FILE=$CONF_DIR/compilation_components.json
20
21 # Utility scripts
22 readonly SCR_WAIT_UNTIL=$SCR_DIR/wait_until.py
23 readonly SCR_GIT_CHANGES=$SCR_DIR/changes.py
24 readonly SCR_GEN_TEST_REPORTS=$SCR_DIR/test_reporter.py
25
26 # Busybox settings
27 readonly BUSYBOX_VERSION=1.17.3
28 readonly BUSYBOX_DL_URL=http://www.busybox.net/downloads/busybox-1.17.3.tar.bz2
29 readonly BUSYBOX_CONF_FILE=tools/patches/busybox/busybox-1.17.3-config
30
31 ################################################################################
32 ###############                 HELPER FUNCTIONS                 ###############
33 ################################################################################
34 function last_stable_build() {
35         curl -s localhost:8080/job/$JOB_NAME/lastStableBuild/api/json?tree=actions%5bbuildsByBranchName%5brevision%5bSHA1%5d%5d%5d | \
36                 python -c 'import sys, json; print json.load(sys.stdin)["actions"][3]["buildsByBranchName"]["'$GIT_BRANCH'"]["revision"]["SHA1"]'
37 }
38
39 ################################################################################
40 ###############                   INITIAL SETUP                  ###############
41 ################################################################################
42
43 if [ "$INITIAL_SETUP" == true ]; then
44         echo -e "\n[INITIAL_SETUP]: Begin"
45         # Create directory for tests and other temp files.
46         mkdir -p $TMP_DIR
47         mkdir -p $TEST_OUTPUT_DIR
48         mkdir -p $DOWNLOADS_DIR
49
50         # Compile QEMU launcher
51         mkdir -p $WORKSPACE/install/qemu_launcher/
52         gcc $SCR_DIR/qemu_launcher.c -o install/qemu_launcher/qemu_launcher
53
54         echo "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *"
55         echo "Set up finished succesfully."
56         echo "Please run sudo chown root:root install/qemu_launcher/qemu_launcher"
57         echo "Please run sudo chmod 4755 install/qemu_launcher/qemu_launcher"
58         echo "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *"
59         echo ""
60         echo -e "[INITIAL_SETUP]: End\n"
61         exit 0
62 fi
63
64
65
66 ################################################################################
67 ###############                 PRE BUILD SETUP                  ###############
68 ################################################################################
69
70 function add_cross_compiler_to_path() {
71         export PATH=$WORKSPACE/install/riscv-ros-gcc/bin:$PATH
72         export PATH=$WORKSPACE/install/i686-ros-gcc/bin:$PATH
73         export PATH=$WORKSPACE/install/x86_64-ros-gcc/bin:$PATH
74 }
75
76 # Clean these two directories
77 rm $TMP_DIR/* $TEST_OUTPUT_DIR/* -f
78 add_cross_compiler_to_path
79
80
81 ################################################################################
82 ###############                    COMPILATION                   ###############
83 ################################################################################
84
85 function build_config() {
86         echo -e "\n[SET_MAKE_CONFIG]: Begin"
87
88         # Begin with default configuration.
89         case "$COMPILATION_ARCH" in
90         RISCV)  make ARCH=riscv defconfig
91             ;;
92         I686)  make ARCH=x86 defconfig
93                    sed -i -e 's/CONFIG_64BIT=y/# CONFIG_64BIT is not set/' \
94                           -e 's/# CONFIG_X86_32 is not set/CONFIG_X86_32=y/' \
95                           -e 's/CONFIG_X86_64=y/# CONFIG_X86_64 is not set/' \
96                           .config
97             ;;
98         X86_64)  make ARCH=x86 defconfig
99             ;;
100         esac
101
102         # Enable tests to run.
103         # These don't take much to execute so we can run them always and just parse
104         # results if needed.
105         echo "CONFIG_KERNEL_POSTBOOT_TESTING=y" >> .config
106         echo "CONFIG_USERSPACE_TESTING=y" >> .config
107         # Set all config variables dependent on the above changes to their defaults
108         # without prompting
109         make olddefconfig 
110
111         echo -e "[SET_MAKE_CONFIG]: End\n"
112 }
113
114 function build_cross_compiler() {
115         declare -A ARCH_SUBDIRS=( ["RISCV"]="riscv-ros-gcc" \
116                                   ["I686"]="i686-ros-gcc" \
117                                   ["X86_64"]="x86_64-ros-gcc" )
118
119         echo -e "\n[BUILD_CROSS_COMPILER]: Begin"
120
121         cd tools/compilers/gcc-glibc
122
123         # Clean everything up
124         # TODO: Possibly down the line try to optimize this to only clean the 
125         # architecture that we need to rebuild.
126         make clean
127
128         # Define cross compiler Makelocal.
129         echo "# Number of make jobs to spawn.  
130 MAKE_JOBS := 3
131 RISCV_INSTDIR         := $WORKSPACE/install/${ARCH_SUBDIRS["RISCV"]}/
132 I686_INSTDIR          := $WORKSPACE/install/${ARCH_SUBDIRS["I686"]}/
133 X86_64_INSTDIR        := $WORKSPACE/install/${ARCH_SUBDIRS["X86_64"]}/
134 " > Makelocal
135
136         # Create / clean directory where the cross compiler will be installed.
137         CROSS_COMP_DIR=$WORKSPACE/install/${ARCH_SUBDIRS["$COMPILATION_ARCH"]}/
138         mkdir -p CROSS_COMP_DIR
139         rm -rf CROSS_COMP_DIR*
140
141         # Compile cross compiler.
142         case "$COMPILATION_ARCH" in
143         RISCV)  make riscv
144             ;;
145         I686)  make i686
146             ;;
147         X86_64)  make x86_64
148             ;;
149         esac
150
151         # Go back to root directory.
152         cd ../../..
153         echo -e "[BUILD_CROSS_COMPILER]: End\n"
154 }
155
156 function build_kernel() {
157         echo -e "\n[BUILD_KERNEL]: Begin"
158         make clean
159         make
160         echo -e "[BUILD_KERNEL]: End\n"
161 }
162
163 function build_userspace() {
164         echo -e "\n[BUILD_USERSPACE]: Begin"
165         # This is needed because of a bug that won't let tests to be compiled
166         # unless the following files are present.
167         cd kern/kfs/bin
168         touch busybox
169         touch chmod
170         cd -
171
172         # Build and install user libs.
173         make userclean
174         make install-libs
175
176         # Compile tests.
177         make utest
178
179         # Fill memory with tests.
180         make fill-kfs
181
182         echo -e "[BUILD_USERSPACE]: End\n"
183 }
184
185 function build_busybox() {
186         echo -e "\n[BUILD_BUSYBOX]: Begin"
187         
188         BUSYBOX_DIR=busybox-$BUSYBOX_VERSION
189         
190         cd $DOWNLOADS_DIR
191         
192         # Download busybox if we do not have it yet.
193         if [[ ! -d "$BUSYBOX_DIR" ]]; then
194                 echo "Trying to download from $BUSYBOX_DL_URL ..."
195                 
196                 wget $BUSYBOX_DL_URL -O busybox-$BUSYBOX_VERSION.tar.bz2
197                 tar -jxvf busybox-$BUSYBOX_VERSION.tar.bz2
198                 rm busybox-$BUSYBOX_VERSION.tar.bz2
199                 cp ../$BUSYBOX_CONF_FILE $BUSYBOX_DIR/.config
200         fi
201
202         # Build busybox and copy it into kfs
203         echo -e "Build busybox"
204         cd $BUSYBOX_DIR
205         make
206         cp busybox_unstripped ../../kern/kfs/bin/busybox
207         cd ../../
208
209         # Recompile kernel to include busybox
210         echo -e "Recompile kernel to include busybox"
211         make
212
213         echo -e "[BUILD_BUSYBOX]: End\n"
214 }
215
216 function build_all() {
217         build_config
218         build_cross_compiler
219         build_userspace
220         build_busybox
221         build_kernel
222 }
223
224 # TODO: This won't work for RISCV, it must be changed to whatever is used.
225 function run_qemu() {
226         echo -e "\n[RUN_AKAROS_IN_QEMU]: Begin"
227
228         echo "-include $CONF_DIR/Makelocal_qemu" > Makelocal
229         export PATH=$WORKSPACE/install/qemu_launcher/:$PATH
230         make qemu > $AKAROS_OUTPUT_FILE &
231         MAKE_PID=$!
232
233         WAIT_RESULT=`$SCR_WAIT_UNTIL $AKAROS_OUTPUT_FILE END_USERSPACE_TESTS \
234             ${MAX_RUN_TIME:-100}`
235
236         # Extract Qemu_launcher PID
237         QEMU_PID=`ps --ppid $MAKE_PID | grep qemu_launcher | sed -e 's/^\s*//' | \
238                   cut -d' ' -f1`
239
240         # To kill qemu we need to send a USR1 signal to Qemu_launcher.
241         kill -10 $QEMU_PID
242
243         # Then we wait....
244         wait $MAKE_PID
245
246         # clean and cat the output of the file so we can see it in the console log
247         sed -i -e 's/\r//g' $AKAROS_OUTPUT_FILE
248         cat $AKAROS_OUTPUT_FILE
249
250         echo -e "[RUN_AKAROS_IN_QEMU]: End\n"
251
252         # If the run was terminated via a timeout, then we finish with an error.
253         if [[ "$WAIT_RESULT" == TIMEOUT ]]; then
254                 echo "AKAROS was terminated after running for $MAX_RUN_TIME seconds."
255                 exit 1
256         fi
257 }
258
259
260
261 if [ "$COMPILE_ALL" == true ]; then
262         echo "Building all AKAROS"
263         build_all
264         run_qemu
265
266         AFFECTED_COMPONENTS="cross-compiler kernel userspace busybox"
267 else
268         # Save changed files between last tested commit and current one.
269         git diff --stat $(last_stable_build) $GIT_COMMIT > $DIFF_FILE
270         if [ $? -ne 0 ]; then
271                 echo "Diff failed, rebuild everything"
272                 build_all
273                 run_qemu
274
275                 AFFECTED_COMPONENTS="cross-compiler kernel userspace busybox"
276         else 
277                 # Extract build targets by parsing diff file.
278                 AFFECTED_COMPONENTS=`$SCR_GIT_CHANGES $DIFF_FILE $CONF_COMP_COMPONENTS_FILE`
279                 # Can contain {cross-compiler, kernel, userspace, busybox}
280
281                 if [[ -n $AFFECTED_COMPONENTS ]]; 
282                 then
283                         echo "Detected changes in "$AFFECTED_COMPONENTS
284                         build_config
285
286                         if [[ $AFFECTED_COMPONENTS == *cross-compiler* ]]
287                         then
288                                 build_cross_compiler
289                                 build_userspace
290                                 build_busybox
291                                 build_kernel
292                         else 
293                                 if [[ $AFFECTED_COMPONENTS == *userspace* ]]
294                                 then
295                                         build_userspace
296                                 fi
297
298                                 if [[ $AFFECTED_COMPONENTS == *busybox* ]]
299                                 then
300                                         build_busybox
301                                 fi
302
303                                 if [[ $AFFECTED_COMPONENTS == *kernel* ]]
304                                 then
305                                         build_kernel
306                                 fi
307                         fi
308                 else
309                         echo "Skipping build. No changes detected."
310                 fi
311
312                 run_qemu
313         fi
314 fi
315
316
317 ################################################################################
318 ###############                  TEST REPORTING                  ###############
319 ################################################################################
320
321 echo -e "\n[TEST_REPORTING]: Begin"
322
323 # Generate test report
324 $SCR_GEN_TEST_REPORTS $AKAROS_OUTPUT_FILE $TEST_OUTPUT_DIR 
325 echo "Tests generated in $TEST_OUTPUT_DIR"
326
327 echo -e "[TEST_REPORTING]: End\n"