Turn on userspace tests and set defaults properly
[akaros.git] / tools / jenkins / launcher.sh
1 #!/bin/bash
2 # This script should be called from Jenkins when a new commit has been pushed 
3 # to the repo. 
4 # It analyzes what parts of the codebase have been modified, compiles everything
5 # that is needed, and reports on the results. 
6
7 set -e
8
9 readonly TMP_DIR=tmp
10 readonly DIFF_FILE=$TMP_DIR/changes.txt
11 readonly AKAROS_OUTPUT_FILE=$TMP_DIR/akaros_out.txt
12 readonly TEST_OUTPUT_DIR=output-tests
13 readonly TEST_DIR=tools/jenkins
14 readonly SCR_DIR=tools/jenkins/utils
15 readonly DOWNLOADS_DIR=dl
16
17 # Config files
18 readonly CONF_DIR=tools/jenkins/config
19 readonly CONF_COMP_COMPONENTS_FILE=$CONF_DIR/compilation_components.json
20
21 # Utility scripts
22 readonly SCR_WAIT_UNTIL=$SCR_DIR/wait_until.py
23 readonly SCR_GIT_CHANGES=$SCR_DIR/changes.py
24 readonly SCR_GEN_TEST_REPORTS=$SCR_DIR/test_reporter.py
25
26 # Busybox settings
27 readonly BUSYBOX_VERSION=1.17.3
28 readonly BUSYBOX_DL_URL=http://www.busybox.net/downloads/busybox-1.17.3.tar.bz2
29 readonly BUSYBOX_CONF_FILE=tools/patches/busybox/busybox-1.17.3-config
30
31 ################################################################################
32 ###############                   INITIAL SETUP                  ###############
33 ################################################################################
34
35 if [ "$INITIAL_SETUP" == true ]; then
36         echo -e "\n[INITIAL_SETUP]: Begin"
37         # Create directory for tests and other temp files.
38         mkdir -p $TMP_DIR
39         mkdir -p $TEST_OUTPUT_DIR
40         mkdir -p $DOWNLOADS_DIR
41
42         # Compile QEMU launcher
43         mkdir -p $WORKSPACE/install/qemu_launcher/
44         gcc $SCR_DIR/qemu_launcher.c -o install/qemu_launcher/qemu_launcher
45
46         echo "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *"
47         echo "Set up finished succesfully."
48         echo "Please run sudo chown root:root install/qemu_launcher/qemu_launcher"
49         echo "Please run sudo chmod 4755 install/qemu_launcher/qemu_launcher"
50         echo "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *"
51         echo ""
52         echo -e "[INITIAL_SETUP]: End\n"
53         exit 0
54 fi
55
56
57
58 ################################################################################
59 ###############                 PRE BUILD SETUP                  ###############
60 ################################################################################
61
62 function add_cross_compiler_to_path() {
63         export PATH=$WORKSPACE/install/riscv-ros-gcc/bin:$PATH
64         export PATH=$WORKSPACE/install/i686-ros-gcc/bin:$PATH
65         export PATH=$WORKSPACE/install/x86_64-ros-gcc/bin:$PATH
66 }
67
68 # Clean these two directories
69 rm $TMP_DIR/* $TEST_OUTPUT_DIR/* -f
70 add_cross_compiler_to_path
71
72
73 ################################################################################
74 ###############                    COMPILATION                   ###############
75 ################################################################################
76
77 function build_config() {
78         echo -e "\n[SET_MAKE_CONFIG]: Begin"
79
80         # Begin with default configuration.
81         case "$COMPILATION_ARCH" in
82         RISCV)  make ARCH=riscv defconfig
83             ;;
84         I686)  make ARCH=x86 defconfig
85                    sed -i -e 's/CONFIG_64BIT=y/# CONFIG_64BIT is not set/' \
86                           -e 's/# CONFIG_X86_32 is not set/CONFIG_X86_32=y/' \
87                           -e 's/CONFIG_X86_64=y/# CONFIG_X86_64 is not set/' \
88                           .config
89             ;;
90         X86_64)  make ARCH=x86 defconfig
91             ;;
92         esac
93
94         # Enable tests to run.
95         # These don't take much to execute so we can run them always and just parse
96         # results if needed.
97         echo "CONFIG_POSTBOOT_KERNEL_TESTING=y" >> .config
98         echo "CONFIG_USERSPACE_TESTING=y" >> .config
99         # Set all config variables dependent on the above changes to their defaults
100         # without prompting
101         make olddefconfig 
102
103         echo -e "[SET_MAKE_CONFIG]: End\n"
104 }
105
106 function build_cross_compiler() {
107         declare -A ARCH_SUBDIRS=( ["RISCV"]="riscv-ros-gcc" \
108                                   ["I686"]="i686-ros-gcc" \
109                                   ["X86_64"]="x86_64-ros-gcc" )
110
111         echo -e "\n[BUILD_CROSS_COMPILER]: Begin"
112
113         cd tools/compilers/gcc-glibc
114
115         # Clean everything up
116         # TODO: Possibly down the line try to optimize this to only clean the 
117         # architecture that we need to rebuild.
118         make clean
119
120         # Define cross compiler Makelocal.
121         echo "# Number of make jobs to spawn.  
122 MAKE_JOBS := 3
123 RISCV_INSTDIR         := $WORKSPACE/install/${ARCH_SUBDIRS["RISCV"]}/
124 I686_INSTDIR          := $WORKSPACE/install/${ARCH_SUBDIRS["I686"]}/
125 X86_64_INSTDIR        := $WORKSPACE/install/${ARCH_SUBDIRS["X86_64"]}/
126 " > Makelocal
127
128         # Create / clean directory where the cross compiler will be installed.
129         CROSS_COMP_DIR=$WORKSPACE/install/${ARCH_SUBDIRS["$COMPILATION_ARCH"]}/
130         mkdir -p CROSS_COMP_DIR
131         rm -rf CROSS_COMP_DIR*
132
133         # Compile cross compiler.
134         case "$COMPILATION_ARCH" in
135         RISCV)  make riscv
136             ;;
137         I686)  make i686
138             ;;
139         X86_64)  make x86_64
140             ;;
141         esac
142
143         # Go back to root directory.
144         cd ../../..
145         echo -e "[BUILD_CROSS_COMPILER]: End\n"
146 }
147
148 function build_kernel() {
149         echo -e "\n[BUILD_KERNEL]: Begin"
150         make clean
151         make
152         echo -e "[BUILD_KERNEL]: End\n"
153 }
154
155 function build_userspace() {
156         echo -e "\n[BUILD_USERSPACE]: Begin"
157         # This is needed because of a bug that won't let tests to be compiled
158         # unless the following files are present.
159         cd kern/kfs/bin
160         touch busybox
161         touch chmod
162         cd -
163
164         # Build and install user libs.
165         make userclean
166         make install-libs
167
168         # Compile tests.
169         make testclean
170         make tests
171
172         # Fill memory with tests.
173         make fill-kfs
174
175         # Recompile kernel.
176         make
177         echo -e "[BUILD_USERSPACE]: End\n"
178 }
179
180 function build_busybox() {
181         echo -e "\n[BUILD_BUSYBOX]: Begin"
182         
183         BUSYBOX_DIR=busybox-$BUSYBOX_VERSION
184         
185         cd $DOWNLOADS_DIR
186         
187         # Download busybox if we do not have it yet.
188         if [[ ! -d "$BUSYBOX_DIR" ]]; then
189                 echo "Trying to download from $BUSYBOX_DL_URL ..."
190                 
191                 wget $BUSYBOX_DL_URL -O busybox-$BUSYBOX_VERSION.tar.bz2
192                 tar -jxvf busybox-$BUSYBOX_VERSION.tar.bz2
193                 rm busybox-$BUSYBOX_VERSION.tar.bz2
194                 cp ../$BUSYBOX_CONF_FILE $BUSYBOX_DIR/.config
195         fi
196
197         # Build busybox and copy it into kfs
198         cd $BUSYBOX_DIR
199         make
200         cp busybox_unstripped ../../kern/kfs/bin/busybox
201         cd ../../
202
203         # Recompile kernel to include busybox
204         make
205
206         echo -e "[BUILD_BUSYBOX]: End\n"
207 }
208
209 # TODO: This won't work for RISCV, it must be changed to whatever is used.
210 function run_qemu() {
211         echo -e "\n[RUN_AKAROS_IN_QEMU]: Begin"
212
213         echo "-include $CONF_DIR/Makelocal_qemu" > Makelocal
214         export PATH=$WORKSPACE/install/qemu_launcher/:$PATH
215         make qemu > $AKAROS_OUTPUT_FILE &
216         MAKE_PID=$!
217
218         # TODO: Rather than finishing after Kernel PB Tests, put a generic 
219         #       "C'est fini" statement somewhere and look for it
220         WAIT_RESULT=`$SCR_WAIT_UNTIL $AKAROS_OUTPUT_FILE END_KERNEL_POSTBOOT_TESTS \
221             ${MAX_RUN_TIME:-100}`
222
223         # Extract Qemu_launcher PID
224         QEMU_PID=`ps --ppid $MAKE_PID | grep qemu_launcher | sed -e 's/^\s*//' | \
225                   cut -d' ' -f1`
226
227         # To kill qemu we need to send a USR1 signal to Qemu_launcher.
228         kill -10 $QEMU_PID
229
230         wait $MAKE_PID
231
232         echo -e "[RUN_AKAROS_IN_QEMU]: End\n"
233
234         # If the run was terminated via a timeout, then we finish with an error.
235         if [[ "$WAIT_RESULT" == TIMEOUT ]]; then
236                 echo "AKAROS was terminated after running for $MAX_RUN_TIME seconds."
237                 exit 1
238         fi
239 }
240
241
242
243 if [ "$COMPILE_ALL" == true ]; then
244         echo "Building all AKAROS"
245         build_config
246         
247         build_cross_compiler
248         build_kernel
249         build_userspace
250         build_busybox
251
252         run_qemu
253
254         AFFECTED_COMPONENTS="cross-compiler kernel userspace busybox"
255 else
256         # Save changed files between last tested commit and current one.
257         git diff --stat $GIT_PREVIOUS_COMMIT $GIT_COMMIT > $DIFF_FILE
258
259         # Extract build targets by parsing diff file.
260         AFFECTED_COMPONENTS=`$SCR_GIT_CHANGES $DIFF_FILE $CONF_COMP_COMPONENTS_FILE`
261         # Can contain {cross-compiler, kernel, userspace, busybox}
262
263         if [[ -n $AFFECTED_COMPONENTS ]]; 
264         then
265                 echo "Detected changes in "$AFFECTED_COMPONENTS
266                 build_config
267
268                 if [[ $AFFECTED_COMPONENTS == *cross-compiler* ]]
269                 then
270                         build_cross_compiler
271                         build_kernel
272                         build_userspace
273                         build_busybox
274                 else 
275                         if [[ $AFFECTED_COMPONENTS == *kernel* ]]
276                         then
277                                 build_kernel
278                         fi
279
280                         if [[ $AFFECTED_COMPONENTS == *userspace* ]]
281                         then
282                                 build_userspace
283                         fi
284
285                         if [[ $AFFECTED_COMPONENTS == *busybox* ]]
286                         then
287                                 build_busybox
288                         fi
289                 fi
290         else
291                 echo "Skipping build. No changes detected."
292         fi
293
294         run_qemu
295 fi
296
297
298 ################################################################################
299 ###############                  TEST REPORTING                  ###############
300 ################################################################################
301
302 echo -e "\n[TEST_REPORTING]: Begin"
303
304 TESTS_TO_RUN="KERNEL_POSTBOOT" # TODO(alfongj): Remove this when not needed.
305 # for COMPONENT in "${AFFECTED_COMPONENTS_ARRAY[@]}"; 
306 # do
307 #       # TODO(alfongj): Add to tests to run the name of the test suites to be ran.
308 #       # TESTS_TO_RUN="$TESTS_TO_RUN SOMETHING"
309 # done
310
311 # Generate test report
312 $SCR_GEN_TEST_REPORTS $AKAROS_OUTPUT_FILE $TEST_OUTPUT_DIR $TESTS_TO_RUN
313 echo "Tests generated in $TEST_OUTPUT_DIR"
314
315 echo -e "[TEST_REPORTING]: End\n"