net: tcp: Remove the RETRAN flag

[akaros.git] / Makefile

diff --git a/Makefile b/Makefile
index dd10cc1..57e79e7 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -36,11 +36,52 @@
 #
 #      - There are a few other TODOs sprinkled throughout the makefile.
 
 #
 #      - There are a few other TODOs sprinkled throughout the makefile.
 

+# Number of make jobs to spawn.  Can override this in Makelocal
+MAKE_JOBS ?= $(shell expr `cat /proc/cpuinfo | grep processor | wc -l` - 1)
+
+# Allow people to override our setting of the --no-print-directory option in
+# their Makelocal. This is useful, for example, to allow emacs to find the
+# correct file when errors are encountered using its builtin 'M-x compile'
+# command.
+VERSION = 0
+PATCHLEVEL = 1
+SUBLEVEL = 0
+EXTRAVERSION =
+VERNAME = Nanwan
+
+KERNELVERSION = $(VERSION)$(if $(PATCHLEVEL),.$(PATCHLEVEL)$(if $(SUBLEVEL),.$(SUBLEVEL)))$(EXTRAVERSION)
+
+export KERNELVERSION VERNAME
+
+NO_PRINT_DIRECTORY ?= --no-print-directory
+
+# Save the ability to export the parent's original environment for future use
+export_parent_env := $(shell export | sed 's/$$/;/')
+
+# Save the ability to clear the current environment for future use
+clear_current_env := for c in $$(awk 'BEGIN{for (v in ENVIRON){print v}}'); do unset $$c; done;
+
+define export_user_variables
+       CROSS_COMPILE="$(CROSS_COMPILE)"\
+       CROSS_INCLUDE="$(XCC_TARGET_INCLUDE)"\
+       ROS_CFLAGS="$(CFLAGS_USER)"\
+       ROS_LDFLAGS="$(LDFLAGS_USER)"
+endef
+
+# Define a set of commands to reset the environment to the parent's environment
+# and then run a local make target
+define make_as_parent
+       $(clear_current_env)\
+       $(export_parent_env)\
+       $(call export_user_variables)\
+       $(MAKE) $(NO_PRINT_DIRECTORY) -j $(MAKE_JOBS) $(1)
+endef
+

 # Do not:
 # o  use make's built-in rules and variables
 #    (this increases performance and avoids hard-to-debug behaviour);
 # o  print "Entering directory ...";
 # Do not:
 # o  use make's built-in rules and variables
 #    (this increases performance and avoids hard-to-debug behaviour);
 # o  print "Entering directory ...";

-MAKEFLAGS += -rR --no-print-directory
+MAKEFLAGS += -rR $(NO_PRINT_DIRECTORY) -j $(MAKE_JOBS)

 
 # That's our default target when none is given on the command line
 # This can be overriden with a Makelocal
 
 # That's our default target when none is given on the command line
 # This can be overriden with a Makelocal


@@ -55,6 +96,10 @@ export AKAROS_ROOT
 # generate for non-kbuild targets
 OBJDIR ?= obj
 
 # generate for non-kbuild targets
 OBJDIR ?= obj
 

+# Don't need to export these, since the Makelocal is included.
+KERNEL_OBJ := $(OBJDIR)/kern/akaros-kernel
+CMP_KERNEL_OBJ := $(KERNEL_OBJ).gz
+

 # Symlinks
 # =========================================================================
 # We have a few symlinks so that code can include <arch/whatever.h>.  This
 # Symlinks
 # =========================================================================
 # We have a few symlinks so that code can include <arch/whatever.h>.  This


@@ -80,7 +125,7 @@ endif
 
 PHONY += symlinks clean_symlinks
 clean_symlinks: objclean
 
 PHONY += symlinks clean_symlinks
 clean_symlinks: objclean

-       @rm -f kern/include/arch kern/boot user/parlib/include/arch
+       @rm -f kern/include/arch kern/boot user/parlib/include/parlib/arch

 
 arch-link := $(notdir $(shell readlink kern/include/arch))
 valid-arches := $(notdir $(wildcard kern/arch/*))
 
 arch-link := $(notdir $(shell readlink kern/include/arch))
 valid-arches := $(notdir $(wildcard kern/arch/*))


@@ -98,7 +143,7 @@ symlinks: clean_symlinks
        @echo Making symlinks...
        $(Q)ln -fs ../arch/$(ARCH) kern/include/arch
        $(Q)ln -fs arch/$(ARCH)/boot kern/boot
        @echo Making symlinks...
        $(Q)ln -fs ../arch/$(ARCH) kern/include/arch
        $(Q)ln -fs arch/$(ARCH)/boot kern/boot

-       $(Q)ln -fs $(ARCH) user/parlib/include/arch
+       $(Q)ln -fs $(ARCH) user/parlib/include/parlib/arch

        $(Q)$(MAKE) -f $(srctree)/Makefile clean
 
     else
        $(Q)$(MAKE) -f $(srctree)/Makefile clean
 
     else


@@ -164,7 +209,7 @@ export CONFIG_SHELL HOSTCC HOSTCXX HOSTCFLAGS HOSTCXXFLAGS
 #         cmd_cc_o_c       = $(CC) $(c_flags) -c -o $@ $<
 #
 # If $(quiet) is empty, the whole command will be printed.
 #         cmd_cc_o_c       = $(CC) $(c_flags) -c -o $@ $<
 #
 # If $(quiet) is empty, the whole command will be printed.

-# If it is set to "quiet_", only the short version will be printed. 
+# If it is set to "quiet_", only the short version will be printed.

 # If it is set to "silent_", nothing will be printed at all, since
 # the variable $(silent_cmd_cc_o_c) doesn't exist.
 #
 # If it is set to "silent_", nothing will be printed at all, since
 # the variable $(silent_cmd_cc_o_c) doesn't exist.
 #


@@ -288,10 +333,10 @@ endif # $(dot-config)
 AKAROSINCLUDE   := -I$(srctree)/kern/include/
 
 # CROSS_COMPILE is defined per-arch.  Each arch can set other makeflags, kbuild
 AKAROSINCLUDE   := -I$(srctree)/kern/include/
 
 # CROSS_COMPILE is defined per-arch.  Each arch can set other makeflags, kbuild

-# directories, etc. 
+# directories, etc.

 -include $(srctree)/kern/arch/$(ARCH)/Makefile
 
 -include $(srctree)/kern/arch/$(ARCH)/Makefile
 

-CC         := $(CROSS_COMPILE)gcc 
+CC         := $(CROSS_COMPILE)gcc

 CPP        := $(CROSS_COMPILE)g++
 AS         := $(CROSS_COMPILE)as
 AR         := $(CROSS_COMPILE)ar
 CPP        := $(CROSS_COMPILE)g++
 AS         := $(CROSS_COMPILE)as
 AR         := $(CROSS_COMPILE)ar


@@ -318,8 +363,8 @@ CFLAGS_KERNEL += -O2 -pipe -MD
 CFLAGS_KERNEL += -std=gnu99 -fgnu89-inline
 CFLAGS_KERNEL += -fno-strict-aliasing -fno-omit-frame-pointer
 CFLAGS_KERNEL += -fno-stack-protector
 CFLAGS_KERNEL += -std=gnu99 -fgnu89-inline
 CFLAGS_KERNEL += -fno-strict-aliasing -fno-omit-frame-pointer
 CFLAGS_KERNEL += -fno-stack-protector

-CFLAGS_KERNEL += -Wall -Wno-format -Wno-unused
-CFLAGS_KERNEL += -DROS_KERNEL 
+CFLAGS_KERNEL += -Wall -Wno-format -Wno-unused -Werror -Wreturn-type
+CFLAGS_KERNEL += -DROS_KERNEL

 CFLAGS_KERNEL += -include include/generated/autoconf.h -include include/common.h
 CFLAGS_KERNEL += -fplan9-extensions
 ifeq ($(CONFIG_64BIT),y)
 CFLAGS_KERNEL += -include include/generated/autoconf.h -include include/common.h
 CFLAGS_KERNEL += -fplan9-extensions
 ifeq ($(CONFIG_64BIT),y)


@@ -327,6 +372,9 @@ CFLAGS_KERNEL += -m64 -g
 else
 CFLAGS_KERNEL += -m32 -gstabs
 endif
 else
 CFLAGS_KERNEL += -m32 -gstabs
 endif

+ifeq ($(CONFIG_BETTER_BACKTRACE),y)
+CFLAGS_KERNEL += -fno-optimize-sibling-calls
+endif

 
 # TODO: do we need this, or can we rely on the compiler's defines?
 CFLAGS_KERNEL += -D$(ARCH)
 
 # TODO: do we need this, or can we rely on the compiler's defines?
 CFLAGS_KERNEL += -D$(ARCH)


@@ -347,7 +395,8 @@ export CFLAGS_KERNEL AFLAGS_KERNEL
 export NOSTDINC_FLAGS XCC_TARGET_ROOT XCC_TARGET_LIB XCC_TARGET_INCLUDE
 export KBUILD_BUILTIN KBUILD_CHECKSRC
 
 export NOSTDINC_FLAGS XCC_TARGET_ROOT XCC_TARGET_LIB XCC_TARGET_INCLUDE
 export KBUILD_BUILTIN KBUILD_CHECKSRC
 

-CFLAGS_USER += -O2 -std=gnu99 -fno-stack-protector -fgnu89-inline
+CFLAGS_USER += -O2 -std=gnu99 -fno-stack-protector -fgnu89-inline \
+               -Wsystem-headers -Werror -Wreturn-type

 CXXFLAGS_USER += -O2
 CFLAGS_USER_LIBS += -fPIC -static -fno-omit-frame-pointer -g
 
 CXXFLAGS_USER += -O2
 CFLAGS_USER_LIBS += -fPIC -static -fno-omit-frame-pointer -g
 


@@ -360,10 +409,6 @@ export CFLAGS_USER CXXFLAGS_USER CFLAGS_USER_LIBS
 # kernel build output.  Right now, it's only passed down to tests/
 dummy-1 := $(shell mkdir -p $(OBJDIR)/kern/)
 
 # kernel build output.  Right now, it's only passed down to tests/
 dummy-1 := $(shell mkdir -p $(OBJDIR)/kern/)
 

-# Don't need to export these, since the Makelocal is included.
-KERNEL_OBJ := $(OBJDIR)/kern/akaros-kernel
-CMP_KERNEL_OBJ := $(KERNEL_OBJ).gz
-

 # Since we're doing this outside of the dot-config part, some targets, such as
 # clean, won't read in our .config/auto.conf, and won't know about the
 # KFS_PATH.  Future rules related to KFS will have issues (mkdir with no
 # Since we're doing this outside of the dot-config part, some targets, such as
 # clean, won't read in our .config/auto.conf, and won't know about the
 # KFS_PATH.  Future rules related to KFS will have issues (mkdir with no


@@ -375,8 +420,9 @@ kfs-paths := kern/kfs
 endif
 
 FIRST_KFS_PATH = $(firstword $(kfs-paths))
 endif
 
 FIRST_KFS_PATH = $(firstword $(kfs-paths))

+ABS_KFS_PATH = $(abspath $(FIRST_KFS_PATH))

 
 

-export OBJDIR FIRST_KFS_PATH
+export OBJDIR FIRST_KFS_PATH ABS_KFS_PATH

 
 # Avoiding implicit rules
 $(srctree)/Makelocal: ;
 
 # Avoiding implicit rules
 $(srctree)/Makelocal: ;


@@ -387,7 +433,7 @@ $(srctree)/Makelocal: ;
 # Akaros Kernel Build
 # =========================================================================
 # Add top level directories, either to an existing entry (core-y) or to its
 # Akaros Kernel Build
 # =========================================================================
 # Add top level directories, either to an existing entry (core-y) or to its

-# own. 
+# own.

 #
 # From these, we determine deps and dirs.  We recursively make through the
 # dirs, generating built-in.o at each step, which are the deps from which we
 #
 # From these, we determine deps and dirs.  We recursively make through the
 # dirs, generating built-in.o at each step, which are the deps from which we


@@ -456,8 +502,10 @@ $(kern_cpio) initramfs: $(kern_initramfs_files)
         cd $$OLDPWD; \
     done;
 
         cd $$OLDPWD; \
     done;
 

-ld_emulation = $(shell $(OBJDUMP) -i | grep -v BFD | grep ^[a-z] |head -n1)
-ld_arch = $(shell $(OBJDUMP) -i | grep -v BFD | grep "^  [a-z]" | head -n1)
+ld_emulation = $(shell $(OBJDUMP) -i 2>/dev/null | \
+                       grep -v BFD | grep ^[a-z] | head -n1)
+ld_arch = $(shell $(OBJDUMP) -i 2>/dev/null |\
+                  grep -v BFD | grep "^  [a-z]" | head -n1)

 
 # Our makefile doesn't detect a change in subarch, and old binary objects that
 # don't need to be updated won't get rebuilt, but they also can't link with the
 
 # Our makefile doesn't detect a change in subarch, and old binary objects that
 # don't need to be updated won't get rebuilt, but they also can't link with the


@@ -492,11 +540,12 @@ $(ext2_bdev_obj): $(ext2-bdev)
 # Not the worlds most elegant link command.  link-kernel takes the obj output
 # name, then the linker script, then everything else you'd dump on the ld
 # command line, including linker options and objects to link together.
 # Not the worlds most elegant link command.  link-kernel takes the obj output
 # name, then the linker script, then everything else you'd dump on the ld
 # command line, including linker options and objects to link together.

-# 
+#

 # After the script is done, we run the arch-specific command directly.
 quiet_cmd_link-akaros = LINK    $@
       cmd_link-akaros = $(CONFIG_SHELL) scripts/link-kernel.sh $@ \
                         kern/arch/$(ARCH)/$(KERNEL_LD) $(LDFLAGS_KERNEL) \
 # After the script is done, we run the arch-specific command directly.
 quiet_cmd_link-akaros = LINK    $@
       cmd_link-akaros = $(CONFIG_SHELL) scripts/link-kernel.sh $@ \
                         kern/arch/$(ARCH)/$(KERNEL_LD) $(LDFLAGS_KERNEL) \

+                        --build-id=sha1 \

                         $(akaros-deps) $(gcc-lib) $(kern_cpio_obj) \
                         $(ext2_bdev_obj); \
                         $(ARCH_POST_LINK_CMD)
                         $(akaros-deps) $(gcc-lib) $(kern_cpio_obj) \
                         $(ext2_bdev_obj); \
                         $(ARCH_POST_LINK_CMD)


@@ -542,39 +591,50 @@ endif #ifeq ($(mixed-targets),1)
 # List all userspace directories here, and state any dependencies between them,
 # such as how pthread depends on parlib.
 
 # List all userspace directories here, and state any dependencies between them,
 # such as how pthread depends on parlib.
 

-user-dirs = parlib pthread benchutil iplib ndblib
+# Critical libraries, also built during the toolchain install
+user-base-dirs = parlib pthread benchutil iplib ndblib perfmon

 benchutil: parlib
 pthread: parlib benchutil
 iplib: parlib
 ndblib: iplib
 
 benchutil: parlib
 pthread: parlib benchutil
 iplib: parlib
 ndblib: iplib
 

-PHONY += install-libs $(user-dirs)
-install-libs: $(user-dirs) symlinks cc-exists
+# Higher-level libraries.  Built before tests/, but after apps-install.
+# TODO: would like to move perfmon here, since it's not meant to be low-level.
+# But the apps-install has perf, which depends on user/perfmon.
+user-extra-dirs = vmm electric-fence
+$(user-extra-dirs): $(user-base-dirs)
+
+user-dirs = $(user-base-dirs) $(user-extra-dirs)
+
+PHONY += install-base-libs $(user-dirs)
+install-base-libs: $(user-base-dirs) symlinks cc-exists

 
 $(user-dirs):
 
 $(user-dirs):

-       @cd user/$@ && $(MAKE) DEPLIBS="$^" && $(MAKE) install
+       @$(MAKE) -C user/$@ DEPLIBS="$^" && $(MAKE) -C user/$@ install

 
 

+PHONY += user
+user: $(user-dirs)

 
 PHONY += userclean $(clean-user-dirs)
 clean-user-dirs := $(addprefix _clean_user_,$(user-dirs))
 userclean: $(clean-user-dirs) testclean utestclean
 
 $(clean-user-dirs):
 
 PHONY += userclean $(clean-user-dirs)
 clean-user-dirs := $(addprefix _clean_user_,$(user-dirs))
 userclean: $(clean-user-dirs) testclean utestclean
 
 $(clean-user-dirs):

-       @cd user/$(patsubst _clean_user_%,%,$@) && $(MAKE) clean
+       @$(MAKE) -C user/$(patsubst _clean_user_%,%,$@) clean

 
 tests/: tests
 
 tests/: tests

-tests: install-libs
+tests: user

        @$(MAKE) -f tests/Makefile
 
 PHONY += utest
 utest: $(user-dirs)
        @$(MAKE) -f tests/Makefile
 
 PHONY += utest
 utest: $(user-dirs)

-       @cd user/$@ && $(MAKE) 
+       @$(MAKE) -C user/$@

 
 testclean:
        @$(MAKE) -f tests/Makefile clean
 
 utestclean:
 
 testclean:
        @$(MAKE) -f tests/Makefile clean
 
 utestclean:

-       @cd user/utest && $(MAKE) clean
+       @$(MAKE) -C user/utest clean

 
 # KFS related stuff
 PHONY += fill-kfs unfill-kfs
 
 # KFS related stuff
 PHONY += fill-kfs unfill-kfs


@@ -587,22 +647,22 @@ $(OBJDIR)/.dont-force-fill-kfs:
        @echo "Cross Compiler 'so' files removed from KFS"
        @$(MAKE) -f tests/Makefile uninstall
        @echo "Apps from /test removed from KFS"
        @echo "Cross Compiler 'so' files removed from KFS"
        @$(MAKE) -f tests/Makefile uninstall
        @echo "Apps from /test removed from KFS"

-       @cd user/utest && $(MAKE) uninstall
+       @$(MAKE) -C user/utest uninstall

        @echo "User space tests removed from KFS"
        @touch $(OBJDIR)/.dont-force-fill-kfs
 
        @echo "User space tests removed from KFS"
        @touch $(OBJDIR)/.dont-force-fill-kfs
 

-fill-kfs: $(OBJDIR)/.dont-force-fill-kfs install-libs
+fill-kfs: $(OBJDIR)/.dont-force-fill-kfs user tests

        @mkdir -p $(FIRST_KFS_PATH)/lib
        $(Q)cp -uP $(xcc-so-files) $(FIRST_KFS_PATH)/lib
        @echo "Cross Compiler 'so' files installed to KFS"
        @$(MAKE) -f tests/Makefile install
        @echo "Apps from /test installed to KFS"
        @mkdir -p $(FIRST_KFS_PATH)/lib
        $(Q)cp -uP $(xcc-so-files) $(FIRST_KFS_PATH)/lib
        @echo "Cross Compiler 'so' files installed to KFS"
        @$(MAKE) -f tests/Makefile install
        @echo "Apps from /test installed to KFS"

-       @cd user/utest && $(MAKE) install
+       @$(MAKE) -C user/utest install

        @echo "User space tests installed to KFS"
 
 # Use doxygen to make documentation for ROS (Untested since 2010 or so)
 doxygen-dir := $(CUR_DIR)/Documentation/doxygen
        @echo "User space tests installed to KFS"
 
 # Use doxygen to make documentation for ROS (Untested since 2010 or so)
 doxygen-dir := $(CUR_DIR)/Documentation/doxygen

-docs: 
+docs:

        @echo "  Making doxygen"
        @doxygen-dir=$(doxygen-dir) doxygen $(doxygen-dir)/rosdoc.cfg
        @if [ ! -d $(doxygen-dir)/rosdoc/html/img ]; \
        @echo "  Making doxygen"
        @doxygen-dir=$(doxygen-dir) doxygen $(doxygen-dir)/rosdoc.cfg
        @if [ ! -d $(doxygen-dir)/rosdoc/html/img ]; \


@@ -620,6 +680,96 @@ objclean:
 
 realclean: userclean mrproper doxyclean objclean
 
 
 realclean: userclean mrproper doxyclean objclean
 

+# Bundled apps
+# =========================================================================
+
+app-dirs =
+tagged-app-dirs := $(subst /,__,$(app-dirs))
+app-dirs-install := $(addprefix _install_,$(tagged-app-dirs))
+app-dirs-clean := $(addprefix _clean_,$(tagged-app-dirs))
+
+PHONY += $(app-dirs-install) $(app-dirs-clean)
+
+$(app-dirs-install):
+       @$(MAKE) -C $(patsubst _install_%,%,$(subst __,/,$@)) install
+
+$(app-dirs-clean):
+       @$(MAKE) -C $(patsubst _clean_%,%,$(subst __,/,$@)) clean
+
+PHONY +=  apps-install
+apps-install: $(app-dirs-install)
+       @$(shell mkdir -p $(AKAROS_ROOT)/build_logs)
+       @$(call make_as_parent, -C tools/apps/busybox) > build_logs/busybox_install.log 2>&1
+       @echo busybox install succeeded.
+       @$(call make_as_parent, -C tools/apps/ipconfig install) > build_logs/ipconfig_install.log 2>&1
+       @echo ipconfig install succeeded.
+       @$(call make_as_parent, -C tools/dev-libs/elfutils install) > build_logs/elfutils_install.log 2>&1
+       @echo elfutils install succeeded.
+       @$(call make_as_parent, -C tools/dev-util/perf install) > build_logs/perf_install.log 2>&1
+       @echo perf install succeeded.
+       @$(call make_as_parent, -C tools/sys-apps/bash install) > build_logs/bash_install.log 2>&1
+       @echo bash install succeeded.
+
+PHONY += apps-clean
+apps-clean: $(app-dirs-clean)
+       @$(shell mkdir -p $(AKAROS_ROOT)/build_logs)
+       @$(call make_as_parent, -C tools/apps/busybox clean) > build_logs/busybox_clean.log 2>&1
+       @echo busybox clean succeeded.
+       @$(call make_as_parent, -C tools/apps/ipconfig clean) > build_logs/ipconfig_clean.log 2>&1
+       @echo ipconfig clean succeeded.
+       @$(call make_as_parent, -C tools/dev-libs/elfutils clean) > build_logs/elfutils_clean.log 2>&1
+       @echo elfutils clean succeeded.
+       @$(call make_as_parent, -C tools/dev-util/perf clean) > build_logs/perf_clean.log 2>&1
+       @echo perf clean succeeded.
+       @$(call make_as_parent, -C tools/sys-apps/bash clean) > build_logs/bash_clean.log 2>&1
+       @echo bash clean succeeded.
+
+# Cross Compiler
+# =========================================================================
+
+xcc_build_dir := tools/compilers/gcc-glibc
+xcc_target := $(ARCH)
+ifeq ($(xcc_target),x86)
+    xcc_target := $(xcc_target)_64
+endif
+xcc_cleans := $(shell $(MAKE) -C $(xcc_build_dir) -pn |\
+                      grep "VALID_CLEANS := " |\
+                      sed -e 's/VALID_CLEANS := //')
+xcc_subcmds := $(shell $(MAKE) -C $(xcc_build_dir) -pn |\
+                       grep "VALID_SUBCMDS := " |\
+                       sed -e 's/VALID_SUBCMDS := //')
+xcc_clean_goals := $(patsubst %, xcc-%, $(xcc_cleans))
+xcc_subcmd_goals := $(patsubst %, xcc-%, $(xcc_subcmds))
+
+PHONY += xcc
+xcc: xcc-build
+
+PHONY += $(xcc_clean_goals)
+$(xcc_clean_goals):
+       @target="$(patsubst xcc-%,%,$(@))";\
+       $(call make_as_parent, -C $(xcc_build_dir) $${target})
+
+PHONY += $(xcc_subcmd_goals)
+$(xcc_subcmd_goals):
+       @subcmd="$(patsubst xcc-%,%,$(@))";\
+       target="$(xcc_target) $${subcmd}";\
+       $(call make_as_parent, -C $(xcc_build_dir) $${target})
+
+PHONY += xcc-upgrade
+xcc-upgrade: xcc
+       @$(MAKE) userclean
+       @$(MAKE) install-base-libs
+       @$(MAKE) testclean utestclean
+       @$(call make_as_parent, apps-clean)
+       @$(call make_as_parent, apps-install)
+       @$(MAKE) tests utest
+       @$(MAKE) fill-kfs
+       @$(MAKE) akaros-kernel
+
+PHONY += xcc-upgrade-from-scratch
+xcc-upgrade-from-scratch: xcc-clean xcc-uninstall
+       @$(call make_as_parent, xcc-upgrade)
+

 # Cleaning
 # =========================================================================
 # This is mostly the Linux kernel cleaning.  We could hook in to the userspace
 # Cleaning
 # =========================================================================
 # This is mostly the Linux kernel cleaning.  We could hook in to the userspace