Makefile: add option to build kernel with most symbols hidden

This patch adds new build configuration option - conf_hide_symbols -
that allows to build OSv kernel with all non-glibc symbols hidden
when enabled (set to 1). By default the conf_hide_symbols is set to disabled
so the kernel is still built with all symbols exported. In order to build
kernel with most symbols hidden, one can use following command:
```
./scripts/build image=native-example fs=rofs conf_hide_symbols=1 -j$(nproc)
```

The main idea behind the changes to the makefile below, is to compile
all source files except the ones under musl/ and libc/ directories with
the special compiler flags - '-fvisibility=hidden' and
'-fvisibility-inlines-hidden' (C++ only) if conf_hide_symbols is
enabled. This makes the symbols in all those relevant files as hidden
except the ones annotated with OSV_***_API macros to expose them as
public.
On other hand, the musl sources come with its own symbol visibility
mechanism where the symbols to be hidden are annotated with the 'hidden'
macro and everything else is public. Therefore we do not need to compile
the musl files with the visibility flags. Same goes for the files under
libc/ that originate from musl.
Lastly, the C++ files under libc/ that have been written from scratch
to provide parts of glibc API (like libc/pthread.cc) are compiled with
the compatibility flags. They are part of the libc_to_hide set.
Also depending on conf_hide_symbols, the makefile uses different linker
flags to link the standard C++ and others fully or not.

Relatedly, when conf_hide_symbols is enabled, the OSv dynamic linker
(core/elf.cc) does not advertise libstdc++.so.6 anymore.

The symbol hiding mechanism enabled with conf_hide_symbols is powerful
enough to hide most non-glibc symbols and leaves under 1,700 symbols
exported including some vtable and typeinfo left C++ ones which is ~10%
of the original number. The remaining C++ symbols will be removed from symbols
table once we enable version script when linking in future patches.

With conf_hide_symbols on, the resulting kernel-stripped.elf is ~ 5.1 MB in
size, down from 6.7 MB, mainly due to libstdc++.a not linked fully. Once
we enable linker garbage collection, the size should go down even more.

Please note that the kernel with hidden symbols does not support
building ZFS images as some of the symbols libzfs.so, zfs.so and zpool.so
depend on are no longer visible. To fix this we will probably need to
change how this apps are linked so they do not depend on those symbols
exported by kernel.

In addition around 35 unit tests cannot run on the kernel with most
hidden symbols as they directly use OSv internal symbols. Finally most
OSv apps and modules like httpserver.so rely on OSv specific API symbols
and they will not work either. To address this, we will need to expose
some of the OSv C++ API as C.

It is not clear if this patch fully addresses the issue #97. We could
however close it and open smaller ones to address remaining gaps.

Refs #97

Signed-off-by: Waldemar Kozaczuk <[email protected]>

Author: wkozaczuk

Makefile

@@ -279,7 +279,12 @@ $(out)/bsd/%.o: source-dialects =
 $(out)/libc/%.o: source-dialects =
 $(out)/musl/%.o: source-dialects =
 
-kernel-defines = -D_KERNEL $(source-dialects)
+# do not hide symbols in musl/libc because it has it's own hiding mechanism
+$(out)/libc/%.o: cc-hide-flags =
+$(out)/libc/%.o: cxx-hide-flags =
+$(out)/musl/%.o: cc-hide-flags =
+
+kernel-defines = -D_KERNEL $(source-dialects) $(cc-hide-flags)
 
 # This play the same role as "_KERNEL", but _KERNEL unfortunately is too
 # overloaded. A lot of files will expect it to be set no matter what, specially
@@ -310,9 +315,17 @@ tracing-flags-0 =
 tracing-flags-1 = -finstrument-functions -finstrument-functions-exclude-file-list=c++,trace.cc,trace.hh,align.hh,mmintrin.h
 tracing-flags = $(tracing-flags-$(conf-tracing))
 
+cc-hide-flags-0 =
+cc-hide-flags-1 = -fvisibility=hidden
+cc-hide-flags = $(cc-hide-flags-$(conf_hide_symbols))
+
+cxx-hide-flags-0 =
+cxx-hide-flags-1 = -fvisibility-inlines-hidden
+cxx-hide-flags = $(cxx-hide-flags-$(conf_hide_symbols))
+
 gcc-opt-Og := $(call compiler-flag, -Og, -Og, compiler/empty.cc)
 
-CXXFLAGS = -std=gnu++11 $(COMMON)
+CXXFLAGS = -std=gnu++11 $(COMMON) $(cxx-hide-flags)
 CFLAGS = -std=gnu99 $(COMMON)
 
 # should be limited to files under libc/ eventually
@@ -794,6 +807,7 @@ drivers += drivers/clock-common.o
 drivers += drivers/clockevent.o
 drivers += drivers/isa-serial-base.o
 drivers += core/elf.o
+$(out)/core/elf.o: CXXFLAGS += -DHIDE_SYMBOLS=$(conf_hide_symbols)
 drivers += drivers/random.o
 drivers += drivers/zfs.o
 drivers += drivers/null.o
@@ -949,6 +963,7 @@ objects += core/options.o
 
 #include $(src)/libc/build.mk:
 libc =
+libc_to_hide =
 musl =
 environ_libc =
 environ_musl =
@@ -960,6 +975,7 @@ musl_arch = aarch64
 endif
 
 libc += internal/_chk_fail.o
+libc_to_hide += internal/_chk_fail.o
 libc += internal/floatscan.o
 libc += internal/intscan.o
 libc += internal/libc.o
@@ -1029,6 +1045,7 @@ libc += errno/strerror.o
 
 musl += locale/catclose.o
 musl += locale/__mo_lookup.o
+$(out)/musl/src/locale/__mo_lookup.o: CFLAGS += $(cc-hide-flags-$(conf_hide_symbols))
 musl += locale/pleval.o
 musl += locale/catgets.o
 libc += locale/catopen.o
@@ -1316,13 +1333,16 @@ musl += misc/ffsll.o
 musl += misc/get_current_dir_name.o
 libc += misc/gethostid.o
 libc += misc/getopt.o
+libc_to_hide += misc/getopt.o
 libc += misc/getopt_long.o
+libc_to_hide += misc/getopt_long.o
 musl += misc/getsubopt.o
 libc += misc/realpath.o
 libc += misc/backtrace.o
 libc += misc/uname.o
 libc += misc/lockf.o
 libc += misc/mntent.o
+libc_to_hide += misc/mntent.o
 musl += misc/nftw.o
 libc += misc/__longjmp_chk.o
 
@@ -1360,7 +1380,7 @@ musl += network/gethostbyaddr_r.o
 musl += network/gethostbyaddr.o
 musl += network/resolvconf.o
 musl += network/res_msend.o
-$(out)/musl/src/network/res_msend.o: CFLAGS += -Wno-maybe-uninitialized --include libc/syscall_to_function.h --include libc/internal/pthread_stubs.h
+$(out)/musl/src/network/res_msend.o: CFLAGS += -Wno-maybe-uninitialized --include libc/syscall_to_function.h --include libc/internal/pthread_stubs.h $(cc-hide-flags-$(conf_hide_symbols))
 $(out)/libc/multibyte/mbsrtowcs.o: CFLAGS += -Imusl/src/multibyte
 musl += network/lookup_ipliteral.o
 libc += network/getaddrinfo.o
@@ -1374,7 +1394,9 @@ musl += network/lookup_name.o
 musl += network/lookup_serv.o
 libc += network/getnameinfo.o
 libc += network/__dns.o
+libc_to_hide += network/__dns.o
 libc += network/__ipparse.o
+libc_to_hide += network/__ipparse.o
 musl += network/inet_addr.o
 musl += network/inet_aton.o
 musl += network/inet_pton.o
@@ -1423,6 +1445,7 @@ ifeq ($(arch),x64)
 libc += arch/$(arch)/ucontext/getcontext.o
 libc += arch/$(arch)/ucontext/setcontext.o
 libc += arch/$(arch)/ucontext/start_context.o
+libc_to_hide += arch/$(arch)/ucontext/start_context.o
 libc += arch/$(arch)/ucontext/ucontext.o
 libc += string/memmove.o
 endif
@@ -1590,11 +1613,13 @@ musl += string/memccpy.o
 musl += string/memchr.o
 musl += string/memcmp.o
 libc += string/memcpy.o
+libc_to_hide += string/memcpy.o
 musl += string/memmem.o
 musl += string/mempcpy.o
 musl += string/memrchr.o
 libc += string/__memmove_chk.o
 libc += string/memset.o
+libc_to_hide += string/memset.o
 libc += string/__memset_chk.o
 libc += string/rawmemchr.o
 musl += string/rindex.o
@@ -1628,6 +1653,7 @@ musl += string/strpbrk.o
 musl += string/strrchr.o
 musl += string/strsep.o
 libc += string/stresep.o
+libc_to_hide += string/stresep.o
 musl += string/strsignal.o
 musl += string/strspn.o
 musl += string/strstr.o
@@ -1727,21 +1753,31 @@ musl += regex/tre-mem.o
 $(out)/musl/src/regex/tre-mem.o: CFLAGS += -UNDEBUG
 
 libc += pthread.o
+libc_to_hide += pthread.o
 libc += pthread_barrier.o
 libc += libc.o
 libc += dlfcn.o
 libc += time.o
+libc_to_hide += time.o
 libc += signal.o
+libc_to_hide += signal.o
 libc += mman.o
+libc_to_hide += mman.o
 libc += sem.o
+libc_to_hide += sem.o
 libc += pipe_buffer.o
+libc_to_hide += pipe_buffer.o
 libc += pipe.o
+libc_to_hide += pipe.o
 libc += af_local.o
+libc_to_hide += af_local.o
 libc += user.o
 libc += resource.o
 libc += mount.o
 libc += eventfd.o
+libc_to_hide += eventfd.o
 libc += timerfd.o
+libc_to_hide += timerfd.o
 libc += shm.o
 libc += inotify.o
 libc += __pread64_chk.o
@@ -1814,6 +1850,10 @@ objects += $(addprefix fs/, $(fs_objs))
 objects += $(addprefix libc/, $(libc))
 objects += $(addprefix musl/src/, $(musl))
 
+libc_objects_to_hide = $(addprefix $(out)/libc/, $(libc_to_hide))
+$(libc_objects_to_hide): cc-hide-flags = $(cc-hide-flags-$(conf_hide_symbols))
+$(libc_objects_to_hide): cxx-hide-flags = $(cxx-hide-flags-$(conf_hide_symbols))
+
 libstdc++.a := $(shell $(CXX) -print-file-name=libstdc++.a)
 ifeq ($(filter /%,$(libstdc++.a)),)
 ifeq ($(arch),aarch64)
@@ -1914,15 +1954,18 @@ $(loader_options_dep): stage1
 		echo -n "APP_LOCAL_EXEC_TLS_SIZE = $(app_local_exec_tls_size);" > $(loader_options_dep) ; \
 	fi
 
+ifeq ($(conf_hide_symbols),1)
+linker_archives_options = --no-whole-archive $(libstdc++.a) $(libgcc.a) $(libgcc_eh.a) $(boost-libs) --exclude-libs libstdc++.a
+else
+linker_archives_options = --whole-archive $(libstdc++.a) $(libgcc_eh.a) $(boost-libs) --no-whole-archive $(libgcc.a)
+endif
+
 $(out)/loader.elf: $(stage1_targets) arch/$(arch)/loader.ld $(out)/bootfs.o $(loader_options_dep)
 	$(call quiet, $(LD) -o $@ --defsym=OSV_KERNEL_BASE=$(kernel_base) \
 	    --defsym=OSV_KERNEL_VM_BASE=$(kernel_vm_base) --defsym=OSV_KERNEL_VM_SHIFT=$(kernel_vm_shift) \
 		-Bdynamic --export-dynamic --eh-frame-hdr --enable-new-dtags -L$(out)/arch/$(arch) \
 	    $(^:%.ld=-T %.ld) \
-	    --whole-archive \
-	      $(libstdc++.a) $(libgcc_eh.a) \
-	      $(boost-libs) \
-	    --no-whole-archive $(libgcc.a), \
+	    $(linker_archives_options), \
 		LINK loader.elf)
 	@# Build libosv.so matching this loader.elf. This is not a separate
 	@# rule because that caused bug #545.
@@ -1935,10 +1978,7 @@ $(out)/kernel.elf: $(stage1_targets) arch/$(arch)/loader.ld $(out)/empty_bootfs.
 	    --defsym=OSV_KERNEL_VM_BASE=$(kernel_vm_base) --defsym=OSV_KERNEL_VM_SHIFT=$(kernel_vm_shift) \
 		-Bdynamic --export-dynamic --eh-frame-hdr --enable-new-dtags -L$(out)/arch/$(arch) \
 	    $(^:%.ld=-T %.ld) \
-	    --whole-archive \
-	      $(libstdc++.a) $(libgcc_eh.a) \
-	      $(boost-libs) \
-	    --no-whole-archive $(libgcc.a), \
+	    $(linker_archives_options), \
 		LINK kernel.elf)
 	$(call quiet, $(STRIP) $(out)/kernel.elf -o $(out)/kernel-stripped.elf, STRIP kernel.elf -> kernel-stripped.elf )
 	$(call very-quiet, cp $(out)/kernel-stripped.elf $(out)/kernel.elf)

conf/base.mk

@@ -11,3 +11,4 @@ conf-logger_debug=0
 conf-DEBUG_BUILD=0
 
 conf-debug_elf=0
+conf_hide_symbols=0

core/elf.cc

@@ -1315,19 +1315,25 @@ program::program(void* addr)
           // this library, it will not be visible for the application and
           // it will need to load its own version of this library.
 #if BOOST_VERSION < 106900
+#if HIDE_SYMBOLS < 1
           "libboost_system.so.1.55.0",
 #endif
+#endif
 #endif /* __x86_64__ */
 #ifdef __aarch64__
           "ld-linux-aarch64.so.1",
 #if BOOST_VERSION < 106900
+#if HIDE_SYMBOLS < 1
           "libboost_system-mt.so.1.55.0",
 #endif
+#endif
 #endif /* __aarch64__ */
           "libpthread.so.0",
           "libdl.so.2",
           "librt.so.1",
+#if HIDE_SYMBOLS < 1
           "libstdc++.so.6",
+#endif
           "libaio.so.1",
           "libxenstore.so.3.0",
           "libcrypt.so.1",

scripts/manifest_from_host.sh

@@ -68,9 +68,15 @@ output_manifest()
 	echo "# --------------------" | tee -a $OUTPUT
 	echo "# Dependencies" | tee -a $OUTPUT
 	echo "# --------------------" | tee -a $OUTPUT
-	lddtree $so_path | grep -v "not found" | grep -v "$so_path" | grep -v "ld-linux-${MACHINE}" | \
-		grep -Pv 'lib(gcc_s|resolv|c|m|pthread|dl|rt|stdc\+\+|aio|xenstore|crypt|selinux)\.so([\d.]+)?' | \
-		sed 's/ =>/:/' | sed 's/^\s*lib/\/usr\/lib\/lib/' | sort | uniq | tee -a $OUTPUT
+	if [[ $conf_hide_symbols == 1 ]]; then
+		lddtree $so_path | grep -v "not found" | grep -v "$so_path" | grep -v "ld-linux-${MACHINE}" | \
+			grep -Pv 'lib(gcc_s|resolv|c|m|pthread|dl|rt|aio|xenstore|crypt|selinux)\.so([\d.]+)?' | \
+			sed 's/ =>/:/' | sed 's/^\s*lib/\/usr\/lib\/lib/' | sort | uniq | tee -a $OUTPUT
+	else
+		lddtree $so_path | grep -v "not found" | grep -v "$so_path" | grep -v "ld-linux-${MACHINE}" | \
+			grep -Pv 'lib(gcc_s|resolv|c|m|pthread|dl|rt|stdc\+\+|aio|xenstore|crypt|selinux)\.so([\d.]+)?' | \
+			sed 's/ =>/:/' | sed 's/^\s*lib/\/usr\/lib\/lib/' | sort | uniq | tee -a $OUTPUT
+	fi
 }
 
 detect_elf()
@@ -144,9 +150,15 @@ if [[ -d $NAME_OR_PATH ]]; then
 		echo "# --------------------" | tee -a $OUTPUT
 		echo "# Dependencies" | tee -a $OUTPUT
 		echo "# --------------------" | tee -a $OUTPUT
-		lddtree $SO_FILES | grep -v "not found" | grep -v "$NAME_OR_PATH/$SUBDIRECTORY_PATH" | grep -v "ld-linux-${MACHINE}" | \
-			grep -Pv 'lib(gcc_s|resolv|c|m|pthread|dl|rt|stdc\+\+|aio|xenstore|crypt|selinux)\.so([\d.]+)?' | \
-			sed 's/ =>/:/' | sed 's/^\s*lib/\/usr\/lib\/lib/' | sort | uniq | tee -a $OUTPUT
+		if [[ $conf_hide_symbols == 1 ]]; then
+			lddtree $SO_FILES | grep -v "not found" | grep -v "$NAME_OR_PATH/$SUBDIRECTORY_PATH" | grep -v "ld-linux-${MACHINE}" | \
+				grep -Pv 'lib(gcc_s|resolv|c|m|pthread|dl|rt|aio|xenstore|crypt|selinux)\.so([\d.]+)?' | \
+				sed 's/ =>/:/' | sed 's/^\s*lib/\/usr\/lib\/lib/' | sort | uniq | tee -a $OUTPUT
+		else
+			lddtree $SO_FILES | grep -v "not found" | grep -v "$NAME_OR_PATH/$SUBDIRECTORY_PATH" | grep -v "ld-linux-${MACHINE}" | \
+				grep -Pv 'lib(gcc_s|resolv|c|m|pthread|dl|rt|stdc\+\+|aio|xenstore|crypt|selinux)\.so([\d.]+)?' | \
+				sed 's/ =>/:/' | sed 's/^\s*lib/\/usr\/lib\/lib/' | sort | uniq | tee -a $OUTPUT
+		fi
 	fi
 	exit 0
 fi