From d4d8ce85f2e4d9c569d6e7cba2d1007c7db3ef7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 3 Dec 2019 08:22:50 +0100 Subject: [PATCH 1/2] Pin specific commit for all Git clones This helps ensure that container builds are reproducible if thirdparty repositories change. --- Dockerfile.ios | 1 + Dockerfile.javascript | 1 + Dockerfile.xcode | 2 ++ 3 files changed, 4 insertions(+) diff --git a/Dockerfile.ios b/Dockerfile.ios index 7d1bf2c..f389681 100644 --- a/Dockerfile.ios +++ b/Dockerfile.ios @@ -5,6 +5,7 @@ RUN dnf -y install --setopt=install_weak_deps=False \ dnf clean all && \ git clone https://github.com/tpoechtrager/cctools-port.git && \ cd /root/cctools-port && \ + git checkout 8239a5211bcf07d6b9d359782e1a889ec1d7cce5 && \ sed -i 's#./autogen.sh#libtoolize -c -i --force\n./autogen.sh#' usage_examples/ios_toolchain/build.sh && \ usage_examples/ios_toolchain/build.sh /root/files/iPhoneOS12.4.sdk.tar.xz arm64 && \ mkdir -p /root/ioscross/arm64 && \ diff --git a/Dockerfile.javascript b/Dockerfile.javascript index 5ca876e..d699116 100644 --- a/Dockerfile.javascript +++ b/Dockerfile.javascript @@ -7,6 +7,7 @@ RUN dnf -y install --setopt=install_weak_deps=False \ dnf clean all && \ git clone https://github.com/emscripten-core/emsdk && \ cd emsdk && \ + git checkout a5082b232617c762cb65832429f896c838df2483 && \ ./emsdk install 1.38.47-upstream && \ ./emsdk activate 1.38.47-upstream && \ echo "source /root/emsdk/emsdk_env.sh" >> /root/.bashrc diff --git a/Dockerfile.xcode b/Dockerfile.xcode index 417c05d..a5c5345 100644 --- a/Dockerfile.xcode +++ b/Dockerfile.xcode @@ -5,11 +5,13 @@ RUN dnf -y install --setopt=install_weak_deps=False \ dnf clean all && \ git clone https://github.com/mackyle/xar.git && \ cd xar/xar && \ + git checkout 66d451dab1ef859dd0c83995f2379335d35e53c9 && \ ./autogen.sh --prefix=/usr && \ make -j && make install && \ cd /root && \ git clone https://github.com/NiklasRosenstein/pbzx && \ cd pbzx && \ + git checkout 2a4d7c3300c826d918def713a24d25c237c8ed53 && \ clang -O3 -llzma -lxar -I /usr/local/include pbzx.c -o pbzx CMD mkdir -p /root/xcode && \ From e6e211f49568090ec40ad244c302ad065c94ee21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 3 Dec 2019 09:54:20 +0100 Subject: [PATCH 2/2] JavaScript: Update and fix mono patch for Emscripten 1.38.42+ Cherry-pick the whole https://github.com/mono/mono/pull/16636 We were also attempting to patch the wrong folder previously (I had it working in my local tests but committed the wrong patch when rebasing to make the Git history pretty... :)). --- Dockerfile.javascript | 5 +- .../mono-pr16636-wasm-bugfix-and-update.diff | 144 ++++++++++++++++++ ...ono-wasm-avoid-aligning-stack-bounds.patch | 59 ------- 3 files changed, 147 insertions(+), 61 deletions(-) create mode 100644 files/patches/mono-pr16636-wasm-bugfix-and-update.diff delete mode 100644 files/patches/mono-wasm-avoid-aligning-stack-bounds.patch diff --git a/Dockerfile.javascript b/Dockerfile.javascript index d699116..b0cf92e 100644 --- a/Dockerfile.javascript +++ b/Dockerfile.javascript @@ -13,14 +13,15 @@ RUN dnf -y install --setopt=install_weak_deps=False \ echo "source /root/emsdk/emsdk_env.sh" >> /root/.bashrc RUN git clone https://github.com/mono/mono --branch ${mono_version} --single-branch && \ - cd mono && git submodule update --init && cd .. && \ + cd mono && git submodule update --init && \ + git apply -3 /root/files/patches/mono-pr16636-wasm-bugfix-and-update.diff && \ + cd .. && \ export MONO_SOURCE_ROOT=/root/mono && \ export make="make -j" && \ git clone https://github.com/godotengine/godot-mono-builds && \ cd godot-mono-builds && \ git checkout bd129da22b8b9c96f3e8b07af348cc5fb61504bf && \ python3 patch_emscripten.py && \ - git am -3 /root/files/patches/mono-wasm-avoid-aligning-stack-bounds.patch && \ python3 wasm.py configure --target=runtime && \ python3 wasm.py make --target=runtime && \ cd /root/mono && git clean -fdx && NOCONFIGURE=1 ./autogen.sh && \ diff --git a/files/patches/mono-pr16636-wasm-bugfix-and-update.diff b/files/patches/mono-pr16636-wasm-bugfix-and-update.diff new file mode 100644 index 0000000..d368aac --- /dev/null +++ b/files/patches/mono-pr16636-wasm-bugfix-and-update.diff @@ -0,0 +1,144 @@ +diff --git a/mono/utils/mono-threads-wasm.c b/mono/utils/mono-threads-wasm.c +index 687e6c4c93e7..c61d800f4be2 100644 +--- a/mono/utils/mono-threads-wasm.c ++++ b/mono/utils/mono-threads-wasm.c +@@ -145,10 +145,10 @@ mono_threads_platform_yield (void) + void + mono_threads_platform_get_stack_bounds (guint8 **staddr, size_t *stsize) + { ++ int tmp; + #ifdef __EMSCRIPTEN_PTHREADS__ + pthread_attr_t attr; + gint res; +- int tmp; + + *staddr = NULL; + *stsize = (size_t)-1; +@@ -173,13 +173,13 @@ mono_threads_platform_get_stack_bounds (guint8 **staddr, size_t *stsize) + *staddr = (guint8*)wasm_get_stack_base (); + *stsize = wasm_get_stack_size (); + } +- +- g_assert (&tmp > *staddr); +- g_assert (&tmp < (char*)*staddr + *stsize); + #else + *staddr = (guint8*)wasm_get_stack_base (); + *stsize = wasm_get_stack_size (); + #endif ++ ++ g_assert ((guint8*)&tmp > *staddr); ++ g_assert ((guint8*)&tmp < (guint8*)*staddr + *stsize); + } + + gboolean +diff --git a/mono/utils/mono-threads.c b/mono/utils/mono-threads.c +index 546704ff9e8f..bf73e01c7970 100644 +--- a/mono/utils/mono-threads.c ++++ b/mono/utils/mono-threads.c +@@ -1569,8 +1569,10 @@ mono_thread_info_get_stack_bounds (guint8 **staddr, size_t *stsize) + /* Sanity check the result */ + g_assert ((current > *staddr) && (current < *staddr + *stsize)); + ++#ifndef TARGET_WASM + /* When running under emacs, sometimes staddr is not aligned to a page size */ + *staddr = (guint8*)((gssize)*staddr & ~(mono_pagesize () - 1)); ++#endif + } + + gboolean +diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh +index 514d869e3768..e65076cbb827 100755 +--- a/scripts/ci/run-jenkins.sh ++++ b/scripts/ci/run-jenkins.sh +@@ -330,7 +330,7 @@ if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]]; + if [[ ${CI_TAGS} == *'debug'* ]]; then + echo "CONFIGURATION=debug" >> sdks/Make.config + fi +- echo "ENABLE_WASM_THREADS=1" >> sdks/Make.config ++ #echo "ENABLE_WASM_THREADS=1" >> sdks/Make.config + + export aot_test_suites="System.Core" + export mixed_test_suites="System.Core" +@@ -359,7 +359,7 @@ if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]]; + # disable for now until https://github.com/mono/mono/pull/13622 goes in + #${TESTCMD} --label=debugger --timeout=20m $gnumake -C sdks/wasm test-debugger + ${TESTCMD} --label=browser --timeout=20m $gnumake -C sdks/wasm run-browser-tests +- ${TESTCMD} --label=browser-threads --timeout=20m $gnumake -C sdks/wasm run-browser-threads-tests ++ #${TESTCMD} --label=browser-threads --timeout=20m $gnumake -C sdks/wasm run-browser-threads-tests + ${TESTCMD} --label=v8-corlib --timeout=20m $gnumake -C sdks/wasm run-v8-corlib + ${TESTCMD} --label=aot-mini --timeout=20m $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini + ${TESTCMD} --label=build-aot-all --timeout=20m $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all +diff --git a/sdks/builds/emscripten-pr-8457.diff b/sdks/builds/emscripten-pr-8457.diff +index 0f051a914551..360af836be69 100644 +--- a/sdks/builds/emscripten-pr-8457.diff ++++ b/sdks/builds/emscripten-pr-8457.diff +@@ -1,8 +1,6 @@ +-diff --git a/emcc.py b/emcc.py +-index c241f396a..1fcffc993 100755 +---- a/emcc.py +-+++ b/emcc.py +-@@ -844,6 +844,9 @@ There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR P ++--- a/emcc.py 2019-08-13 20:07:34.000000000 -0500 +++++ b/emcc.py 2019-08-27 08:05:39.000000000 -0500 ++@@ -890,6 +890,9 @@ + lib_dirs = [shared.path_from_root('system', 'local', 'lib'), + shared.path_from_root('system', 'lib')] + +@@ -12,7 +10,7 @@ index c241f396a..1fcffc993 100755 + # find input files this a simple heuristic. we should really analyze + # based on a full understanding of gcc params, right now we just assume that + # what is left contains no more |-x OPT| things +-@@ -869,7 +872,7 @@ There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR P ++@@ -915,7 +918,7 @@ + file_suffix = get_file_suffix(arg) + if file_suffix in SOURCE_ENDINGS + BITCODE_ENDINGS + DYNAMICLIB_ENDINGS + ASSEMBLY_ENDINGS + HEADER_ENDINGS or shared.Building.is_ar(arg): # we already removed -o , so all these should be inputs + newargs[i] = '' +@@ -21,16 +19,15 @@ index c241f396a..1fcffc993 100755 + input_files.append((i, arg)) + has_source_inputs = True + elif file_suffix.endswith(HEADER_ENDINGS): +-@@ -938,8 +941,6 @@ There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR P ++@@ -984,7 +987,6 @@ + +- newargs = [a for a in newargs if a is not ''] ++ newargs = [a for a in newargs if a != ''] + +-- # -c means do not link in gcc, and for us, the parallel is to not go all the way to JS, but stop at bitcode + - has_dash_c = '-c' in newargs +- if has_dash_c: +- assert has_source_inputs or has_header_inputs, 'Must have source code or header inputs to use -c' +- target = target_basename + '.o' +-@@ -1666,7 +1667,7 @@ There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR P ++ has_dash_S = '-S' in newargs ++ if has_dash_c or has_dash_S: ++ assert has_source_inputs or has_header_inputs, 'Must have source code or header inputs to use -c or -S' ++@@ -1838,7 +1840,7 @@ + # First, generate LLVM bitcode. For each input file, we get base.o with bitcode + for i, input_file in input_files: + file_ending = get_file_suffix(input_file) +diff --git a/sdks/builds/wasm.mk b/sdks/builds/wasm.mk +index 3e34e1cbaf08..ab7ef306a0b8 100644 +--- a/sdks/builds/wasm.mk ++++ b/sdks/builds/wasm.mk +@@ -1,7 +1,7 @@ + #emcc has lots of bash'isms + SHELL:=/bin/bash + +-EMSCRIPTEN_VERSION=1.38.38 ++EMSCRIPTEN_VERSION=1.38.43 + EMSCRIPTEN_LOCAL_SDK_DIR=$(TOP)/sdks/builds/toolchains/emsdk + + EMSCRIPTEN_SDK_DIR ?= $(EMSCRIPTEN_LOCAL_SDK_DIR) +diff --git a/tools/offsets-tool-py/offsets-tool.py b/tools/offsets-tool-py/offsets-tool.py +index caaf1b96f1e2..10929bfdcf25 100644 +--- a/tools/offsets-tool-py/offsets-tool.py ++++ b/tools/offsets-tool-py/offsets-tool.py +@@ -80,7 +80,7 @@ def require_emscipten_path (args): + + if "wasm" in args.abi: + require_emscipten_path (args) +- self.sys_includes = [args.emscripten_path + "/system/include/libc"] ++ self.sys_includes = [args.emscripten_path + "/system/include/libc", args.emscripten_path + "/system/lib/libc/musl/arch/emscripten"] + self.target = Target ("TARGET_WASM", None, []) + self.target_args += ["-target", args.abi] + diff --git a/files/patches/mono-wasm-avoid-aligning-stack-bounds.patch b/files/patches/mono-wasm-avoid-aligning-stack-bounds.patch deleted file mode 100644 index ba13f26..0000000 --- a/files/patches/mono-wasm-avoid-aligning-stack-bounds.patch +++ /dev/null @@ -1,59 +0,0 @@ -From aad54942583f8658ab6d7e7af5ffe3b76b3f35b3 Mon Sep 17 00:00:00 2001 -From: Zoltan Varga -Date: Tue, 3 Sep 2019 13:27:24 -0400 -Subject: [PATCH] [wasm] Avoid aligning the stack bounds, they are not page - aligned on wasm. - ---- - mono/utils/mono-threads-wasm.c | 8 ++++---- - mono/utils/mono-threads.c | 2 ++ - 2 files changed, 6 insertions(+), 4 deletions(-) - -diff --git a/mono/utils/mono-threads-wasm.c b/mono/utils/mono-threads-wasm.c -index 687e6c4c93e7..c61d800f4be2 100644 ---- a/mono/utils/mono-threads-wasm.c -+++ b/mono/utils/mono-threads-wasm.c -@@ -145,10 +145,10 @@ mono_threads_platform_yield (void) - void - mono_threads_platform_get_stack_bounds (guint8 **staddr, size_t *stsize) - { -+ int tmp; - #ifdef __EMSCRIPTEN_PTHREADS__ - pthread_attr_t attr; - gint res; -- int tmp; - - *staddr = NULL; - *stsize = (size_t)-1; -@@ -173,13 +173,13 @@ mono_threads_platform_get_stack_bounds (guint8 **staddr, size_t *stsize) - *staddr = (guint8*)wasm_get_stack_base (); - *stsize = wasm_get_stack_size (); - } -- -- g_assert (&tmp > *staddr); -- g_assert (&tmp < (char*)*staddr + *stsize); - #else - *staddr = (guint8*)wasm_get_stack_base (); - *stsize = wasm_get_stack_size (); - #endif -+ -+ g_assert ((guint8*)&tmp > *staddr); -+ g_assert ((guint8*)&tmp < (guint8*)*staddr + *stsize); - } - - gboolean -diff --git a/mono/utils/mono-threads.c b/mono/utils/mono-threads.c -index 546704ff9e8f..bf73e01c7970 100644 ---- a/mono/utils/mono-threads.c -+++ b/mono/utils/mono-threads.c -@@ -1569,8 +1569,10 @@ mono_thread_info_get_stack_bounds (guint8 **staddr, size_t *stsize) - /* Sanity check the result */ - g_assert ((current > *staddr) && (current < *staddr + *stsize)); - -+#ifndef TARGET_WASM - /* When running under emacs, sometimes staddr is not aligned to a page size */ - *staddr = (guint8*)((gssize)*staddr & ~(mono_pagesize () - 1)); -+#endif - } - - gboolean