[SCons] Refactor build tools.

Split dependency builders into separate tools, optimize OpenSSL builds.

.github/workflows/build_release.yml

@@ -164,7 +164,7 @@ jobs:
 
       - name: Compile Extension - template_debug - ${{ matrix.platform }} - ${{ matrix.arch }}
         run: |
-          scons target=template_debug generate_bindings=yes
+          scons target=template_debug
 
       - name: Compile Extension - template_release - ${{ matrix.platform }} - ${{ matrix.arch }}
         run: |

.gitignore

@@ -5,3 +5,5 @@ bin/*
 .sconsign.dblite
 *.obj
 *.swp
+__pycache__/*
+*.pyc

SConstruct

@@ -2,8 +2,6 @@
 
 import os, sys, platform, json, subprocess
 
-import builders
-
 
 def add_sources(sources, dirpath, extension):
     for f in os.listdir(dirpath):
@@ -101,27 +99,12 @@ else:
     result_path = os.path.join("bin", "extension", "webrtc")
 
 # Dependencies
-env.Append(BUILDERS={
+for tool in ["cmake", "common", "ssl", "rtc"]:
-    "BuildOpenSSL": env.Builder(action=builders.ssl_action, emitter=builders.ssl_emitter),
+    env.Tool(tool, toolpath=["tools"])
-    "BuildLibDataChannel": env.Builder(action=builders.rtc_action, emitter=builders.rtc_emitter),
-})
 
-# SSL
+ssl = env.BuildOpenSSL()
-ssl = env.BuildOpenSSL(env.Dir(builders.get_ssl_build_dir(env)), env.Dir(builders.get_ssl_source_dir(env)))
-env.Depends(ssl, [env.File("builders.py"), env.File(builders.get_ssl_source_dir(env) + "/VERSION.dat")])
 env.NoCache(ssl) # Needs refactoring to properly cache generated headers.
-
+rtc = env.BuildLibDataChannel()
-env.Prepend(CPPPATH=[builders.get_ssl_include_dir(env)])
-env.Prepend(LIBPATH=[builders.get_ssl_build_dir(env)])
-env.Append(LIBS=[builders.get_ssl_libs(env)])
-
-# RTC
-rtc = env.BuildLibDataChannel(env.Dir(builders.get_rtc_build_dir(env)), [env.Dir(builders.get_rtc_source_dir(env))] + ssl)
-env.Depends(rtc, [env.File("builders.py"), env.File(builders.get_rtc_source_dir(env) + "/CMakeLists.txt")])
-
-env.Append(LIBPATH=[builders.get_rtc_build_dir(env)])
-env.Append(CPPPATH=[builders.get_rtc_include_dir(env)])
-env.Prepend(LIBS=[builders.get_rtc_libs(env)])
 
 # Our includes and sources
 env.Append(CPPPATH=["src/"])
@@ -140,15 +123,10 @@ else:
     sources.append("src/init_gdnative.cpp")
     add_sources(sources, "src/net/", "cpp")
 
-env.Depends(sources, [builders.get_ssl_libs(env), builders.get_rtc_libs(env)])
+env.Depends(sources, [ssl, rtc])
 
 # Make the shared library
 result_name = "webrtc_native{}{}".format(env["suffix"], env["SHLIBSUFFIX"])
-env.Depends(sources, ssl)
-
-if env["platform"] == "windows" and env["use_mingw"]:
-    env.Append(LIBS=["iphlpapi", "ws2_32", "bcrypt"])
-
 library = env.SharedLibrary(target=os.path.join(result_path, "lib", result_name), source=sources)
 Default(library)
 

builders.py

@@ -1,247 +0,0 @@
-import os
-from SCons.Defaults import Mkdir
-from SCons.Script import Environment
-
-
-def get_android_api(env):
-    return env["android_api_level"] if int(env["android_api_level"]) > 28 else "28"
-
-
-def get_deps_dir(env):
-    return env.Dir("#thirdparty").abspath
-
-
-def get_deps_build_dir(env):
-    return env.Dir("#bin/thirdparty").abspath + "/{}.{}.dir".format(env["suffix"][1:], "RelWithDebInfo" if env["debug_symbols"] else "Release")
-
-
-def get_rtc_source_dir(env):
-    return get_deps_dir(env) + "/libdatachannel"
-
-
-def get_rtc_build_dir(env):
-    return get_deps_build_dir(env) + "/libdatachannel"
-
-
-def get_rtc_include_dir(env):
-    return get_rtc_source_dir(env) + "/include"
-
-
-def get_rtc_libs(env):
-    bdir = get_rtc_build_dir(env)
-    libs = [
-        "libdatachannel-static.a",
-        "deps/libjuice/libjuice-static.a",
-        "deps/libsrtp/libsrtp2.a",
-        "deps/usrsctp/usrsctplib/libusrsctp.a"
-    ]
-    return [env.File(bdir + "/" + lib) for lib in libs]
-
-
-def get_ssl_source_dir(env):
-    return get_deps_dir(env) + "/openssl"
-
-
-def get_ssl_build_dir(env):
-    return get_deps_build_dir(env) + "/openssl"
-
-
-def get_ssl_install_dir(env):
-    return get_ssl_build_dir(env) + "/dest"
-
-
-def get_ssl_include_dir(env):
-    return get_ssl_install_dir(env) + "/include"
-
-
-def get_ssl_libs(env):
-    bdir = get_ssl_build_dir(env)
-    return [env.File(bdir + "/" + lib) for lib in ["libssl.a", "libcrypto.a"]]
-
-
-def ssl_emitter(target, source, env):
-    return get_ssl_libs(env), source
-
-
-def ssl_action(target, source, env):
-    build_dir = get_ssl_build_dir(env)
-    source_dir = source[0].abspath
-
-    ssl_env = env.Clone()
-    install_dir = get_ssl_install_dir(env)
-    args = [
-        "no-ssl3",
-        "no-weak-ssl-ciphers",
-        "no-legacy",
-        "--prefix=%s" % install_dir,
-        "--openssldir=%s" % install_dir,
-    ]
-    if env["debug_symbols"]:
-        args.append("-d")
-
-    if env["platform"] != "windows":
-        args.append("no-shared")  # Windows "app" doesn't like static-only builds.
-    if env["platform"] == "linux":
-        if env["arch"] == "x86_32":
-            args.extend(["linux-x86"])
-        else:
-            args.extend(["linux-x86_64"])
-
-    elif env["platform"] == "android":
-        args.extend([
-            {
-                "arm64": "android-arm64",
-                "arm32": "android-arm",
-                "x86_32": "android-x86",
-                "x86_64": "android-x86_64",
-            }[env["arch"]],
-            "-D__ANDROID_API__=%s" % get_android_api(env),
-        ])
-        # Setup toolchain path.
-        ssl_env.PrependENVPath("PATH", os.path.dirname(env["CC"]))
-        ssl_env["ENV"]["ANDROID_NDK_ROOT"] = os.environ.get("ANDROID_NDK_ROOT", "")
-
-    elif env["platform"] == "macos":
-        if env["arch"] == "x86_64":
-            args.extend(["darwin64-x86_64"])
-        elif env["arch"] == "arm64":
-            args.extend(["darwin64-arm64"])
-        else:
-            raise ValueError("macOS architecture not supported: %s" % env["arch"])
-
-    elif env["platform"] == "ios":
-        if env["ios_simulator"]:
-                args.extend(["iossimulator-xcrun"])
-        elif env["arch"] == "arm32":
-            args.extend(["ios-xcrun"])
-        elif env["arch"] == "arm64":
-            args.extend(["ios64-xcrun"])
-        else:
-            raise ValueError("iOS architecture not supported: %s" % env["arch"])
-
-    elif env["platform"] == "windows":
-        if env["arch"] == "x86_32":
-            if env["use_mingw"]:
-                args.extend([
-                    "mingw",
-                    "--cross-compile-prefix=i686-w64-mingw32-",
-                ])
-            else:
-                args.extend(["VC-WIN32"])
-        else:
-            if env["use_mingw"]:
-                args.extend([
-                    "mingw64",
-                    "--cross-compile-prefix=x86_64-w64-mingw32-",
-                ])
-            else:
-                args.extend(["VC-WIN64A"])
-
-    jobs = env.GetOption("num_jobs")
-    ssl_env.Execute([
-            Mkdir(build_dir),
-            "cd %s && perl %s/Configure %s" % (build_dir, source_dir, " ".join(['"%s"' % a for a in args])),
-            "make -C %s -j%s" % (build_dir, jobs),
-            "make -C %s install_sw install_ssldirs -j%s" % (build_dir, jobs),
-        ]
-    )
-    return None
-
-
-def rtc_emitter(target, source, env):
-    return get_rtc_libs(env), source
-
-
-def rtc_action(target, source, env):
-    build_dir = get_rtc_build_dir(env)
-    source_dir = source[0].abspath
-    args = [
-        "cmake",
-        "-B",
-        build_dir,
-        "-DUSE_NICE=0",
-        "-DNO_WEBSOCKET=1",
-        #"-DNO_MEDIA=1", # Windows builds fail without it.
-        "-DNO_EXAMPLES=1",
-        "-DNO_TESTS=1",
-        "-DOPENSSL_USE_STATIC_LIBS=1",
-        "-DOPENSSL_INCLUDE_DIR=%s" % get_ssl_include_dir(env),
-        "-DOPENSSL_SSL_LIBRARY=%s/libssl.a" % get_ssl_build_dir(env),
-        "-DOPENSSL_CRYPTO_LIBRARY=%s/libcrypto.a" % get_ssl_build_dir(env),
-        "-DCMAKE_BUILD_TYPE=%s" % ("RelWithDebInfo" if env["debug_symbols"] else "Release"),
-    ]
-    if env["platform"] == "android":
-        abi = {
-            "arm64": "arm64-v8a",
-            "arm32": "armeabi-v7a",
-            "x86_32": "x86",
-            "x86_64": "x86_64",
-        }[env["arch"]]
-        args.extend([
-            "-DCMAKE_SYSTEM_NAME=Android",
-            "-DCMAKE_SYSTEM_VERSION=%s" % get_android_api(env),
-            "-DCMAKE_ANDROID_ARCH_ABI=%s" % abi,
-            "-DANDROID_ABI=%s" % abi,
-            "-DCMAKE_TOOLCHAIN_FILE=%s/build/cmake/android.toolchain.cmake" % os.environ.get("ANDROID_NDK_ROOT", ""),
-            "-DCMAKE_ANDROID_STL_TYPE=c++_static",
-        ])
-    elif env["platform"] == "linux":
-        if env["arch"] == "x86_32":
-            args.extend([
-                "-DCMAKE_C_FLAGS=-m32",
-                "-DCMAKE_CXX_FLAGS=-m32"
-            ])
-        else:
-            args.extend([
-                "-DCMAKE_C_FLAGS=-m64",
-                "-DCMAKE_CXX_FLAGS=-m64"
-            ])
-    elif env["platform"] == "macos":
-        if env["macos_deployment_target"] != "default":
-            args.extend(["-DCMAKE_OSX_DEPLOYMENT_TARGET=%s" % env["macos_deployment_target"]])
-        if env["arch"] == "x86_64":
-            args.extend(["-DCMAKE_OSX_ARCHITECTURES=x86_64"])
-        elif env["arch"] == "arm64":
-            args.extend(["-DCMAKE_OSX_ARCHITECTURES=arm64"])
-        else:
-            raise ValueError("OSX architecture not supported: %s" % env["arch"])
-    elif env["platform"] == "ios":
-        if env["arch"] == "universal":
-            raise ValueError("iOS architecture not supported: %s" % env["arch"])
-        args.extend([
-            "-DCMAKE_SYSTEM_NAME=iOS",
-            "-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0",
-            "-DCMAKE_OSX_ARCHITECTURES=%s" % env["arch"],
-        ])
-        if env["ios_simulator"]:
-                args.extend(["-DCMAKE_OSX_SYSROOT=iphonesimulator"])
-    elif env["platform"] == "windows":
-        args.extend(["-DOPENSSL_ROOT_DIR=%s" % get_ssl_build_dir(env)])
-        if env["arch"] == "x86_32":
-            if env["use_mingw"]:
-                args.extend([
-                    "-G",
-                    "Unix Makefiles",
-                    "-DCMAKE_C_COMPILER=i686-w64-mingw32-gcc",
-                    "-DCMAKE_CXX_COMPILER=i686-w64-mingw32-g++",
-                    "-DCMAKE_SYSTEM_NAME=Windows",
-                ])
-        else:
-            if env["use_mingw"]:
-                args.extend([
-                    "-G",
-                    "Unix Makefiles",
-                    "-DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc",
-                    "-DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++",
-                    "-DCMAKE_SYSTEM_NAME=Windows"
-                ])
-
-    args.append(source_dir)
-    jobs = env.GetOption("num_jobs")
-    rtc_env = env.Clone()
-    rtc_env.Execute([
-            " ".join(['"%s"' % a for a in args]),
-            "cmake --build %s -t datachannel-static -j%s" % (build_dir, jobs)
-        ]
-    )
-    return None

tools/cmake.py

@@ -0,0 +1,27 @@
+
+def exists(env):
+    return True
+
+
+def generate(env):
+    env.AddMethod(cmake_configure, "CMakeConfigure")
+    env.AddMethod(cmake_build, "CMakeBuild")
+
+
+def cmake_configure(env, source, target, opt_args):
+    args = [
+        "-B",
+        target,
+    ]
+    if env["platform"] == "windows" and env["use_mingw"]:
+        args.extend(["-G", "Unix Makefiles"])
+    for arg in opt_args:
+        args.append(arg)
+    args.append(source)
+    return env.Execute("cmake " + " ".join(['"%s"' % a for a in args]))
+
+
+def cmake_build(env, source, target=""):
+    jobs = env.GetOption("num_jobs")
+    env = env.Clone()
+    return env.Execute("cmake --build %s %s -j%s" % (source, "-t %s" % target if target else "", jobs))

tools/common.py

@@ -0,0 +1,8 @@
+
+def exists(env):
+    return True
+
+
+def generate(env):
+    env["DEPS_SOURCE"] = env.Dir("#thirdparty").abspath
+    env["DEPS_BUILD"] = env.Dir("#bin/thirdparty").abspath + "/{}.{}.dir".format(env["suffix"][1:], "RelWithDebInfo" if env["debug_symbols"] else "Release")

tools/rtc.py

@@ -0,0 +1,95 @@
+import os
+
+def rtc_cmake_config(env):
+    config = {
+        "USE_NICE": 0,
+        "NO_WEBSOCKET": 1,
+        "NO_EXAMPLES": 1,
+        "NO_TESTS": 1,
+        "OPENSSL_USE_STATIC_LIBS": 1,
+        "OPENSSL_INCLUDE_DIR": env["SSL_INCLUDE"],
+        "OPENSSL_SSL_LIBRARY": env["SSL_LIBRARY"],
+        "OPENSSL_CRYPTO_LIBRARY": env["SSL_CRYPTO_LIBRARY"],
+        "OPENSSL_ROOT_DIR": env["SSL_BUILD"],
+        "CMAKE_BUILD_TYPE": "%s" % ("RelWithDebInfo" if env["debug_symbols"] else "Release"),
+    }
+    if "CC" in env:
+        config["CMAKE_C_COMPILER"] = env["CC"]
+    if "CXX" in env:
+        config["CMAKE_CXX_COMPILER"] = env["CXX"]
+
+    if env["platform"] == "android":
+        api = env["android_api_level"] if int(env["android_api_level"]) > 28 else "28"
+        abi = {
+            "arm64": "arm64-v8a",
+            "arm32": "armeabi-v7a",
+            "x86_32": "x86",
+            "x86_64": "x86_64",
+        }[env["arch"]]
+        config["CMAKE_SYSTEM_NAME"] = "Android"
+        config["CMAKE_SYSTEM_VERSION"] = api
+        config["CMAKE_ANDROID_ARCH_ABI"] = abi
+        config["ANDROID_ABI"] = abi
+        config["CMAKE_TOOLCHAIN_FILE"] = "%s/build/cmake/android.toolchain.cmake" % os.environ.get("ANDROID_NDK_ROOT", "")
+        config["CMAKE_ANDROID_STL_TYPE"] = "c++_static"
+    elif env["platform"] == "linux":
+        march = "-m32" if env["arch"] == "x86_32" else "-m64"
+        config["CMAKE_C_FLAGS"] = march
+        config["CMAKE_CXX_FLAGS"] = march
+    elif env["platform"] == "macos":
+        if env["arch"] == "universal":
+            raise ValueError("OSX architecture not supported: %s" % env["arch"])
+        config["CMAKE_OSX_ARCHITECTURES"] = env["arch"]
+        if env["macos_deployment_target"] != "default":
+            config["CMAKE_OSX_DEPLOYMENT_TARGET"] = env["macos_deployment_target"]
+    elif env["platform"] == "ios":
+        if env["arch"] == "universal":
+            raise ValueError("iOS architecture not supported: %s" % env["arch"])
+        config["CMAKE_SYSTEM_NAME"] = "iOS"
+        config["CMAKE_OSX_DEPLOYMENT_TARGET"] = "11.0"
+        config["CMAKE_OSX_ARCHITECTURES"] = env["arch"]
+        if env["ios_simulator"]:
+                config["CMAKE_OSX_SYSROOT"] = "iphonesimulator"
+    elif env["platform"] == "windows":
+        config["CMAKE_SYSTEM_NAME"] = "Windows"
+    return config
+
+
+def rtc_emitter(target, source, env):
+    env.Depends(env["RTC_LIBS"], env["SSL_LIBS"])
+    env.Depends(env["RTC_LIBS"], [env.File(__file__), env.Dir(env["RTC_SOURCE"]), env.File(env["RTC_SOURCE"] + "/CMakeLists.txt")])
+    return env["RTC_LIBS"], env.Dir(env["RTC_SOURCE"])
+
+
+def rtc_action(target, source, env):
+    rtc_env = env.Clone()
+    build_dir = env["RTC_BUILD"]
+    source_dir = env["RTC_SOURCE"]
+    opts = rtc_cmake_config(rtc_env)
+    rtc_env.CMakeConfigure(source_dir, build_dir, ["-D%s=%s" % it for it in opts.items()])
+    rtc_env.CMakeBuild(build_dir, "datachannel-static")
+    return None
+
+
+def exists(env):
+    return "CMakeConfigure" in env and "CMakeBuild" in env
+
+
+def generate(env):
+    env["RTC_SOURCE"] = env["DEPS_SOURCE"] + "/libdatachannel"
+    env["RTC_BUILD"] = env["DEPS_BUILD"] + "/libdatachannel"
+    env["RTC_INCLUDE"] = env["RTC_SOURCE"] + "/include"
+    env["RTC_LIBS"] = [env.File(env["RTC_BUILD"] + "/" + lib) for lib in [
+        "libdatachannel-static.a",
+        "deps/libjuice/libjuice-static.a",
+        "deps/libsrtp/libsrtp2.a",
+        "deps/usrsctp/usrsctplib/libusrsctp.a"
+    ]]
+    env.Append(BUILDERS={
+        "BuildLibDataChannel": env.Builder(action=rtc_action, emitter=rtc_emitter)
+    })
+    env.Append(LIBPATH=[env["RTC_BUILD"]])
+    env.Append(CPPPATH=[env["RTC_INCLUDE"]])
+    env.Prepend(LIBS=env["RTC_LIBS"])
+    if env["platform"] == "windows":
+        env.AppendUnique(LIBS=["iphlpapi", "bcrypt"])

tools/ssl.py

@@ -0,0 +1,115 @@
+import os
+from SCons.Defaults import Mkdir
+
+def ssl_emitter(target, source, env):
+    env.Depends(env["SSL_LIBS"], env.File(__file__))
+    return env["SSL_LIBS"], [env.Dir(env["SSL_SOURCE"]), env.File(env["SSL_SOURCE"] + "/VERSION.dat")]
+
+
+def ssl_action(target, source, env):
+    build_dir = env["SSL_BUILD"]
+    source_dir = env["SSL_SOURCE"]
+    install_dir = env["SSL_INSTALL"]
+
+    ssl_env = env.Clone()
+    args = [
+        "no-ssl2",
+        "no-ssl3",
+        "no-weak-ssl-ciphers",
+        "no-legacy",
+        "no-shared",
+        "no-tests",
+        "--prefix=%s" % install_dir,
+        "--openssldir=%s" % install_dir,
+    ]
+    if env["debug_symbols"]:
+        args.append("-d")
+
+    if env["platform"] == "linux":
+        if env["arch"] == "x86_32":
+            args.extend(["linux-x86"])
+        else:
+            args.extend(["linux-x86_64"])
+
+    elif env["platform"] == "android":
+        api = env["android_api_level"] if int(env["android_api_level"]) > 28 else "28"
+        args.extend([
+            {
+                "arm64": "android-arm64",
+                "arm32": "android-arm",
+                "x86_32": "android-x86",
+                "x86_64": "android-x86_64",
+            }[env["arch"]],
+            "-D__ANDROID_API__=%s" % api,
+        ])
+        # Setup toolchain path.
+        ssl_env.PrependENVPath("PATH", os.path.dirname(env["CC"]))
+        ssl_env["ENV"]["ANDROID_NDK_ROOT"] = os.environ.get("ANDROID_NDK_ROOT", "")
+
+    elif env["platform"] == "macos":
+        if env["arch"] == "x86_64":
+            args.extend(["darwin64-x86_64"])
+        elif env["arch"] == "arm64":
+            args.extend(["darwin64-arm64"])
+        else:
+            raise ValueError("macOS architecture not supported: %s" % env["arch"])
+
+    elif env["platform"] == "ios":
+        if env["ios_simulator"]:
+                args.extend(["iossimulator-xcrun"])
+        elif env["arch"] == "arm32":
+            args.extend(["ios-xcrun"])
+        elif env["arch"] == "arm64":
+            args.extend(["ios64-xcrun"])
+        else:
+            raise ValueError("iOS architecture not supported: %s" % env["arch"])
+
+    elif env["platform"] == "windows":
+        if env["arch"] == "x86_32":
+            if env["use_mingw"]:
+                args.extend([
+                    "mingw",
+                    "--cross-compile-prefix=i686-w64-mingw32-",
+                ])
+            else:
+                args.extend(["VC-WIN32"])
+        else:
+            if env["use_mingw"]:
+                args.extend([
+                    "mingw64",
+                    "--cross-compile-prefix=x86_64-w64-mingw32-",
+                ])
+            else:
+                args.extend(["VC-WIN64A"])
+
+    jobs = env.GetOption("num_jobs")
+    ssl_env.Execute([
+            Mkdir(build_dir),
+            "cd %s && perl %s/Configure %s" % (build_dir, source_dir, " ".join(['"%s"' % a for a in args])),
+            "make -C %s -j%s" % (build_dir, jobs),
+            "make -C %s install_sw install_ssldirs -j%s" % (build_dir, jobs),
+        ]
+    )
+    return None
+
+
+def exists(env):
+    return True
+
+
+def generate(env):
+    env["SSL_SOURCE"] = env["DEPS_SOURCE"] + "/openssl"
+    env["SSL_BUILD"] = env["DEPS_BUILD"] + "/openssl"
+    env["SSL_INSTALL"] = env["SSL_BUILD"] + "/dest"
+    env["SSL_INCLUDE"] = env["SSL_INSTALL"] + "/include"
+    env["SSL_LIBRARY"] = env.File(env["SSL_BUILD"] + "/libssl.a")
+    env["SSL_CRYPTO_LIBRARY"] = env.File(env["SSL_BUILD"] + "/libcrypto.a")
+    env["SSL_LIBS"] = [env["SSL_LIBRARY"], env["SSL_CRYPTO_LIBRARY"]]
+    env.Append(BUILDERS={
+        "BuildOpenSSL": env.Builder(action=ssl_action, emitter=ssl_emitter)
+    })
+    env.Prepend(CPPPATH=[env["SSL_INCLUDE"]])
+    env.Prepend(LIBPATH=[env["SSL_BUILD"]])
+    env.Append(LIBS=env["SSL_LIBS"])
+    if env["platform"] == "windows":
+        env.AppendUnique(LIBS=["ws2_32", "gdi32", "advapi32", "crypt32", "user32"])