[SCons] Refactor build tools.

Split dependency builders into separate tools, optimize OpenSSL builds.
This commit is contained in:
Fabio Alessandrelli
2022-12-29 19:19:55 +01:00
parent 5348407de9
commit bdd9d50745
8 changed files with 253 additions and 275 deletions

View File

@@ -164,7 +164,7 @@ jobs:
- name: Compile Extension - template_debug - ${{ matrix.platform }} - ${{ matrix.arch }}
run: |
scons target=template_debug generate_bindings=yes
scons target=template_debug
- name: Compile Extension - template_release - ${{ matrix.platform }} - ${{ matrix.arch }}
run: |

2
.gitignore vendored
View File

@@ -5,3 +5,5 @@ bin/*
.sconsign.dblite
*.obj
*.swp
__pycache__/*
*.pyc

View File

@@ -2,8 +2,6 @@
import os, sys, platform, json, subprocess
import builders
def add_sources(sources, dirpath, extension):
for f in os.listdir(dirpath):
@@ -101,27 +99,12 @@ else:
result_path = os.path.join("bin", "extension", "webrtc")
# Dependencies
env.Append(BUILDERS={
"BuildOpenSSL": env.Builder(action=builders.ssl_action, emitter=builders.ssl_emitter),
"BuildLibDataChannel": env.Builder(action=builders.rtc_action, emitter=builders.rtc_emitter),
})
for tool in ["cmake", "common", "ssl", "rtc"]:
env.Tool(tool, toolpath=["tools"])
# SSL
ssl = env.BuildOpenSSL(env.Dir(builders.get_ssl_build_dir(env)), env.Dir(builders.get_ssl_source_dir(env)))
env.Depends(ssl, [env.File("builders.py"), env.File(builders.get_ssl_source_dir(env) + "/VERSION.dat")])
ssl = env.BuildOpenSSL()
env.NoCache(ssl) # Needs refactoring to properly cache generated headers.
env.Prepend(CPPPATH=[builders.get_ssl_include_dir(env)])
env.Prepend(LIBPATH=[builders.get_ssl_build_dir(env)])
env.Append(LIBS=[builders.get_ssl_libs(env)])
# RTC
rtc = env.BuildLibDataChannel(env.Dir(builders.get_rtc_build_dir(env)), [env.Dir(builders.get_rtc_source_dir(env))] + ssl)
env.Depends(rtc, [env.File("builders.py"), env.File(builders.get_rtc_source_dir(env) + "/CMakeLists.txt")])
env.Append(LIBPATH=[builders.get_rtc_build_dir(env)])
env.Append(CPPPATH=[builders.get_rtc_include_dir(env)])
env.Prepend(LIBS=[builders.get_rtc_libs(env)])
rtc = env.BuildLibDataChannel()
# Our includes and sources
env.Append(CPPPATH=["src/"])
@@ -140,15 +123,10 @@ else:
sources.append("src/init_gdnative.cpp")
add_sources(sources, "src/net/", "cpp")
env.Depends(sources, [builders.get_ssl_libs(env), builders.get_rtc_libs(env)])
env.Depends(sources, [ssl, rtc])
# Make the shared library
result_name = "webrtc_native{}{}".format(env["suffix"], env["SHLIBSUFFIX"])
env.Depends(sources, ssl)
if env["platform"] == "windows" and env["use_mingw"]:
env.Append(LIBS=["iphlpapi", "ws2_32", "bcrypt"])
library = env.SharedLibrary(target=os.path.join(result_path, "lib", result_name), source=sources)
Default(library)

View File

@@ -1,247 +0,0 @@
import os
from SCons.Defaults import Mkdir
from SCons.Script import Environment
def get_android_api(env):
return env["android_api_level"] if int(env["android_api_level"]) > 28 else "28"
def get_deps_dir(env):
return env.Dir("#thirdparty").abspath
def get_deps_build_dir(env):
return env.Dir("#bin/thirdparty").abspath + "/{}.{}.dir".format(env["suffix"][1:], "RelWithDebInfo" if env["debug_symbols"] else "Release")
def get_rtc_source_dir(env):
return get_deps_dir(env) + "/libdatachannel"
def get_rtc_build_dir(env):
return get_deps_build_dir(env) + "/libdatachannel"
def get_rtc_include_dir(env):
return get_rtc_source_dir(env) + "/include"
def get_rtc_libs(env):
bdir = get_rtc_build_dir(env)
libs = [
"libdatachannel-static.a",
"deps/libjuice/libjuice-static.a",
"deps/libsrtp/libsrtp2.a",
"deps/usrsctp/usrsctplib/libusrsctp.a"
]
return [env.File(bdir + "/" + lib) for lib in libs]
def get_ssl_source_dir(env):
return get_deps_dir(env) + "/openssl"
def get_ssl_build_dir(env):
return get_deps_build_dir(env) + "/openssl"
def get_ssl_install_dir(env):
return get_ssl_build_dir(env) + "/dest"
def get_ssl_include_dir(env):
return get_ssl_install_dir(env) + "/include"
def get_ssl_libs(env):
bdir = get_ssl_build_dir(env)
return [env.File(bdir + "/" + lib) for lib in ["libssl.a", "libcrypto.a"]]
def ssl_emitter(target, source, env):
return get_ssl_libs(env), source
def ssl_action(target, source, env):
build_dir = get_ssl_build_dir(env)
source_dir = source[0].abspath
ssl_env = env.Clone()
install_dir = get_ssl_install_dir(env)
args = [
"no-ssl3",
"no-weak-ssl-ciphers",
"no-legacy",
"--prefix=%s" % install_dir,
"--openssldir=%s" % install_dir,
]
if env["debug_symbols"]:
args.append("-d")
if env["platform"] != "windows":
args.append("no-shared") # Windows "app" doesn't like static-only builds.
if env["platform"] == "linux":
if env["arch"] == "x86_32":
args.extend(["linux-x86"])
else:
args.extend(["linux-x86_64"])
elif env["platform"] == "android":
args.extend([
{
"arm64": "android-arm64",
"arm32": "android-arm",
"x86_32": "android-x86",
"x86_64": "android-x86_64",
}[env["arch"]],
"-D__ANDROID_API__=%s" % get_android_api(env),
])
# Setup toolchain path.
ssl_env.PrependENVPath("PATH", os.path.dirname(env["CC"]))
ssl_env["ENV"]["ANDROID_NDK_ROOT"] = os.environ.get("ANDROID_NDK_ROOT", "")
elif env["platform"] == "macos":
if env["arch"] == "x86_64":
args.extend(["darwin64-x86_64"])
elif env["arch"] == "arm64":
args.extend(["darwin64-arm64"])
else:
raise ValueError("macOS architecture not supported: %s" % env["arch"])
elif env["platform"] == "ios":
if env["ios_simulator"]:
args.extend(["iossimulator-xcrun"])
elif env["arch"] == "arm32":
args.extend(["ios-xcrun"])
elif env["arch"] == "arm64":
args.extend(["ios64-xcrun"])
else:
raise ValueError("iOS architecture not supported: %s" % env["arch"])
elif env["platform"] == "windows":
if env["arch"] == "x86_32":
if env["use_mingw"]:
args.extend([
"mingw",
"--cross-compile-prefix=i686-w64-mingw32-",
])
else:
args.extend(["VC-WIN32"])
else:
if env["use_mingw"]:
args.extend([
"mingw64",
"--cross-compile-prefix=x86_64-w64-mingw32-",
])
else:
args.extend(["VC-WIN64A"])
jobs = env.GetOption("num_jobs")
ssl_env.Execute([
Mkdir(build_dir),
"cd %s && perl %s/Configure %s" % (build_dir, source_dir, " ".join(['"%s"' % a for a in args])),
"make -C %s -j%s" % (build_dir, jobs),
"make -C %s install_sw install_ssldirs -j%s" % (build_dir, jobs),
]
)
return None
def rtc_emitter(target, source, env):
return get_rtc_libs(env), source
def rtc_action(target, source, env):
build_dir = get_rtc_build_dir(env)
source_dir = source[0].abspath
args = [
"cmake",
"-B",
build_dir,
"-DUSE_NICE=0",
"-DNO_WEBSOCKET=1",
#"-DNO_MEDIA=1", # Windows builds fail without it.
"-DNO_EXAMPLES=1",
"-DNO_TESTS=1",
"-DOPENSSL_USE_STATIC_LIBS=1",
"-DOPENSSL_INCLUDE_DIR=%s" % get_ssl_include_dir(env),
"-DOPENSSL_SSL_LIBRARY=%s/libssl.a" % get_ssl_build_dir(env),
"-DOPENSSL_CRYPTO_LIBRARY=%s/libcrypto.a" % get_ssl_build_dir(env),
"-DCMAKE_BUILD_TYPE=%s" % ("RelWithDebInfo" if env["debug_symbols"] else "Release"),
]
if env["platform"] == "android":
abi = {
"arm64": "arm64-v8a",
"arm32": "armeabi-v7a",
"x86_32": "x86",
"x86_64": "x86_64",
}[env["arch"]]
args.extend([
"-DCMAKE_SYSTEM_NAME=Android",
"-DCMAKE_SYSTEM_VERSION=%s" % get_android_api(env),
"-DCMAKE_ANDROID_ARCH_ABI=%s" % abi,
"-DANDROID_ABI=%s" % abi,
"-DCMAKE_TOOLCHAIN_FILE=%s/build/cmake/android.toolchain.cmake" % os.environ.get("ANDROID_NDK_ROOT", ""),
"-DCMAKE_ANDROID_STL_TYPE=c++_static",
])
elif env["platform"] == "linux":
if env["arch"] == "x86_32":
args.extend([
"-DCMAKE_C_FLAGS=-m32",
"-DCMAKE_CXX_FLAGS=-m32"
])
else:
args.extend([
"-DCMAKE_C_FLAGS=-m64",
"-DCMAKE_CXX_FLAGS=-m64"
])
elif env["platform"] == "macos":
if env["macos_deployment_target"] != "default":
args.extend(["-DCMAKE_OSX_DEPLOYMENT_TARGET=%s" % env["macos_deployment_target"]])
if env["arch"] == "x86_64":
args.extend(["-DCMAKE_OSX_ARCHITECTURES=x86_64"])
elif env["arch"] == "arm64":
args.extend(["-DCMAKE_OSX_ARCHITECTURES=arm64"])
else:
raise ValueError("OSX architecture not supported: %s" % env["arch"])
elif env["platform"] == "ios":
if env["arch"] == "universal":
raise ValueError("iOS architecture not supported: %s" % env["arch"])
args.extend([
"-DCMAKE_SYSTEM_NAME=iOS",
"-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0",
"-DCMAKE_OSX_ARCHITECTURES=%s" % env["arch"],
])
if env["ios_simulator"]:
args.extend(["-DCMAKE_OSX_SYSROOT=iphonesimulator"])
elif env["platform"] == "windows":
args.extend(["-DOPENSSL_ROOT_DIR=%s" % get_ssl_build_dir(env)])
if env["arch"] == "x86_32":
if env["use_mingw"]:
args.extend([
"-G",
"Unix Makefiles",
"-DCMAKE_C_COMPILER=i686-w64-mingw32-gcc",
"-DCMAKE_CXX_COMPILER=i686-w64-mingw32-g++",
"-DCMAKE_SYSTEM_NAME=Windows",
])
else:
if env["use_mingw"]:
args.extend([
"-G",
"Unix Makefiles",
"-DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc",
"-DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++",
"-DCMAKE_SYSTEM_NAME=Windows"
])
args.append(source_dir)
jobs = env.GetOption("num_jobs")
rtc_env = env.Clone()
rtc_env.Execute([
" ".join(['"%s"' % a for a in args]),
"cmake --build %s -t datachannel-static -j%s" % (build_dir, jobs)
]
)
return None

27
tools/cmake.py Normal file
View File

@@ -0,0 +1,27 @@
def exists(env):
return True
def generate(env):
env.AddMethod(cmake_configure, "CMakeConfigure")
env.AddMethod(cmake_build, "CMakeBuild")
def cmake_configure(env, source, target, opt_args):
args = [
"-B",
target,
]
if env["platform"] == "windows" and env["use_mingw"]:
args.extend(["-G", "Unix Makefiles"])
for arg in opt_args:
args.append(arg)
args.append(source)
return env.Execute("cmake " + " ".join(['"%s"' % a for a in args]))
def cmake_build(env, source, target=""):
jobs = env.GetOption("num_jobs")
env = env.Clone()
return env.Execute("cmake --build %s %s -j%s" % (source, "-t %s" % target if target else "", jobs))

8
tools/common.py Normal file
View File

@@ -0,0 +1,8 @@
def exists(env):
return True
def generate(env):
env["DEPS_SOURCE"] = env.Dir("#thirdparty").abspath
env["DEPS_BUILD"] = env.Dir("#bin/thirdparty").abspath + "/{}.{}.dir".format(env["suffix"][1:], "RelWithDebInfo" if env["debug_symbols"] else "Release")

95
tools/rtc.py Normal file
View File

@@ -0,0 +1,95 @@
import os
def rtc_cmake_config(env):
config = {
"USE_NICE": 0,
"NO_WEBSOCKET": 1,
"NO_EXAMPLES": 1,
"NO_TESTS": 1,
"OPENSSL_USE_STATIC_LIBS": 1,
"OPENSSL_INCLUDE_DIR": env["SSL_INCLUDE"],
"OPENSSL_SSL_LIBRARY": env["SSL_LIBRARY"],
"OPENSSL_CRYPTO_LIBRARY": env["SSL_CRYPTO_LIBRARY"],
"OPENSSL_ROOT_DIR": env["SSL_BUILD"],
"CMAKE_BUILD_TYPE": "%s" % ("RelWithDebInfo" if env["debug_symbols"] else "Release"),
}
if "CC" in env:
config["CMAKE_C_COMPILER"] = env["CC"]
if "CXX" in env:
config["CMAKE_CXX_COMPILER"] = env["CXX"]
if env["platform"] == "android":
api = env["android_api_level"] if int(env["android_api_level"]) > 28 else "28"
abi = {
"arm64": "arm64-v8a",
"arm32": "armeabi-v7a",
"x86_32": "x86",
"x86_64": "x86_64",
}[env["arch"]]
config["CMAKE_SYSTEM_NAME"] = "Android"
config["CMAKE_SYSTEM_VERSION"] = api
config["CMAKE_ANDROID_ARCH_ABI"] = abi
config["ANDROID_ABI"] = abi
config["CMAKE_TOOLCHAIN_FILE"] = "%s/build/cmake/android.toolchain.cmake" % os.environ.get("ANDROID_NDK_ROOT", "")
config["CMAKE_ANDROID_STL_TYPE"] = "c++_static"
elif env["platform"] == "linux":
march = "-m32" if env["arch"] == "x86_32" else "-m64"
config["CMAKE_C_FLAGS"] = march
config["CMAKE_CXX_FLAGS"] = march
elif env["platform"] == "macos":
if env["arch"] == "universal":
raise ValueError("OSX architecture not supported: %s" % env["arch"])
config["CMAKE_OSX_ARCHITECTURES"] = env["arch"]
if env["macos_deployment_target"] != "default":
config["CMAKE_OSX_DEPLOYMENT_TARGET"] = env["macos_deployment_target"]
elif env["platform"] == "ios":
if env["arch"] == "universal":
raise ValueError("iOS architecture not supported: %s" % env["arch"])
config["CMAKE_SYSTEM_NAME"] = "iOS"
config["CMAKE_OSX_DEPLOYMENT_TARGET"] = "11.0"
config["CMAKE_OSX_ARCHITECTURES"] = env["arch"]
if env["ios_simulator"]:
config["CMAKE_OSX_SYSROOT"] = "iphonesimulator"
elif env["platform"] == "windows":
config["CMAKE_SYSTEM_NAME"] = "Windows"
return config
def rtc_emitter(target, source, env):
env.Depends(env["RTC_LIBS"], env["SSL_LIBS"])
env.Depends(env["RTC_LIBS"], [env.File(__file__), env.Dir(env["RTC_SOURCE"]), env.File(env["RTC_SOURCE"] + "/CMakeLists.txt")])
return env["RTC_LIBS"], env.Dir(env["RTC_SOURCE"])
def rtc_action(target, source, env):
rtc_env = env.Clone()
build_dir = env["RTC_BUILD"]
source_dir = env["RTC_SOURCE"]
opts = rtc_cmake_config(rtc_env)
rtc_env.CMakeConfigure(source_dir, build_dir, ["-D%s=%s" % it for it in opts.items()])
rtc_env.CMakeBuild(build_dir, "datachannel-static")
return None
def exists(env):
return "CMakeConfigure" in env and "CMakeBuild" in env
def generate(env):
env["RTC_SOURCE"] = env["DEPS_SOURCE"] + "/libdatachannel"
env["RTC_BUILD"] = env["DEPS_BUILD"] + "/libdatachannel"
env["RTC_INCLUDE"] = env["RTC_SOURCE"] + "/include"
env["RTC_LIBS"] = [env.File(env["RTC_BUILD"] + "/" + lib) for lib in [
"libdatachannel-static.a",
"deps/libjuice/libjuice-static.a",
"deps/libsrtp/libsrtp2.a",
"deps/usrsctp/usrsctplib/libusrsctp.a"
]]
env.Append(BUILDERS={
"BuildLibDataChannel": env.Builder(action=rtc_action, emitter=rtc_emitter)
})
env.Append(LIBPATH=[env["RTC_BUILD"]])
env.Append(CPPPATH=[env["RTC_INCLUDE"]])
env.Prepend(LIBS=env["RTC_LIBS"])
if env["platform"] == "windows":
env.AppendUnique(LIBS=["iphlpapi", "bcrypt"])

115
tools/ssl.py Normal file
View File

@@ -0,0 +1,115 @@
import os
from SCons.Defaults import Mkdir
def ssl_emitter(target, source, env):
env.Depends(env["SSL_LIBS"], env.File(__file__))
return env["SSL_LIBS"], [env.Dir(env["SSL_SOURCE"]), env.File(env["SSL_SOURCE"] + "/VERSION.dat")]
def ssl_action(target, source, env):
build_dir = env["SSL_BUILD"]
source_dir = env["SSL_SOURCE"]
install_dir = env["SSL_INSTALL"]
ssl_env = env.Clone()
args = [
"no-ssl2",
"no-ssl3",
"no-weak-ssl-ciphers",
"no-legacy",
"no-shared",
"no-tests",
"--prefix=%s" % install_dir,
"--openssldir=%s" % install_dir,
]
if env["debug_symbols"]:
args.append("-d")
if env["platform"] == "linux":
if env["arch"] == "x86_32":
args.extend(["linux-x86"])
else:
args.extend(["linux-x86_64"])
elif env["platform"] == "android":
api = env["android_api_level"] if int(env["android_api_level"]) > 28 else "28"
args.extend([
{
"arm64": "android-arm64",
"arm32": "android-arm",
"x86_32": "android-x86",
"x86_64": "android-x86_64",
}[env["arch"]],
"-D__ANDROID_API__=%s" % api,
])
# Setup toolchain path.
ssl_env.PrependENVPath("PATH", os.path.dirname(env["CC"]))
ssl_env["ENV"]["ANDROID_NDK_ROOT"] = os.environ.get("ANDROID_NDK_ROOT", "")
elif env["platform"] == "macos":
if env["arch"] == "x86_64":
args.extend(["darwin64-x86_64"])
elif env["arch"] == "arm64":
args.extend(["darwin64-arm64"])
else:
raise ValueError("macOS architecture not supported: %s" % env["arch"])
elif env["platform"] == "ios":
if env["ios_simulator"]:
args.extend(["iossimulator-xcrun"])
elif env["arch"] == "arm32":
args.extend(["ios-xcrun"])
elif env["arch"] == "arm64":
args.extend(["ios64-xcrun"])
else:
raise ValueError("iOS architecture not supported: %s" % env["arch"])
elif env["platform"] == "windows":
if env["arch"] == "x86_32":
if env["use_mingw"]:
args.extend([
"mingw",
"--cross-compile-prefix=i686-w64-mingw32-",
])
else:
args.extend(["VC-WIN32"])
else:
if env["use_mingw"]:
args.extend([
"mingw64",
"--cross-compile-prefix=x86_64-w64-mingw32-",
])
else:
args.extend(["VC-WIN64A"])
jobs = env.GetOption("num_jobs")
ssl_env.Execute([
Mkdir(build_dir),
"cd %s && perl %s/Configure %s" % (build_dir, source_dir, " ".join(['"%s"' % a for a in args])),
"make -C %s -j%s" % (build_dir, jobs),
"make -C %s install_sw install_ssldirs -j%s" % (build_dir, jobs),
]
)
return None
def exists(env):
return True
def generate(env):
env["SSL_SOURCE"] = env["DEPS_SOURCE"] + "/openssl"
env["SSL_BUILD"] = env["DEPS_BUILD"] + "/openssl"
env["SSL_INSTALL"] = env["SSL_BUILD"] + "/dest"
env["SSL_INCLUDE"] = env["SSL_INSTALL"] + "/include"
env["SSL_LIBRARY"] = env.File(env["SSL_BUILD"] + "/libssl.a")
env["SSL_CRYPTO_LIBRARY"] = env.File(env["SSL_BUILD"] + "/libcrypto.a")
env["SSL_LIBS"] = [env["SSL_LIBRARY"], env["SSL_CRYPTO_LIBRARY"]]
env.Append(BUILDERS={
"BuildOpenSSL": env.Builder(action=ssl_action, emitter=ssl_emitter)
})
env.Prepend(CPPPATH=[env["SSL_INCLUDE"]])
env.Prepend(LIBPATH=[env["SSL_BUILD"]])
env.Append(LIBS=env["SSL_LIBS"])
if env["platform"] == "windows":
env.AppendUnique(LIBS=["ws2_32", "gdi32", "advapi32", "crypt32", "user32"])