Add support for building Mesa + Zink.

Uses a conditional gclient variable "angle_mesa". You must set
this to True in your .gclient to download and configure Mesa.

After that you must also set angle_build_mesa=true in your GN
args. We call through to meson in order to setup and build.

Mesa sources and targets are written into an auto-generated GNI
file. We must update this file on the same cadence as Mesa rolls.

Because the version of libdrm we use in our sysroot is older than
Mesa requires, this CL also includes a custom libdrm build. Also
in order to run bison and flex on the old sysroot that Chrome
infra runs with, this CL includes a new version of flex and bison
that run without newer versions of GLIBC.

Bug: angleproject:7689
Change-Id: I1aeeda1e6f3cf1d5f878aeab5209afa7e1fb5324
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3933062
Reviewed-by: Cody Northrop <cnorthrop@google.com>
Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>
Commit-Queue: Jamie Madill <jmadill@chromium.org>

.gitignore

@@ -52,10 +52,12 @@
 /third_party/jdk/extras/java_8
 /third_party/jinja2
 /third_party/jsoncpp
+/third_party/libdrm
 /third_party/libjpeg_turbo
 /third_party/libpng/src
 /third_party/llvm-build
 /third_party/markupsafe
+/third_party/meson
 /third_party/nasm
 /third_party/ninja
 /third_party/OpenCL-Docs/src

.vpython3

@@ -517,3 +517,14 @@ wheel: <
   name: "infra/python/wheels/selenium-py3"
   version: "version:3.14.0"
 >
+
+# Used by meson
+wheel: <
+  name: "infra/python/wheels/markupsafe/${vpython_platform}"
+  version: "version:2.0.1"
+>
+
+wheel: <
+  name: "infra/python/wheels/mako-py3"
+  version: "version:1.2.3"
+>

BUILD.gn

@@ -88,6 +88,9 @@ if (angle_build_all) {
     if (angle_enable_cl) {
       deps += [ "$angle_root/src/libOpenCL:angle_cl" ]
     }
+    if (angle_build_mesa) {
+      deps += [ "$angle_root/third_party/mesa" ]
+    }
   }
 }
 

DEPS

@@ -38,6 +38,9 @@ vars = {
   # build ARC++ support libraries.
   'checkout_android_native_support': 'checkout_android or checkout_chromeos',
 
+  # Check out Mesa and libdrm in ANGLE's third_party folder.
+  'angle_mesa': False,
+
   # Version of Chromium our Chromium-based DEPS are mirrored from.
   'chromium_revision': 'c8d10c0b09aba42a2849e283bfea79db014d2ac5',
   # We never want to checkout chromium,
@@ -416,6 +419,12 @@ deps = {
     'condition': 'checkout_android and not build_with_chromium',
   },
 
+  # We use the upstream/main branch.
+  'third_party/libdrm': {
+    'url': '{chromium_git}/chromiumos/third_party/libdrm@474894ed17a037a464e5bd845a0765a50f647898',
+    'condition': 'angle_mesa',
+  },
+
   # libjpeg_turbo is used by glmark2.
   'third_party/libjpeg_turbo': {
     'url': '{chromium_git}/chromium/deps/libjpeg_turbo.git@ed683925e4897a84b3bffc5c1414c85b97a129a3',
@@ -464,6 +473,18 @@ deps = {
     'condition': 'checkout_android and not build_with_chromium',
   },
 
+  # We use the upstream/main branch.
+  'third_party/mesa/src': {
+    'url': '{chromium_git}/external/github.com/Mesa3D/mesa@0a6aa58acae2a5b27ef783c22e976ec9b0d33ddc',
+    'condition': 'angle_mesa',
+  },
+
+  # We use the upstream/master branch.
+  'third_party/meson': {
+    'url': '{chromium_git}/external/github.com/mesonbuild/meson@9fd5eb605674067ce6f8876dc27e5e116024e8a6',
+    'condition': 'angle_mesa',
+  },
+
   'third_party/nasm': {
     'url': '{chromium_git}/chromium/deps/nasm.git@5fd9246276814b5ebb7e56b821cedeecab9b674d',
     'condition': 'not build_with_chromium',
@@ -4531,6 +4552,14 @@ hooks = [
                 '-d', 'tools/flex-bison/windows/',
     ],
   },
+
+  # Set up an input file for the Mesa setup step.
+  {
+    'name': 'mesa_input',
+    'pattern': '.',
+    'condition': 'angle_mesa',
+    'action': [ 'python3', 'third_party/mesa/mesa_build.py', 'runhook', ],
+  }
 ]
 
 recursedeps = [

gni/angle.gni

@@ -276,6 +276,9 @@ declare_args() {
   # Enable overlay by default when debug layers are enabled.  This is currently only implemented on
   # Vulkan.
   angle_enable_overlay = angle_enable_vulkan && angle_debug_layers_enabled
+
+  # Build Mesa as a third party dependency
+  angle_build_mesa = false
 }
 
 declare_args() {

scripts/code_generation_hashes/ANGLE_shader_preprocessor.json

@@ -12,9 +12,9 @@
   "src/compiler/preprocessor/preprocessor_tab_autogen.cpp":
     "3f15f3cc604a1ac5668711728ddd8430",
   "tools/flex-bison/linux/bison.sha1":
-    "6eba651f1769883db395dd4e485ba838",
+    "dfc9209e0c76eddd9bed0601c6c189e5",
   "tools/flex-bison/linux/flex.sha1":
-    "3e73ddbd801cf3e8ed759c49572c8bf2",
+    "621fc8947f2114812fb7a2c9df71bdcf",
   "tools/flex-bison/windows/bison.exe.sha1":
     "fd6d2461a2c256179faca7f5f142e689",
   "tools/flex-bison/windows/flex.exe.sha1":

scripts/code_generation_hashes/ANGLE_shader_translator.json

@@ -14,9 +14,9 @@
   "src/compiler/translator/glslang_tab_autogen.h":
     "028bdaebf359aefbcdaafae466993ebe",
   "tools/flex-bison/linux/bison.sha1":
-    "6eba651f1769883db395dd4e485ba838",
+    "dfc9209e0c76eddd9bed0601c6c189e5",
   "tools/flex-bison/linux/flex.sha1":
-    "3e73ddbd801cf3e8ed759c49572c8bf2",
+    "621fc8947f2114812fb7a2c9df71bdcf",
   "tools/flex-bison/windows/bison.exe.sha1":
     "fd6d2461a2c256179faca7f5f142e689",
   "tools/flex-bison/windows/flex.exe.sha1":

third_party/mesa/.gitignore

@@ -0,0 +1,3 @@
+libdrm.stamp
+mesa.stamp
+src

third_party/mesa/BUILD.gn

@@ -0,0 +1,58 @@
+# Copyright 2022 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../../gni/angle.gni")
+import("mesa_autogen.gni")
+
+angle_mesa_build_dir = "$root_build_dir/mesa"
+angle_libdrm_build_dir = "$root_build_dir/libdrm"
+
+action("libdrm_setup") {
+  script = "mesa_build.py"
+  args = [
+    "setup",
+    "libdrm",
+    rebase_path(root_build_dir, root_build_dir),
+  ]
+  sources = [ "libdrm.stamp" ]
+  outputs = [ "$angle_libdrm_build_dir/libdrm.stamp" ]
+}
+
+action("libdrm") {
+  script = "mesa_build.py"
+  args = [
+    "compile",
+    rebase_path(angle_libdrm_build_dir, root_build_dir),
+  ]
+  sources = rebase_path(angle_libdrm_sources, ".", "../libdrm")
+  outputs = get_path_info(
+          rebase_path(angle_libdrm_outputs, ".", angle_libdrm_build_dir),
+          "abspath")
+  deps = [ ":libdrm_setup" ]
+}
+
+action("mesa_setup") {
+  script = "mesa_build.py"
+  args = [
+    "setup",
+    "mesa",
+    rebase_path(root_build_dir, root_build_dir),
+  ]
+  sources = [ "mesa.stamp" ]
+  outputs = [ "$angle_mesa_build_dir/mesa.stamp" ]
+  deps = [ ":libdrm" ]
+}
+
+action("mesa") {
+  script = "mesa_build.py"
+  args = [
+    "compile",
+    rebase_path(angle_mesa_build_dir, root_build_dir),
+  ]
+  sources = rebase_path(angle_mesa_sources, ".", "src")
+  outputs =
+      get_path_info(rebase_path(angle_mesa_outputs, ".", angle_mesa_build_dir),
+                    "abspath")
+  deps = [ ":mesa_setup" ]
+}

third_party/mesa/angle_cross.ini

@@ -0,0 +1,7 @@
+[constants]
+here = '$place/'
+
+[binaries]
+python = 'vpython3'
+python3 = 'vpython3'
+pkgconfig = 'pkg-config'

third_party/mesa/mesa_build.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env vpython3
+#
+# Copyright 2022 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# mesa_build.py:
+#   Helper script for building Mesa in an ANGLE checkout.
+
+import argparse
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+ANGLE_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
+DEFAULT_LOG_LEVEL = 'info'
+EXIT_SUCCESS = 0
+EXIT_FAILURE = 1
+MESON = os.path.join(ANGLE_DIR, 'third_party', 'meson', 'meson.py')
+MESA_SOURCE_DIR = os.path.join(ANGLE_DIR, 'third_party', 'mesa', 'src')
+LIBDRM_SOURCE_DIR = os.path.join(ANGLE_DIR, 'third_party', 'libdrm')
+LIBDRM_BUILD_DIR = os.path.join(ANGLE_DIR, 'out', 'libdrm-build')
+MESA_STAMP = 'mesa.stamp'
+LIBDRM_STAMP = 'libdrm.stamp'
+
+MESA_OPTIONS = [
+    '-Dzstd=disabled',
+    '-Dplatforms=x11',
+    '-Dgallium-drivers=zink',
+    '-Dvulkan-drivers=',
+    '-Dvalgrind=disabled',
+]
+LIBDRM_OPTIONS = [
+    '-Dtests=false', '-Dintel=disabled', '-Dnouveau=disabled', '-Damdgpu=disabled',
+    '-Dradeon=disabled', '-Dvmwgfx=disabled'
+]
+
+
+def main(raw_args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-l',
+        '--log',
+        '--log-level',
+        help='Logging level. Default is %s.' % DEFAULT_LOG_LEVEL,
+        default=DEFAULT_LOG_LEVEL)
+
+    subparser = parser.add_subparsers(dest='command')
+
+    mesa = subparser.add_parser('mesa')
+    mesa.add_argument('build_dir', help='Target build directory.')
+    mesa.add_argument('-j', '--jobs', help='Compile jobs.')
+
+    compile_parser = subparser.add_parser('compile')
+    compile_parser.add_argument('-j', '--jobs', help='Compile jobs.')
+    compile_parser.add_argument('build_dir', help='Target build directory.')
+
+    gni = subparser.add_parser('gni')
+    gni.add_argument('output', help='Output location for gni file.')
+    gni.add_argument('mesa_build_dir', help='Target Mesa build directory.')
+    gni.add_argument('libdrm_build_dir', help='Target libdrm build directory.')
+
+    libdrm = subparser.add_parser('libdrm')
+
+    runhook_parser = subparser.add_parser('runhook')
+    runhook_parser.add_argument(
+        '-o', '--output', help='Output location for stamp sha1 file.', default=MESA_STAMP)
+
+    setup_parser = subparser.add_parser('setup')
+    setup_parser.add_argument('target', help='Project: mesa or libdrm.')
+    setup_parser.add_argument('build_dir', help='Target build directory.')
+    setup_parser.add_argument('-w', '--wipe', help='Wipe output directory.', action='store_true')
+
+    args, extra_args = parser.parse_known_args(raw_args)
+
+    logging.basicConfig(level=args.log.upper())
+
+    assert os.path.exists(MESON), 'Could not find meson.py: %s' % MESON
+
+    if args.command == 'mesa':
+        SetupBuild(args.build_dir, MESA_SOURCE_DIR, MESA_OPTIONS)
+        Compile(args, args.build_dir)
+    elif args.command == 'compile':
+        Compile(args, args.build_dir)
+    elif args.command == 'gni':
+        GenerateGni(args)
+    elif args.command == 'libdrm':
+        SetupBuild(args.build_dir, LIBDRM_SOURCE_DIR, LIBDRM_OPTIONS)
+        Compile(args, args.build_dir)
+    elif args.command == 'runhook':
+        RunHook(args)
+    elif args.command == 'setup':
+        LazySetup(args, args.build_dir)
+
+    return EXIT_SUCCESS
+
+
+def SetupBuild(build_dir, source_dir, options, pkg_config_paths=[]):
+    if not os.path.exists(build_dir):
+        os.mkdir(build_dir)
+
+    sysroot_dir = os.path.join(ANGLE_DIR, 'build', 'linux', 'debian_bullseye_amd64-sysroot')
+
+    cflags = ' '.join([
+        '--sysroot=%s' % sysroot_dir,
+        '-Wno-constant-conversion',
+        '-Wno-deprecated-builtins',
+        '-Wno-deprecated-declarations',
+        '-Wno-deprecated-non-prototype',
+        '-Wno-enum-compare-conditional',
+        '-Wno-enum-conversion',
+        '-Wno-implicit-const-int-float-conversion',
+        '-Wno-implicit-function-declaration',
+        '-Wno-initializer-overrides',
+        '-Wno-sometimes-uninitialized',
+        '-Wno-unused-but-set-variable',
+        '-Wno-unused-function',
+    ])
+
+    pkg_config_paths += [
+        '%s/usr/share/pkgconfig' % sysroot_dir,
+        '%s/usr/lib/pkgconfig' % sysroot_dir
+    ]
+
+    extra_env = {
+        'CC': 'clang',
+        'CC_LD': 'lld',
+        'CXX': 'clang++',
+        'CXX_LD': 'lld',
+        'CFLAGS': cflags,
+        'CXXFLAGS': cflags,
+        'PKG_CONFIG_PATH': ':'.join(pkg_config_paths),
+    }
+
+    args = [source_dir, build_dir, '--cross-file',
+            os.path.join(SCRIPT_DIR, 'angle_cross.ini')] + options
+    if os.path.isdir(os.path.join(build_dir, 'meson-info')):
+        args += ['--wipe']
+
+    return Meson(build_dir, 'setup', args, extra_env)
+
+
+def Compile(args, build_dir):
+    return Meson(build_dir, 'compile', ['-C', build_dir])
+
+
+def MakeEnv():
+    clang_dir = os.path.join(ANGLE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin')
+    flex_bison_dir = os.path.join(ANGLE_DIR, 'tools', 'flex-bison')
+
+    # TODO: Windows
+    flex_bison_platform = 'linux'
+    flex_bison_bin_dir = os.path.join(flex_bison_dir, flex_bison_platform)
+
+    depot_tools_dir = os.path.join(ANGLE_DIR, 'third_party', 'depot_tools')
+
+    env = os.environ.copy()
+    paths = [clang_dir, flex_bison_bin_dir, depot_tools_dir, env['PATH']]
+    env['PATH'] = ':'.join(paths)
+    env['BISON_PKGDATADIR'] = os.path.join(flex_bison_dir, 'third_party')
+    return env
+
+
+GNI_TEMPLATE = """\
+# GENERATED FILE - DO NOT EDIT.
+# Generated by {script_name}
+#
+# Copyright 2022 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# {filename}: ANGLE build information for Mesa.
+
+angle_mesa_outputs = [
+{angle_mesa_outputs}]
+
+angle_mesa_sources = [
+{angle_mesa_sources}]
+
+angle_libdrm_outputs = [
+{angle_libdrm_outputs}]
+
+angle_libdrm_sources = [
+{angle_libdrm_sources}]
+"""
+
+
+def GenerateGni(args):
+    mesa_sources_filter = lambda target: target['type'] != 'shared library'
+    mesa_outputs_filter = lambda target: target['type'] == 'shared library'
+    mesa_sources, mesa_outputs = GetMesonSourcesAndOutputs(args.mesa_build_dir,
+                                                           mesa_sources_filter,
+                                                           mesa_outputs_filter)
+
+    libdrm_sources_filter = lambda target: True
+    libdrm_outputs_filter = lambda target: target['type'] == 'shared library'
+    libdrm_sources, libdrm_outputs = GetMesonSourcesAndOutputs(args.libdrm_build_dir,
+                                                               libdrm_sources_filter,
+                                                               libdrm_outputs_filter)
+
+    fmt_list = lambda l, rp: ''.join(
+        sorted(list(set(['  "%s",\n' % os.path.relpath(li, rp) for li in l]))))
+
+    format_args = {
+        'script_name': os.path.basename(__file__),
+        'filename': os.path.basename(args.output),
+        'angle_mesa_outputs': fmt_list(mesa_outputs, args.mesa_build_dir),
+        'angle_mesa_sources': fmt_list(mesa_sources, MESA_SOURCE_DIR),
+        'angle_libdrm_outputs': fmt_list(libdrm_outputs, args.libdrm_build_dir),
+        'angle_libdrm_sources': fmt_list(libdrm_sources, LIBDRM_SOURCE_DIR),
+    }
+
+    gni_text = GNI_TEMPLATE.format(**format_args)
+
+    with open(args.output, 'w') as outf:
+        outf.write(gni_text)
+        outf.close()
+        logging.info('Saved GNI data to %s' % args.output)
+
+
+def GetMesonSourcesAndOutputs(build_dir, sources_filter, output_filter):
+    text_data = Meson(build_dir, 'introspect', [build_dir, '--targets'], stdout=subprocess.PIPE)
+    json_data = json.loads(text_data)
+    outputs = []
+    all_sources = []
+    generated = []
+    for target in json_data:
+        generated += target['filename']
+        if output_filter(target):
+            outputs += target['filename']
+        if sources_filter(target):
+            for target_source in target['target_sources']:
+                all_sources += target_source['sources']
+
+    sources = list(filter(lambda s: (s not in generated), all_sources))
+
+    for source in sources:
+        assert os.path.exists(source), '%s does not exist' % source
+
+    return sources, outputs
+
+
+def Meson(build_dir, command, args, extra_env={}, stdout=None):
+    meson_cmd = [MESON, command] + args
+    env = MakeEnv()
+    for k, v in extra_env.items():
+        env[k] = v
+    # TODO: Remove when crbug.com/1373441 is fixed.
+    env['VPYTHON_DEFAULT_SPEC'] = os.path.join(ANGLE_DIR, '.vpython3')
+    logging.info(' '.join(['%s=%s' % (k, v) for (k, v) in extra_env.items()] + meson_cmd))
+    completed = subprocess.run(meson_cmd, env=env, stdout=stdout)
+    if completed.returncode != EXIT_SUCCESS:
+        logging.fatal('Got error from meson:')
+        with open(os.path.join(build_dir, 'meson-logs', 'meson-log.txt')) as logf:
+            lines = logf.readlines()
+            for line in lines[-10:]:
+                logging.fatal('  %s' % line.strip())
+        sys.exit(EXIT_FAILURE)
+    if stdout:
+        return completed.stdout
+
+
+def RunHook(args):
+    output = os.path.join(SCRIPT_DIR, args.output)
+    Stamp(args, MESA_SOURCE_DIR, output)
+    libdrm_out = os.path.join(SCRIPT_DIR, LIBDRM_STAMP)
+    Stamp(args, LIBDRM_SOURCE_DIR, libdrm_out)
+
+
+def Stamp(args, source_dir, output):
+    commit_id = GrabOutput('git rev-parse HEAD', source_dir)
+    with open(output, 'w') as outf:
+        outf.write(commit_id)
+        outf.close()
+        logging.info('Saved git hash data to %s' % output)
+
+
+def GrabOutput(command, cwd):
+    return subprocess.Popen(
+        command, stdout=subprocess.PIPE, shell=True, cwd=cwd).communicate()[0].strip().decode()
+
+
+def LazySetup(args, build_dir):
+    stamp = args.target + '.stamp'
+    in_stamp = os.path.join(SCRIPT_DIR, stamp)
+    out_stamp = os.path.join(build_dir, args.target, stamp)
+    if not args.wipe and SameStamps(in_stamp, out_stamp):
+        logging.info('%s setup up-to-date.' % args.target)
+        sys.exit(EXIT_SUCCESS)
+
+    if args.target == 'mesa':
+        source_dir = MESA_SOURCE_DIR
+        options = MESA_OPTIONS
+        pkg_config_paths = [os.path.join(build_dir, 'libdrm', 'meson-uninstalled')]
+    else:
+        assert (args.target == 'libdrm')
+        source_dir = LIBDRM_SOURCE_DIR
+        options = LIBDRM_OPTIONS
+        pkg_config_paths = []
+
+    SetupBuild(os.path.join(build_dir, args.target), source_dir, options, pkg_config_paths)
+    shutil.copyfile(in_stamp, out_stamp)
+    logging.info('Finished setup and updated %s.' % out_stamp)
+
+
+def SameStamps(in_stamp, out_stamp):
+    assert os.path.exists(in_stamp)
+    if not os.path.exists(out_stamp):
+        return False
+    in_data = ReadFile(in_stamp)
+    out_data = ReadFile(out_stamp)
+    return in_data == out_data
+
+
+def ReadFile(path):
+    with open(path, 'rt') as inf:
+        all_data = inf.read()
+        inf.close()
+        return all_data
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

tools/flex-bison/linux/bison.sha1

@@ -1 +1 @@
-ecf11699b180eb967b8c4d08011b5704a445adeb
+36625019a2442ac8efc92c32e1d61bd3f450b7ab

tools/flex-bison/linux/flex.sha1

@@ -1 +1 @@
-e830e6c0508d9685876e77304bbe7c4a122c08e8
+3c9694c62a4ad0d1f95b45bb748855c3688c923e