From 7108e06992cae41aaf87eb79ed8ef8a53dc8a1c6 Mon Sep 17 00:00:00 2001 From: angle-autoroll Date: Tue, 8 Jun 2021 09:55:11 +0000 Subject: [PATCH] Roll Chromium from 3bcc8fd7c291 to 6c5859c895f5 (510 revisions) https://chromium.googlesource.com/chromium/src.git/+log/3bcc8fd7c291..6c5859c895f5 If this roll has caused a breakage, revert this CL and stop the roller using the controls here: https://autoroll.skia.org/r/chromium-angle-autoroll Please CC syoussefi@google.com on the revert to ensure that a human is aware of the problem. To report a problem with the AutoRoller itself, please file a bug: https://bugs.chromium.org/p/skia/issues/entry?template=Autoroller+Bug Documentation for the AutoRoller is here: https://skia.googlesource.com/buildbot/+doc/master/autoroll/README.md Changed dependencies * build: https://chromium.googlesource.com/chromium/src/build.git/+log/e353b02625..8870cb4120 * buildtools: https://chromium.googlesource.com/chromium/src/buildtools.git/+log/0b443d31ef..c793cca886 * testing: https://chromium.googlesource.com/chromium/src/testing/+log/a45949a818..a62f8260df * third_party/android_deps: https://chromium.googlesource.com/chromium/src/third_party/android_deps/+log/42884eb2ac..887e8d9009 * third_party/catapult: https://chromium.googlesource.com/catapult.git/+log/{catapult_..d598390f02 * third_party/depot_tools: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+log/a5b6b2f8b7..b508ecd932 * third_party/protobuf: https://chromium.googlesource.com/chromium/src/third_party/protobuf/+log/9523daa51d..82f8803671 * tools/clang: https://chromium.googlesource.com/chromium/src/tools/clang.git/+log/0c64e8349d..09481f56be * tools/mb: https://chromium.googlesource.com/chromium/src/tools/mb/+log/d5d794a9f5..94630dfc19 * tools/memory: https://chromium.googlesource.com/chromium/src/tools/memory/+log/71214b910d..84ad259424 No update to Clang. Manually added tools/android/modularization/convenience/lookup_dep.py from crrev.com/889754 And updated .gn to use python3 in GN build by default Bug: angleproject:6042 Tbr: syoussefi@google.com Change-Id: I00384f4d38d14b54778d334aec9c633f54d619bb Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2946825 Reviewed-by: Jamie Madill Commit-Queue: Yuly Novikov --- .gn | 4 + DEPS | 22 +- .../modularization/convenience/lookup_dep.py | 313 ++++++++++++++++++ 3 files changed, 328 insertions(+), 11 deletions(-) create mode 100755 tools/android/modularization/convenience/lookup_dep.py diff --git a/.gn b/.gn index 991595d8e..b2c4b5451 100644 --- a/.gn +++ b/.gn @@ -13,6 +13,10 @@ import("//dotfile_settings.gni") # Location of the build configuration file. buildconfig = "//build/config/BUILDCONFIG.gn" +# The python interpreter to use by default. On Windows, this will look +# for python3.exe and python3.bat. +script_executable = "python3" + # These are the targets to check headers for by default. The files in targets # matching these patterns (see "gn help label_pattern" for format) will have # their includes checked for proper dependencies when you run either diff --git a/DEPS b/DEPS index 3748942b4..34ccfd754 100644 --- a/DEPS +++ b/DEPS @@ -27,7 +27,7 @@ vars = { 'checkout_android_native_support': 'checkout_android or checkout_chromeos', # Version of Chromium our Chromium-based DEPS are mirrored from. - 'chromium_revision': '3bcc8fd7c291561965c986e8b11b4e6774377eb7', + 'chromium_revision': '6c5859c895f578a3fecf477f3b33e4e1720003c3', # We never want to checkout chromium, # but need a dummy DEPS entry for the autoroller 'dummy_checkout_chromium': False, @@ -72,7 +72,7 @@ vars = { # Three lines of non-changing comments so that # the commit queue can handle CLs rolling catapult # and whatever else without interference from each other. - 'catapult_revision': '8bf7a0b3c8bddeccd9c5b3ca4a58dccfa8797702', + 'catapult_revision': 'd598390f022179e4b4dcdf505c8545650f63d47d', # Three lines of non-changing comments so that # the commit queue can handle CLs rolling luci-go @@ -112,12 +112,12 @@ vars = { deps = { 'build': { - 'url': '{chromium_git}/chromium/src/build.git@e353b0262556aa23adfd23b2c13ebcaec59181a3', + 'url': '{chromium_git}/chromium/src/build.git@8870cb41201b9f4a384188af6d3c110ef083dba7', 'condition': 'not build_with_chromium', }, 'buildtools': { - 'url': '{chromium_git}/chromium/src/buildtools.git@0b443d31efacc92fe4d984c5654a4403318e7e03', + 'url': '{chromium_git}/chromium/src/buildtools.git@c793cca886a1d820d03b3cb440d2f1c26a3381cc', 'condition': 'not build_with_chromium', }, @@ -170,7 +170,7 @@ deps = { }, 'testing': { - 'url': '{chromium_git}/chromium/src/testing@a45949a8188e22cc318a1c78b056369c4cd43624', + 'url': '{chromium_git}/chromium/src/testing@a62f8260df45a43023b00ee1fcc2ad19f8ae1146', 'condition': 'not build_with_chromium', }, @@ -218,7 +218,7 @@ deps = { }, 'third_party/android_deps': { - 'url': '{chromium_git}/chromium/src/third_party/android_deps@42884eb2ac71586d87f25591b3aa899692b33e45', + 'url': '{chromium_git}/chromium/src/third_party/android_deps@887e8d90095085239a43c2aa1c27d2394d933d2b', 'condition': 'checkout_android and not build_with_chromium', }, @@ -320,7 +320,7 @@ deps = { }, 'third_party/depot_tools': { - 'url': '{chromium_git}/chromium/tools/depot_tools.git@a5b6b2f8b75c5c0a14a074be91c7c32ddd278388', + 'url': '{chromium_git}/chromium/tools/depot_tools.git@b508ecd932fd2653b4d3e9bccd80b3b7ac98c36a', 'condition': 'not build_with_chromium', }, @@ -427,7 +427,7 @@ deps = { }, 'third_party/protobuf': { - 'url': '{chromium_git}/chromium/src/third_party/protobuf@9523daa51d97225ecd46ffc30f4a0dc137d8e787', + 'url': '{chromium_git}/chromium/src/third_party/protobuf@82f8803671681950cbc32b4b681fa6ae0bb1febe', 'condition': 'not build_with_chromium', }, @@ -524,7 +524,7 @@ deps = { }, 'tools/clang': { - 'url': '{chromium_git}/chromium/src/tools/clang.git@0c64e8349d636a77ddd7af8ca04b83aa51d3ac5f', + 'url': '{chromium_git}/chromium/src/tools/clang.git@09481f56be7b1bbaf5a466be5d81691902825fcf', 'condition': 'not build_with_chromium', }, @@ -559,7 +559,7 @@ deps = { }, 'tools/mb': { - 'url': '{chromium_git}/chromium/src/tools/mb@d5d794a9f57d437cb6f93221be3dc37fb1716467', + 'url': '{chromium_git}/chromium/src/tools/mb@94630dfc19de559cb3603f0f7a7c9e0ba2068d49', 'condition': 'not build_with_chromium', }, @@ -569,7 +569,7 @@ deps = { }, 'tools/memory': { - 'url': '{chromium_git}/chromium/src/tools/memory@71214b910decfe2e7cfc8b0ffc072a1b97da2d36', + 'url': '{chromium_git}/chromium/src/tools/memory@84ad2594247b5046378cfba24feaaa97b33c603d', 'condition': 'not build_with_chromium', }, diff --git a/tools/android/modularization/convenience/lookup_dep.py b/tools/android/modularization/convenience/lookup_dep.py new file mode 100755 index 000000000..96d18a5c5 --- /dev/null +++ b/tools/android/modularization/convenience/lookup_dep.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +r'''Finds which build target(s) contain a particular Java class. + +This is a utility script for finding out which build target dependency needs to +be added to import a given Java class. + +It is a best-effort script. + +Example: + +Find build target with class FooUtil: + tools/android/modularization/convenience/lookup_dep.py FooUtil +''' + +import argparse +import collections +import dataclasses +import json +import logging +import os +import pathlib +import subprocess +import sys +import zipfile +from typing import Dict, List, Set + +_SRC_DIR = pathlib.Path(__file__).parents[4].resolve() + +sys.path.append(str(_SRC_DIR / 'build' / 'android')) +from pylib import constants + +# Import list_java_targets so that the dependency is found by print_python_deps. +import list_java_targets + + +def main(): + arg_parser = argparse.ArgumentParser( + description='Finds which build target contains a particular Java class.') + + arg_parser.add_argument('-C', '--output-directory', help='Build output directory.') + arg_parser.add_argument('--build', action='store_true', help='Build all .build_config files.') + arg_parser.add_argument('classes', nargs='+', help='Java classes to search for') + arg_parser.add_argument('-v', '--verbose', action='store_true', help='Verbose logging.') + + arguments = arg_parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if arguments.verbose else logging.WARNING, + format='%(asctime)s.%(msecs)03d %(levelname).1s %(message)s', + datefmt='%H:%M:%S') + + if arguments.output_directory: + constants.SetOutputDirectory(arguments.output_directory) + constants.CheckOutputDirectory() + abs_out_dir: pathlib.Path = pathlib.Path(constants.GetOutDirectory()).resolve() + + index = ClassLookupIndex(abs_out_dir, arguments.build) + matches = {c: index.match(c) for c in arguments.classes} + + if not arguments.build: + # Try finding match without building because it is faster. + for class_name, match_list in matches.items(): + if len(match_list) == 0: + arguments.build = True + break + if arguments.build: + index = ClassLookupIndex(abs_out_dir, True) + matches = {c: index.match(c) for c in arguments.classes} + + if not arguments.build: + print('Showing potentially stale results. Run lookup.dep.py with --build ' + '(slower) to build any unbuilt GN targets and get full results.') + print() + + for (class_name, class_entries) in matches.items(): + if not class_entries: + print(f'Could not find build target for class "{class_name}"') + elif len(class_entries) == 1: + class_entry = class_entries[0] + print(f'Class {class_entry.full_class_name} found:') + print(f' "{class_entry.target}"') + else: + print(f'Multiple targets with classes that match "{class_name}":') + print() + for class_entry in class_entries: + print(f' "{class_entry.target}"') + print(f' contains {class_entry.full_class_name}') + print() + + +@dataclasses.dataclass(frozen=True) +class ClassEntry: + """An assignment of a Java class to a build target.""" + full_class_name: str + target: str + + +class ClassLookupIndex: + """A map from full Java class to its build targets. + + A class might be in multiple targets if it's bytecode rewritten.""" + + def __init__(self, abs_build_output_dir: pathlib.Path, should_build: bool): + self._abs_build_output_dir = abs_build_output_dir + self._should_build = should_build + self._class_index = self._index_root() + + def match(self, search_string: str) -> List[ClassEntry]: + """Get class/target entries where the class matches search_string""" + # Priority 1: Exact full matches + if search_string in self._class_index: + return self._entries_for(search_string) + + # Priority 2: Match full class name (any case), if it's a class name + matches = [] + lower_search_string = search_string.lower() + if '.' not in lower_search_string: + for full_class_name in self._class_index: + package_and_class = full_class_name.rsplit('.', 1) + if len(package_and_class) < 2: + continue + class_name = package_and_class[1] + class_lower = class_name.lower() + if class_lower == lower_search_string: + matches.extend(self._entries_for(full_class_name)) + if matches: + return matches + + # Priority 3: Match anything + for full_class_name in self._class_index: + if lower_search_string in full_class_name.lower(): + matches.extend(self._entries_for(full_class_name)) + + return matches + + def _entries_for(self, class_name) -> List[ClassEntry]: + return [ClassEntry(class_name, target) for target in self._class_index.get(class_name)] + + def _index_root(self) -> Dict[str, List[str]]: + """Create the class to target index.""" + logging.debug('Running list_java_targets.py...') + list_java_targets_command = [ + 'build/android/list_java_targets.py', '--gn-labels', '--print-build-config-paths', + f'--output-directory={self._abs_build_output_dir}' + ] + if self._should_build: + list_java_targets_command += ['--build'] + + list_java_targets_run = subprocess.run( + list_java_targets_command, cwd=_SRC_DIR, capture_output=True, text=True, check=True) + logging.debug('... done.') + + # Parse output of list_java_targets.py with mapping of build_target to + # build_config + root_build_targets = list_java_targets_run.stdout.split('\n') + class_index = collections.defaultdict(list) + for target_line in root_build_targets: + # Skip empty lines + if not target_line: + continue + + target_line_parts = target_line.split(': ') + assert len(target_line_parts) == 2, target_line_parts + target, build_config_path = target_line_parts + + if not os.path.exists(build_config_path): + assert not self._should_build + continue + + with open(build_config_path) as build_config_contents: + build_config: Dict = json.load(build_config_contents) + deps_info = build_config['deps_info'] + # Checking the library type here instead of in list_java_targets.py avoids + # reading each .build_config file twice. + if deps_info['type'] != 'java_library': + continue + + target = self._compute_toplevel_target(target) + full_class_names = self._compute_full_class_names_for_build_config(deps_info) + for full_class_name in full_class_names: + class_index[full_class_name].append(target) + + return class_index + + @staticmethod + def _compute_toplevel_target(target: str) -> str: + """Computes top level target from the passed-in sub-target.""" + if target.endswith('_java'): + return target + + # Handle android_aar_prebuilt() sub targets. + index = target.find('_java__subjar') + if index >= 0: + return target[0:index + 5] + index = target.find('_java__classes') + if index >= 0: + return target[0:index + 5] + + return target + + def _compute_full_class_names_for_build_config(self, deps_info: Dict) -> Set[str]: + """Returns set of fully qualified class names for build config.""" + + full_class_names = set() + + # Read the location of the java_sources_file from the build_config + sources_path = deps_info.get('java_sources_file') + if sources_path: + # Read the java_sources_file, indexing the classes found + with open(self._abs_build_output_dir / sources_path) as sources_contents: + for source_line in sources_contents: + source_path = pathlib.Path(source_line.strip()) + java_class = self._parse_full_java_class(source_path) + if java_class: + full_class_names.add(java_class) + + # |unprocessed_jar_path| is set for prebuilt targets. (ex: + # android_aar_prebuilt()) + # |unprocessed_jar_path| might be set but not exist if not all targets have + # been built. + unprocessed_jar_path = deps_info.get('unprocessed_jar_path') + if unprocessed_jar_path: + abs_unprocessed_jar_path = (self._abs_build_output_dir / unprocessed_jar_path) + if abs_unprocessed_jar_path.exists(): + # Normalize path but do not follow symlink if .jar is symlink. + abs_unprocessed_jar_path = ( + abs_unprocessed_jar_path.parent.resolve() / abs_unprocessed_jar_path.name) + + full_class_names.update( + self._extract_full_class_names_from_jar(self._abs_build_output_dir, + abs_unprocessed_jar_path)) + + return full_class_names + + @staticmethod + def _extract_full_class_names_from_jar(abs_build_output_dir: pathlib.Path, + abs_jar_path: pathlib.Path) -> Set[str]: + """Returns set of fully qualified class names in passed-in jar.""" + out = set() + jar_namelist = ClassLookupIndex._read_jar_namelist(abs_build_output_dir, abs_jar_path) + for zip_entry_name in jar_namelist: + if not zip_entry_name.endswith('.class'): + continue + # Remove .class suffix + full_java_class = zip_entry_name[:-6] + + full_java_class = full_java_class.replace('/', '.') + dollar_index = full_java_class.find('$') + if dollar_index >= 0: + full_java_class[0:dollar_index] + + out.add(full_java_class) + return out + + @staticmethod + def _read_jar_namelist(abs_build_output_dir: pathlib.Path, + abs_jar_path: pathlib.Path) -> List[str]: + """Returns list of jar members by name.""" + + # Caching namelist speeds up lookup_dep.py runtime by 1.5s. + cache_path = abs_jar_path.with_suffix(abs_jar_path.suffix + '.namelist_cache') + if (not ClassLookupIndex._is_path_relative_to(abs_jar_path, abs_build_output_dir)): + cache_path = (abs_build_output_dir / 'gen' / cache_path.relative_to(_SRC_DIR)) + if (cache_path.exists() and os.path.getmtime(cache_path) > os.path.getmtime(abs_jar_path)): + with open(cache_path) as f: + return [s.strip() for s in f.readlines()] + + with zipfile.ZipFile(abs_jar_path) as z: + namelist = z.namelist() + + cache_path.parent.mkdir(parents=True, exist_ok=True) + with open(cache_path, 'w') as f: + f.write('\n'.join(namelist)) + + return namelist + + @staticmethod + def _is_path_relative_to(path: pathlib.Path, other: pathlib.Path) -> bool: + # PurePath.is_relative_to() was introduced in Python 3.9 + resolved_path = path.resolve() + resolved_other = other.resolve() + return str(resolved_path).startswith(str(resolved_other)) + + @staticmethod + def _parse_full_java_class(source_path: pathlib.Path) -> str: + """Guess the fully qualified class name from the path to the source file.""" + if source_path.suffix != '.java': + logging.warning(f'"{source_path}" does not have the .java suffix') + return None + + directory_path: pathlib.Path = source_path.parent + package_list_reversed = [] + for part in reversed(directory_path.parts): + if part == 'java': + break + package_list_reversed.append(part) + if part in ('com', 'org'): + break + else: + logging.debug(f'File {source_path} not in a subdir of "org" or "com", ' + 'cannot detect package heuristically.') + return None + + package = '.'.join(reversed(package_list_reversed)) + class_name = source_path.stem + return f'{package}.{class_name}' + + +if __name__ == '__main__': + main()