diff --git a/tools/gn2bp/compat.py b/tools/gn2bp/compat.py new file mode 100755 index 0000000000..db587a2190 --- /dev/null +++ b/tools/gn2bp/compat.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from shlex import quote +except ImportError: + from pipes import quote + +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + +try: + xrange = xrange +except NameError: + xrange = range + +try: + basestring = basestring +except NameError: + basestring = str + + +def itervalues(o): + try: + return o.itervalues() + except AttributeError: + return o.values() + + +def iteritems(o): + try: + return o.iteritems() + except AttributeError: + return o.items() diff --git a/tools/gn2bp/gen_android_bp b/tools/gn2bp/gen_android_bp new file mode 100755 index 0000000000..4cacf7cea9 --- /dev/null +++ b/tools/gn2bp/gen_android_bp @@ -0,0 +1,1129 @@ +#!/usr/bin/env python3 +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This tool translates a collection of BUILD.gn files into a mostly equivalent +# Android.bp file for the Android Soong build system. The input to the tool is a +# JSON description of the GN build definition generated with the following +# command: +# +# gn desc out --format=json --all-toolchains "//*" > desc.json +# +# The tool is then given a list of GN labels for which to generate Android.bp +# build rules. The dependencies for the GN labels are squashed to the generated +# Android.bp target, except for actions which get their own genrule. Some +# libraries are also mapped to their Android equivalents -- see |builtin_deps|. + +import argparse +import collections +import json +import os +import re +import sys + +import gn_utils + +from compat import itervalues + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Arguments for the GN output directory. +gn_args = ' '.join([ + 'is_debug=false', + 'is_perfetto_build_generator=true', + 'perfetto_build_with_android=true', + 'target_cpu="arm"', + 'target_os="android"', +]) + +# Default targets to translate to the blueprint file. +default_targets = [ + '//:libperfetto_client_experimental', + '//:libperfetto', + '//:perfetto_integrationtests', + '//:perfetto_unittests', + '//protos/perfetto/trace:perfetto_trace_protos', + '//src/android_internal:libperfetto_android_internal', + '//src/perfetto_cmd:perfetto', + '//src/perfetto_cmd:trigger_perfetto', + '//src/profiling/memory:heapprofd_client', + '//src/profiling/memory:heapprofd_client_api', + '//src/profiling/memory:heapprofd_api_noop', + '//src/profiling/memory:heapprofd', + '//src/profiling/memory:heapprofd_standalone_client', + '//src/profiling/perf:traced_perf', + '//src/traced/probes:traced_probes', + '//src/traced/service:traced', + '//src/trace_processor:trace_processor_shell', + '//test/cts:perfetto_cts_deps', + '//test/cts:perfetto_cts_jni_deps', + '//test:perfetto_gtest_logcat_printer', + '//test/vts:perfetto_vts_deps', +] + +# Host targets +ipc_plugin = '//src/ipc/protoc_plugin:ipc_plugin(%s)' % gn_utils.HOST_TOOLCHAIN +protozero_plugin = '//src/protozero/protoc_plugin:protozero_plugin(%s)' % ( + gn_utils.HOST_TOOLCHAIN) +cppgen_plugin = '//src/protozero/protoc_plugin:cppgen_plugin(%s)' % ( + gn_utils.HOST_TOOLCHAIN) + +default_targets += [ + '//src/traceconv:traceconv(%s)' % gn_utils.HOST_TOOLCHAIN, + protozero_plugin, + ipc_plugin, +] + +# Defines a custom init_rc argument to be applied to the corresponding output +# blueprint target. +target_initrc = { + '//src/traced/service:traced': {'perfetto.rc'}, + '//src/profiling/memory:heapprofd': {'heapprofd.rc'}, + '//src/profiling/perf:traced_perf': {'traced_perf.rc'}, +} + +target_host_supported = [ + '//:libperfetto', + '//:libperfetto_client_experimental', + '//protos/perfetto/trace:perfetto_trace_protos', + '//src/trace_processor:demangle', + '//src/trace_processor:trace_processor_shell', +] + +target_vendor_available = [ + '//:libperfetto_client_experimental', +] + +# Proto target groups which will be made public. +proto_groups = { + 'trace': [ + '//protos/perfetto/trace:non_minimal_source_set', + '//protos/perfetto/trace:minimal_source_set' + ], +} + +# All module names are prefixed with this string to avoid collisions. +module_prefix = 'perfetto_' + +# Shared libraries which are directly translated to Android system equivalents. +shared_library_allowlist = [ + 'android', + 'android.hardware.atrace@1.0', + 'android.hardware.health@2.0', + 'android.hardware.health-V1-ndk', + 'android.hardware.power.stats@1.0', + "android.hardware.power.stats-V1-cpp", + 'base', + 'binder', + 'binder_ndk', + 'cutils', + 'hidlbase', + 'hidltransport', + 'hwbinder', + 'incident', + 'log', + 'services', + 'statssocket', + "tracingproxy", + 'utils', +] + +# Static libraries which are directly translated to Android system equivalents. +static_library_allowlist = [ + 'statslog_perfetto', +] + +# Name of the module which settings such as compiler flags for all other +# modules. +defaults_module = module_prefix + 'defaults' + +# Location of the project in the Android source tree. +tree_path = 'external/perfetto' + +# Path for the protobuf sources in the standalone build. +buildtools_protobuf_src = '//buildtools/protobuf/src' + +# Location of the protobuf src dir in the Android source tree. +android_protobuf_src = 'external/protobuf/src' + +# Compiler flags which are passed through to the blueprint. +cflag_allowlist = r'^-DPERFETTO.*$' + +# Compiler defines which are passed through to the blueprint. +define_allowlist = r'^(GOOGLE_PROTO.*)|(ZLIB_.*)|(USE_MMAP)|(HAVE_HIDDEN)$' + +# The directory where the generated perfetto_build_flags.h will be copied into. +buildflags_dir = 'include/perfetto/base/build_configs/android_tree' + + +def enumerate_data_deps(): + with open(os.path.join(ROOT_DIR, 'tools', 'test_data.txt')) as f: + lines = f.readlines() + for line in (line.strip() for line in lines if not line.startswith('#')): + assert os.path.exists(line), 'file %s should exist' % line + if line.startswith('test/data/'): + # Skip test data files that require GCS. They are only for benchmarks. + # We don't run benchmarks in the android tree. + continue + if line.endswith('/.'): + yield line[:-1] + '**/*' + else: + yield line + + +# Additional arguments to apply to Android.bp rules. +additional_args = { + 'heapprofd_client_api': [ + ('static_libs', {'libasync_safe'}), + # heapprofd_client_api MUST NOT have global constructors. Because it + # is loaded in an __attribute__((constructor)) of libc, we cannot + # guarantee that the global constructors get run before it is used. + ('cflags', {'-Wglobal-constructors', '-Werror=global-constructors'}), + ('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'), + ('stubs', { + 'versions': ['S'], + 'symbol_file': 'src/profiling/memory/heapprofd_client_api.map.txt', + }), + ('export_include_dirs', {'src/profiling/memory/include'}), + ], + 'heapprofd_api_noop': [ + ('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'), + ('stubs', { + 'versions': ['S'], + 'symbol_file': 'src/profiling/memory/heapprofd_client_api.map.txt', + }), + ('export_include_dirs', {'src/profiling/memory/include'}), + ], + 'heapprofd_client': [ + ('include_dirs', {'bionic/libc'}), + ('static_libs', {'libasync_safe'}), + ], + 'heapprofd_standalone_client': [ + ('static_libs', {'libasync_safe'}), + ('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'), + ('export_include_dirs', {'src/profiling/memory/include'}), + ('stl', 'libc++_static'), + ], +# 'perfetto_unittests': [ +# ('data', set(enumerate_data_deps())), +# ('include_dirs', {'bionic/libc/kernel'}), +# ], + 'perfetto_integrationtests': [ + ('test_suites', {'general-tests'}), + ('test_config', 'PerfettoIntegrationTests.xml'), + ], + 'traced_probes': [('required', { + 'libperfetto_android_internal', 'trigger_perfetto', 'traced_perf', + 'mm_events' + }),], + 'libperfetto_android_internal': [('static_libs', {'libhealthhalutils'}),], + 'trace_processor_shell': [ + ('strip', { + 'all': True + }), + ('host', { + 'stl': 'libc++_static', + 'dist': { + 'targets': ['sdk_repo'] + }, + }), + ], + 'libperfetto_client_experimental': [ + ('apex_available', { + '//apex_available:platform', 'com.android.art', + 'com.android.art.debug' + }), + ('min_sdk_version', 'S'), + ('shared_libs', {'liblog'}), + ('export_include_dirs', {'include', buildflags_dir}), + ], + 'perfetto_trace_protos': [ + ('apex_available', { + '//apex_available:platform', 'com.android.art', + 'com.android.art.debug' + }), + ('min_sdk_version', 'S'), + ], + 'libperfetto': [('export_include_dirs', {'include', buildflags_dir}),], +} + + +def enable_gtest_and_gmock(module): + module.static_libs.add('libgmock') + module.static_libs.add('libgtest') + if module.name != 'perfetto_gtest_logcat_printer': + module.whole_static_libs.add('perfetto_gtest_logcat_printer') + + +def enable_protobuf_full(module): + if module.type == 'cc_binary_host': + module.static_libs.add('libprotobuf-cpp-full') + elif module.host_supported: + module.host.static_libs.add('libprotobuf-cpp-full') + module.android.shared_libs.add('libprotobuf-cpp-full') + else: + module.shared_libs.add('libprotobuf-cpp-full') + + +def enable_protobuf_lite(module): + module.shared_libs.add('libprotobuf-cpp-lite') + + +def enable_protoc_lib(module): + if module.type == 'cc_binary_host': + module.static_libs.add('libprotoc') + else: + module.shared_libs.add('libprotoc') + + +def enable_libunwindstack(module): + if module.name != 'heapprofd_standalone_client': + module.shared_libs.add('libunwindstack') + module.shared_libs.add('libprocinfo') + module.shared_libs.add('libbase') + else: + module.static_libs.add('libunwindstack') + module.static_libs.add('libprocinfo') + module.static_libs.add('libbase') + module.static_libs.add('liblzma') + module.static_libs.add('libdexfile_support') + module.runtime_libs.add('libdexfile') # libdexfile_support dependency + + +def enable_libunwind(module): + # libunwind is disabled on Darwin so we cannot depend on it. + pass + + +def enable_sqlite(module): + if module.type == 'cc_binary_host': + module.static_libs.add('libsqlite') + module.static_libs.add('sqlite_ext_percentile') + elif module.host_supported: + # Copy what the sqlite3 command line tool does. + module.android.shared_libs.add('libsqlite') + module.android.shared_libs.add('libicu') + module.android.shared_libs.add('liblog') + module.android.shared_libs.add('libutils') + module.android.static_libs.add('sqlite_ext_percentile') + module.host.static_libs.add('libsqlite') + module.host.static_libs.add('sqlite_ext_percentile') + else: + module.shared_libs.add('libsqlite') + module.shared_libs.add('libicu') + module.shared_libs.add('liblog') + module.shared_libs.add('libutils') + module.static_libs.add('sqlite_ext_percentile') + + +def enable_zlib(module): + if module.type == 'cc_binary_host': + module.static_libs.add('libz') + elif module.host_supported: + module.android.shared_libs.add('libz') + module.host.static_libs.add('libz') + else: + module.shared_libs.add('libz') + + +def enable_uapi_headers(module): + module.include_dirs.add('bionic/libc/kernel') + + +def enable_bionic_libc_platform_headers_on_android(module): + module.header_libs.add('bionic_libc_platform_headers') + + +# Android equivalents for third-party libraries that the upstream project +# depends on. +builtin_deps = { + '//gn:default_deps': + lambda x: None, + '//gn:gtest_main': + lambda x: None, + '//gn:protoc': + lambda x: None, + '//gn:gtest_and_gmock': + enable_gtest_and_gmock, + '//gn:libunwind': + enable_libunwind, + '//gn:protobuf_full': + enable_protobuf_full, + '//gn:protobuf_lite': + enable_protobuf_lite, + '//gn:protoc_lib': + enable_protoc_lib, + '//gn:libunwindstack': + enable_libunwindstack, + '//gn:sqlite': + enable_sqlite, + '//gn:zlib': + enable_zlib, + '//gn:bionic_kernel_uapi_headers': + enable_uapi_headers, + '//src/profiling/memory:bionic_libc_platform_headers_on_android': + enable_bionic_libc_platform_headers_on_android, +} + +# ---------------------------------------------------------------------------- +# End of configuration. +# ---------------------------------------------------------------------------- + + +class Error(Exception): + pass + + +class ThrowingArgumentParser(argparse.ArgumentParser): + + def __init__(self, context): + super(ThrowingArgumentParser, self).__init__() + self.context = context + + def error(self, message): + raise Error('%s: %s' % (self.context, message)) + + +def write_blueprint_key_value(output, name, value, sort=True): + """Writes a Blueprint key-value pair to the output""" + + if isinstance(value, bool): + if value: + output.append(' %s: true,' % name) + else: + output.append(' %s: false,' % name) + return + if not value: + return + if isinstance(value, set): + value = sorted(value) + if isinstance(value, list): + output.append(' %s: [' % name) + for item in sorted(value) if sort else value: + output.append(' "%s",' % item) + output.append(' ],') + return + if isinstance(value, Target): + value.to_string(output) + return + if isinstance(value, dict): + kv_output = [] + for k, v in value.items(): + write_blueprint_key_value(kv_output, k, v) + + output.append(' %s: {' % name) + for line in kv_output: + output.append(' %s' % line) + output.append(' },') + return + output.append(' %s: "%s",' % (name, value)) + + +class Target(object): + """A target-scoped part of a module""" + + def __init__(self, name): + self.name = name + self.shared_libs = set() + self.static_libs = set() + self.whole_static_libs = set() + self.cflags = set() + self.dist = dict() + self.strip = dict() + self.stl = None + + def to_string(self, output): + nested_out = [] + self._output_field(nested_out, 'shared_libs') + self._output_field(nested_out, 'static_libs') + self._output_field(nested_out, 'whole_static_libs') + self._output_field(nested_out, 'cflags') + self._output_field(nested_out, 'stl') + self._output_field(nested_out, 'dist') + self._output_field(nested_out, 'strip') + + if nested_out: + output.append(' %s: {' % self.name) + for line in nested_out: + output.append(' %s' % line) + output.append(' },') + + def _output_field(self, output, name, sort=True): + value = getattr(self, name) + return write_blueprint_key_value(output, name, value, sort) + + +class Module(object): + """A single module (e.g., cc_binary, cc_test) in a blueprint.""" + + def __init__(self, mod_type, name, gn_target): + self.type = mod_type + self.gn_target = gn_target + self.name = name + self.srcs = set() + self.comment = 'GN: ' + gn_utils.label_without_toolchain(gn_target) + self.shared_libs = set() + self.static_libs = set() + self.whole_static_libs = set() + self.runtime_libs = set() + self.tools = set() + self.cmd = None + self.host_supported = False + self.vendor_available = False + self.init_rc = set() + self.out = set() + self.export_include_dirs = set() + self.generated_headers = set() + self.export_generated_headers = set() + self.defaults = set() + self.cflags = set() + self.include_dirs = set() + self.header_libs = set() + self.required = set() + self.user_debug_flag = False + self.tool_files = None + self.android = Target('android') + self.host = Target('host') + self.lto = None + self.stl = None + self.dist = dict() + self.strip = dict() + self.data = set() + self.apex_available = set() + self.min_sdk_version = None + self.proto = dict() + # The genrule_XXX below are properties that must to be propagated back + # on the module(s) that depend on the genrule. + self.genrule_headers = set() + self.genrule_srcs = set() + self.genrule_shared_libs = set() + self.version_script = None + self.test_suites = set() + self.test_config = None + self.stubs = {} + + def to_string(self, output): + if self.comment: + output.append('// %s' % self.comment) + output.append('%s {' % self.type) + self._output_field(output, 'name') + self._output_field(output, 'srcs') + self._output_field(output, 'shared_libs') + self._output_field(output, 'static_libs') + self._output_field(output, 'whole_static_libs') + self._output_field(output, 'runtime_libs') + self._output_field(output, 'tools') + self._output_field(output, 'cmd', sort=False) + if self.host_supported: + self._output_field(output, 'host_supported') + if self.vendor_available: + self._output_field(output, 'vendor_available') + self._output_field(output, 'init_rc') + self._output_field(output, 'out') + self._output_field(output, 'export_include_dirs') + self._output_field(output, 'generated_headers') + self._output_field(output, 'export_generated_headers') + self._output_field(output, 'defaults') + self._output_field(output, 'cflags') + self._output_field(output, 'include_dirs') + self._output_field(output, 'header_libs') + self._output_field(output, 'required') + self._output_field(output, 'dist') + self._output_field(output, 'strip') + self._output_field(output, 'tool_files') + self._output_field(output, 'data') + self._output_field(output, 'stl') + self._output_field(output, 'apex_available') + self._output_field(output, 'min_sdk_version') + self._output_field(output, 'version_script') + self._output_field(output, 'test_suites') + self._output_field(output, 'test_config') + self._output_field(output, 'stubs') + self._output_field(output, 'proto') + + target_out = [] + self._output_field(target_out, 'android') + self._output_field(target_out, 'host') + if target_out: + output.append(' target: {') + for line in target_out: + output.append(' %s' % line) + output.append(' },') + + if self.user_debug_flag: + output.append(' product_variables: {') + output.append(' debuggable: {') + output.append( + ' cflags: ["-DPERFETTO_BUILD_WITH_ANDROID_USERDEBUG"],') + output.append(' },') + output.append(' },') + if self.lto is not None: + output.append(' target: {') + output.append(' android: {') + output.append(' lto: {') + output.append(' thin: %s,' % + 'true' if self.lto else 'false') + output.append(' },') + output.append(' },') + output.append(' },') + output.append('}') + output.append('') + + def add_android_static_lib(self, lib): + if self.type == 'cc_binary_host': + raise Exception('Adding Android static lib for host tool is unsupported') + elif self.host_supported: + self.android.static_libs.add(lib) + else: + self.static_libs.add(lib) + + def add_android_shared_lib(self, lib): + if self.type == 'cc_binary_host': + raise Exception('Adding Android shared lib for host tool is unsupported') + elif self.host_supported: + self.android.shared_libs.add(lib) + else: + self.shared_libs.add(lib) + + def _output_field(self, output, name, sort=True): + value = getattr(self, name) + return write_blueprint_key_value(output, name, value, sort) + + +class Blueprint(object): + """In-memory representation of an Android.bp file.""" + + def __init__(self): + self.modules = {} + + def add_module(self, module): + """Adds a new module to the blueprint, replacing any existing module + with the same name. + + Args: + module: Module instance. + """ + self.modules[module.name] = module + + def to_string(self, output): + for m in sorted(itervalues(self.modules), key=lambda m: m.name): + m.to_string(output) + + +def label_to_module_name(label): + """Turn a GN label (e.g., //:perfetto_tests) into a module name.""" + # If the label is explicibly listed in the default target list, don't prefix + # its name and return just the target name. This is so tools like + # "traceconv" stay as such in the Android tree. + label_without_toolchain = gn_utils.label_without_toolchain(label) + if label in default_targets or label_without_toolchain in default_targets: + return label_without_toolchain.split(':')[-1] + + module = re.sub(r'^//:?', '', label_without_toolchain) + module = re.sub(r'[^a-zA-Z0-9_]', '_', module) + if not module.startswith(module_prefix): + return module_prefix + module + return module + + +def is_supported_source_file(name): + """Returns True if |name| can appear in a 'srcs' list.""" + return os.path.splitext(name)[1] in ['.c', '.cc', '.proto'] + + +def create_proto_modules(blueprint, gn, target): + """Generate genrules for a proto GN target. + + GN actions are used to dynamically generate files during the build. The + Soong equivalent is a genrule. This function turns a specific kind of + genrule which turns .proto files into source and header files into a pair + equivalent genrules. + + Args: + blueprint: Blueprint instance which is being generated. + target: gn_utils.Target object. + + Returns: + The source_genrule module. + """ + assert (target.type == 'proto_library') + + tools = {'aprotoc'} + cpp_out_dir = '$(genDir)/%s/' % tree_path + target_module_name = label_to_module_name(target.name) + + # In GN builds the proto path is always relative to the output directory + # (out/tmp.xxx). + cmd = ['mkdir -p %s &&' % cpp_out_dir, '$(location aprotoc)'] + cmd += ['--proto_path=%s' % tree_path] + + if buildtools_protobuf_src in target.proto_paths: + cmd += ['--proto_path=%s' % android_protobuf_src] + + # We don't generate any targets for source_set proto modules because + # they will be inlined into other modules if required. + if target.proto_plugin == 'source_set': + return None + + # Descriptor targets only generate a single target. + if target.proto_plugin == 'descriptor': + out = '{}.bin'.format(target_module_name) + + cmd += ['--descriptor_set_out=$(out)'] + cmd += ['$(in)'] + + descriptor_module = Module('genrule', target_module_name, target.name) + descriptor_module.cmd = ' '.join(cmd) + descriptor_module.out = [out] + descriptor_module.tools = tools + blueprint.add_module(descriptor_module) + + # Recursively extract the .proto files of all the dependencies and + # add them to srcs. + descriptor_module.srcs.update( + gn_utils.label_to_path(src) for src in target.sources) + for dep in target.transitive_proto_deps: + current_target = gn.get_target(dep) + descriptor_module.srcs.update( + gn_utils.label_to_path(src) for src in current_target.sources) + + return descriptor_module + + # We create two genrules for each proto target: one for the headers and + # another for the sources. This is because the module that depends on the + # generated files needs to declare two different types of dependencies -- + # source files in 'srcs' and headers in 'generated_headers' -- and it's not + # valid to generate .h files from a source dependency and vice versa. + source_module_name = target_module_name + '_gen' + source_module = Module('genrule', source_module_name, target.name) + blueprint.add_module(source_module) + source_module.srcs.update( + gn_utils.label_to_path(src) for src in target.sources) + + header_module = Module('genrule', source_module_name + '_headers', + target.name) + blueprint.add_module(header_module) + header_module.srcs = set(source_module.srcs) + + # TODO(primiano): at some point we should remove this. This was introduced + # by aosp/1108421 when adding "protos/" to .proto include paths, in order to + # avoid doing multi-repo changes and allow old clients in the android tree + # to still do the old #include "perfetto/..." rather than + # #include "protos/perfetto/...". + header_module.export_include_dirs = {'.', 'protos'} + + source_module.genrule_srcs.add(':' + source_module.name) + source_module.genrule_headers.add(header_module.name) + + if target.proto_plugin == 'proto': + suffixes = ['pb'] + source_module.genrule_shared_libs.add('libprotobuf-cpp-lite') + cmd += ['--cpp_out=lite=true:' + cpp_out_dir] + elif target.proto_plugin == 'protozero': + suffixes = ['pbzero'] + plugin = create_modules_from_target(blueprint, gn, protozero_plugin) + tools.add(plugin.name) + cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name] + cmd += ['--plugin_out=wrapper_namespace=pbzero:' + cpp_out_dir] + elif target.proto_plugin == 'cppgen': + suffixes = ['gen'] + plugin = create_modules_from_target(blueprint, gn, cppgen_plugin) + tools.add(plugin.name) + cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name] + cmd += ['--plugin_out=wrapper_namespace=gen:' + cpp_out_dir] + elif target.proto_plugin == 'ipc': + suffixes = ['ipc'] + plugin = create_modules_from_target(blueprint, gn, ipc_plugin) + tools.add(plugin.name) + cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name] + cmd += ['--plugin_out=wrapper_namespace=gen:' + cpp_out_dir] + else: + raise Error('Unsupported proto plugin: %s' % target.proto_plugin) + + cmd += ['$(in)'] + source_module.cmd = ' '.join(cmd) + header_module.cmd = source_module.cmd + source_module.tools = tools + header_module.tools = tools + + for sfx in suffixes: + source_module.out.update('%s/%s' % + (tree_path, src.replace('.proto', '.%s.cc' % sfx)) + for src in source_module.srcs) + header_module.out.update('%s/%s' % + (tree_path, src.replace('.proto', '.%s.h' % sfx)) + for src in header_module.srcs) + return source_module + + +def create_amalgamated_sql_metrics_module(blueprint, target): + bp_module_name = label_to_module_name(target.name) + module = Module('genrule', bp_module_name, target.name) + module.tool_files = [ + 'tools/gen_amalgamated_sql_metrics.py', + ] + module.cmd = ' '.join([ + '$(location tools/gen_amalgamated_sql_metrics.py)', + '--cpp_out=$(out)', + '$(in)', + ]) + module.genrule_headers.add(module.name) + module.out.update(target.outputs) + module.srcs.update(gn_utils.label_to_path(src) for src in target.inputs) + blueprint.add_module(module) + return module + + +def create_cc_proto_descriptor_module(blueprint, target): + bp_module_name = label_to_module_name(target.name) + module = Module('genrule', bp_module_name, target.name) + module.tool_files = [ + 'tools/gen_cc_proto_descriptor.py', + ] + module.cmd = ' '.join([ + '$(location tools/gen_cc_proto_descriptor.py)', '--gen_dir=$(genDir)', + '--cpp_out=$(out)', '$(in)' + ]) + module.genrule_headers.add(module.name) + module.srcs.update( + ':' + label_to_module_name(dep) for dep in target.proto_deps) + module.srcs.update( + gn_utils.label_to_path(src) + for src in target.inputs + if "tmp.gn_utils" not in src) + module.out.update(target.outputs) + blueprint.add_module(module) + return module + + +def create_gen_version_module(blueprint, target, bp_module_name): + module = Module('genrule', bp_module_name, gn_utils.GEN_VERSION_TARGET) + script_path = gn_utils.label_to_path(target.script) + module.genrule_headers.add(bp_module_name) + module.tool_files = [script_path] + module.out.update(target.outputs) + module.srcs.update(gn_utils.label_to_path(src) for src in target.inputs) + module.cmd = ' '.join([ + 'python3 $(location %s)' % script_path, '--no_git', + '--changelog=$(location CHANGELOG)', '--cpp_out=$(out)' + ]) + blueprint.add_module(module) + return module + + +def create_proto_group_modules(blueprint, gn, module_name, target_names): + # TODO(lalitm): today, we're only adding a Java lite module because that's + # the only one used in practice. In the future, if we need other target types + # (e.g. C++, Java full etc.) add them here. + bp_module_name = label_to_module_name(module_name) + '_java_protos' + module = Module('java_library', bp_module_name, bp_module_name) + module.comment = f'''GN: [{', '.join(target_names)}]''' + module.proto = {'type': 'lite', 'canonical_path_from_root': False} + + for name in target_names: + target = gn.get_target(name) + module.srcs.update(gn_utils.label_to_path(src) for src in target.sources) + for dep_label in target.transitive_proto_deps: + dep = gn.get_target(dep_label) + module.srcs.update(gn_utils.label_to_path(src) for src in dep.sources) + + blueprint.add_module(module) + + +def _get_cflags(target): + cflags = {flag for flag in target.cflags if re.match(cflag_allowlist, flag)} + cflags |= set("-D%s" % define + for define in target.defines + if re.match(define_allowlist, define)) + return cflags + + +def create_modules_from_target(blueprint, gn, gn_target_name): + """Generate module(s) for a given GN target. + + Given a GN target name, generate one or more corresponding modules into a + blueprint. The only case when this generates >1 module is proto libraries. + + Args: + blueprint: Blueprint instance which is being generated. + gn: gn_utils.GnParser object. + gn_target_name: GN target for module generation. + """ + bp_module_name = label_to_module_name(gn_target_name) + if bp_module_name in blueprint.modules: + return blueprint.modules[bp_module_name] + target = gn.get_target(gn_target_name) + + name_without_toolchain = gn_utils.label_without_toolchain(target.name) + if target.type == 'executable': + if target.toolchain == gn_utils.HOST_TOOLCHAIN: + module_type = 'cc_binary_host' + elif target.testonly: + module_type = 'cc_test' + else: + module_type = 'cc_binary' + module = Module(module_type, bp_module_name, gn_target_name) + elif target.type == 'static_library': + module = Module('cc_library_static', bp_module_name, gn_target_name) + elif target.type == 'shared_library': + module = Module('cc_library_shared', bp_module_name, gn_target_name) + elif target.type == 'source_set': + module = Module('filegroup', bp_module_name, gn_target_name) + elif target.type == 'group': + # "group" targets are resolved recursively by gn_utils.get_target(). + # There's nothing we need to do at this level for them. + return None + elif target.type == 'proto_library': + module = create_proto_modules(blueprint, gn, target) + if module is None: + return None + elif target.type == 'action': + if 'gen_amalgamated_sql_metrics' in target.name: + module = create_amalgamated_sql_metrics_module(blueprint, target) + elif re.match('.*gen_cc_.*_descriptor$', name_without_toolchain): + module = create_cc_proto_descriptor_module(blueprint, target) + elif target.type == 'action' and \ + name_without_toolchain == gn_utils.GEN_VERSION_TARGET: + module = create_gen_version_module(blueprint, target, bp_module_name) + else: + raise Error('Unhandled action: {}'.format(target.name)) + else: + raise Error('Unknown target %s (%s)' % (target.name, target.type)) + + blueprint.add_module(module) + module.host_supported = (name_without_toolchain in target_host_supported) + module.vendor_available = (name_without_toolchain in target_vendor_available) + module.init_rc = target_initrc.get(target.name, []) + module.srcs.update( + gn_utils.label_to_path(src) + for src in target.sources + if is_supported_source_file(src)) + + if target.type in gn_utils.LINKER_UNIT_TYPES: + module.cflags.update(_get_cflags(target)) + + module_is_compiled = module.type not in ('genrule', 'filegroup') + if module_is_compiled: + # Don't try to inject library/source dependencies into genrules or + # filegroups because they are not compiled in the traditional sense. + module.defaults = [defaults_module] + for lib in target.libs: + # Generally library names should be mangled as 'libXXX', unless they + # are HAL libraries (e.g., android.hardware.health@2.0) or AIDL c++ / NDK + # libraries (e.g. "android.hardware.power.stats-V1-cpp") + android_lib = lib if '@' in lib or "-cpp" in lib or "-ndk" in lib \ + else 'lib' + lib + if lib in shared_library_allowlist: + module.add_android_shared_lib(android_lib) + if lib in static_library_allowlist: + module.add_android_static_lib(android_lib) + + # If the module is a static library, export all the generated headers. + if module.type == 'cc_library_static': + module.export_generated_headers = module.generated_headers + + # Merge in additional hardcoded arguments. + for key, add_val in additional_args.get(module.name, []): + curr = getattr(module, key) + if add_val and isinstance(add_val, set) and isinstance(curr, set): + curr.update(add_val) + elif isinstance(add_val, str) and (not curr or isinstance(curr, str)): + setattr(module, key, add_val) + elif isinstance(add_val, bool) and (not curr or isinstance(curr, bool)): + setattr(module, key, add_val) + elif isinstance(add_val, dict) and isinstance(curr, dict): + curr.update(add_val) + elif isinstance(add_val, dict) and isinstance(curr, Target): + curr.__dict__.update(add_val) + else: + raise Error('Unimplemented type %r of additional_args: %r' % + (type(add_val), key)) + + # dep_name is an unmangled GN target name (e.g. //foo:bar(toolchain)). + all_deps = target.deps | target.source_set_deps | target.transitive_proto_deps + for dep_name in all_deps: + # If the dependency refers to a library which we can replace with an + # Android equivalent, stop recursing and patch the dependency in. + # Don't recurse into //buildtools, builtin_deps are intercepted at + # the //gn:xxx level. + if dep_name.startswith('//buildtools'): + continue + + # Ignore the dependency on the gen_buildflags genrule. That is run + # separately in this generator and the generated file is copied over + # into the repo (see usage of |buildflags_dir| in this script). + if dep_name.startswith(gn_utils.BUILDFLAGS_TARGET): + continue + + dep_module = create_modules_from_target(blueprint, gn, dep_name) + + # For filegroups and genrule, recurse but don't apply the deps. + if not module_is_compiled: + continue + + # |builtin_deps| override GN deps with Android-specific ones. See the + # config in the top of this file. + if gn_utils.label_without_toolchain(dep_name) in builtin_deps: + builtin_deps[gn_utils.label_without_toolchain(dep_name)](module) + continue + + # Don't recurse in any other //gn dep if not handled by builtin_deps. + if dep_name.startswith('//gn:'): + continue + + if dep_module is None: + continue + if dep_module.type == 'cc_library_shared': + module.shared_libs.add(dep_module.name) + elif dep_module.type == 'cc_library_static': + module.static_libs.add(dep_module.name) + elif dep_module.type == 'filegroup': + module.srcs.add(':' + dep_module.name) + elif dep_module.type == 'genrule': + module.generated_headers.update(dep_module.genrule_headers) + module.srcs.update(dep_module.genrule_srcs) + module.shared_libs.update(dep_module.genrule_shared_libs) + elif dep_module.type == 'cc_binary': + continue # Ignore executables deps (used by cmdline integration tests). + else: + raise Error('Unknown dep %s (%s) for target %s' % + (dep_module.name, dep_module.type, module.name)) + + return module + + +def create_blueprint_for_targets(gn, desc, targets): + """Generate a blueprint for a list of GN targets.""" + blueprint = Blueprint() + + # Default settings used by all modules. + defaults = Module('cc_defaults', defaults_module, '//gn:default_deps') + + # We have to use include_dirs passing the path relative to the android tree. + # This is because: (i) perfetto_cc_defaults is used also by + # test/**/Android.bp; (ii) if we use local_include_dirs instead, paths + # become relative to the Android.bp that *uses* cc_defaults (not the one + # that defines it).s + defaults.include_dirs = { + tree_path, tree_path + '/include', tree_path + '/' + buildflags_dir, + tree_path + '/src/profiling/memory/include' + } + defaults.cflags = [ + '-Wno-error=return-type', + '-Wno-sign-compare', + '-Wno-sign-promo', + '-Wno-unused-parameter', + '-fvisibility=hidden', + '-O2', + ] + defaults.user_debug_flag = True + defaults.lto = True + + blueprint.add_module(defaults) + for target in targets: + create_modules_from_target(blueprint, gn, target) + return blueprint + + +def main(): + parser = argparse.ArgumentParser( + description='Generate Android.bp from a GN description.') + parser.add_argument( + '--check-only', + help='Don\'t keep the generated files', + action='store_true') + parser.add_argument( + '--desc', + help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"' + ) + parser.add_argument( + '--extras', + help='Extra targets to include at the end of the Blueprint file', + default=os.path.join(gn_utils.repo_root(), 'Android.bp.extras'), + ) + parser.add_argument( + '--output', + help='Blueprint file to create', + default=os.path.join(gn_utils.repo_root(), 'Android.bp'), + ) + parser.add_argument( + 'targets', + nargs=argparse.REMAINDER, + help='Targets to include in the blueprint (e.g., "//:perfetto_tests")') + args = parser.parse_args() + + if args.desc: + with open(args.desc) as f: + desc = json.load(f) + else: + desc = gn_utils.create_build_description(gn_args) + + gn = gn_utils.GnParser(desc) + blueprint = create_blueprint_for_targets(gn, desc, args.targets or + default_targets) + project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + tool_name = os.path.relpath(os.path.abspath(__file__), project_root) + + # TODO(primiano): enable this on Android after the TODO in + # perfetto_component.gni is fixed. + # Check for ODR violations + # for target_name in default_targets: + # checker = gn_utils.ODRChecker(gn, target_name) + + # Add any proto groups to the blueprint. + for l_name, t_names in proto_groups.items(): + create_proto_group_modules(blueprint, gn, l_name, t_names) + + output = [ + """// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file is automatically generated by %s. Do not edit. +""" % (tool_name) + ] + blueprint.to_string(output) + with open(args.extras, 'r') as r: + for line in r: + output.append(line.rstrip("\n\r")) + + out_files = [] + + # Generate the Android.bp file. + out_files.append(args.output + '.swp') + with open(out_files[-1], 'w') as f: + f.write('\n'.join(output)) + # Text files should have a trailing EOL. + f.write('\n') + + # Generate the perfetto_build_flags.h file. + out_files.append(os.path.join(buildflags_dir, 'perfetto_build_flags.h.swp')) + gn_utils.gen_buildflags(gn_args, out_files[-1]) + + # Either check the contents or move the files to their final destination. + return gn_utils.check_or_commit_generated_files(out_files, args.check_only) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/gn2bp/gn_utils.py b/tools/gn2bp/gn_utils.py new file mode 100644 index 0000000000..91d7a32068 --- /dev/null +++ b/tools/gn2bp/gn_utils.py @@ -0,0 +1,525 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A collection of utilities for extracting build rule information from GN +# projects. + +from __future__ import print_function +import collections +import errno +import filecmp +import json +import os +import re +import shutil +import subprocess +import sys +from compat import iteritems + +BUILDFLAGS_TARGET = '//gn:gen_buildflags' +GEN_VERSION_TARGET = '//src/base:version_gen_h' +TARGET_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host' +HOST_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host' +LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library') + +# TODO(primiano): investigate these, they require further componentization. +ODR_VIOLATION_IGNORE_TARGETS = { + '//test/cts:perfetto_cts_deps', + '//:perfetto_integrationtests', +} + + +def _check_command_output(cmd, cwd): + try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd) + except subprocess.CalledProcessError as e: + print( + 'Command "{}" failed in {}:'.format(' '.join(cmd), cwd), + file=sys.stderr) + print(e.output.decode(), file=sys.stderr) + sys.exit(1) + else: + return output.decode() + + +def repo_root(): + """Returns an absolute path to the repository root.""" + return os.path.join( + os.path.realpath(os.path.dirname(__file__)), os.path.pardir) + + +def _tool_path(name, system_buildtools=False): + # Pass-through to use name if the caller requests to use the system + # toolchain. + if system_buildtools: + return [name] + wrapper = os.path.abspath( + os.path.join(repo_root(), 'tools', 'run_buildtools_binary.py')) + return ['python3', wrapper, name] + + +def prepare_out_directory(gn_args, + name, + root=repo_root(), + system_buildtools=False): + """Creates the JSON build description by running GN. + + Returns (path, desc) where |path| is the location of the output directory + and |desc| is the JSON build description. + """ + out = os.path.join(root, 'out', name) + try: + os.makedirs(out) + except OSError as e: + if e.errno != errno.EEXIST: + raise + _check_command_output( + _tool_path('gn', system_buildtools) + + ['gen', out, '--args=%s' % gn_args], + cwd=repo_root()) + return out + + +def load_build_description(out, system_buildtools=False): + """Creates the JSON build description by running GN.""" + desc = _check_command_output( + _tool_path('gn', system_buildtools) + + ['desc', out, '--format=json', '--all-toolchains', '//*'], + cwd=repo_root()) + return json.loads(desc) + + +def create_build_description(gn_args, root=repo_root()): + """Prepares a GN out directory and loads the build description from it. + + The temporary out directory is automatically deleted. + """ + out = prepare_out_directory(gn_args, 'tmp.gn_utils', root=root) + try: + return load_build_description(out) + finally: + shutil.rmtree(out) + + +def build_targets(out, targets, quiet=False, system_buildtools=False): + """Runs ninja to build a list of GN targets in the given out directory. + + Compiling these targets is required so that we can include any generated + source files in the amalgamated result. + """ + targets = [t.replace('//', '') for t in targets] + with open(os.devnull, 'w') as devnull: + stdout = devnull if quiet else None + cmd = _tool_path('ninja', system_buildtools) + targets + subprocess.check_call(cmd, cwd=os.path.abspath(out), stdout=stdout) + + +def compute_source_dependencies(out, system_buildtools=False): + """For each source file, computes a set of headers it depends on.""" + ninja_deps = _check_command_output( + _tool_path('ninja', system_buildtools) + ['-t', 'deps'], cwd=out) + deps = {} + current_source = None + for line in ninja_deps.split('\n'): + filename = os.path.relpath(os.path.join(out, line.strip()), repo_root()) + if not line or line[0] != ' ': + current_source = None + continue + elif not current_source: + # We're assuming the source file is always listed before the + # headers. + assert os.path.splitext(line)[1] in ['.c', '.cc', '.cpp', '.S'] + current_source = filename + deps[current_source] = [] + else: + assert current_source + deps[current_source].append(filename) + return deps + + +def label_to_path(label): + """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" + assert label.startswith('//') + return label[2:] + + +def label_without_toolchain(label): + """Strips the toolchain from a GN label. + + Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: + gcc_like_host) without the parenthesised toolchain part. + """ + return label.split('(')[0] + + +def label_to_target_name_with_path(label): + """ + Turn a GN label into a target name involving the full path. + e.g., //src/perfetto:tests -> src_perfetto_tests + """ + name = re.sub(r'^//:?', '', label) + name = re.sub(r'[^a-zA-Z0-9_]', '_', name) + return name + + +def gen_buildflags(gn_args, target_file): + """Generates the perfetto_build_flags.h for the given config. + + target_file: the path, relative to the repo root, where the generated + buildflag header will be copied into. + """ + tmp_out = prepare_out_directory(gn_args, 'tmp.gen_buildflags') + build_targets(tmp_out, [BUILDFLAGS_TARGET], quiet=True) + src = os.path.join(tmp_out, 'gen', 'build_config', 'perfetto_build_flags.h') + shutil.copy(src, os.path.join(repo_root(), target_file)) + shutil.rmtree(tmp_out) + + +def check_or_commit_generated_files(tmp_files, check): + """Checks that gen files are unchanged or renames them to the final location + + Takes in input a list of 'xxx.swp' files that have been written. + If check == False, it renames xxx.swp -> xxx. + If check == True, it just checks that the contents of 'xxx.swp' == 'xxx'. + Returns 0 if no diff was detected, 1 otherwise (to be used as exit code). + """ + res = 0 + for tmp_file in tmp_files: + assert (tmp_file.endswith('.swp')) + target_file = os.path.relpath(tmp_file[:-4]) + if check: + if not filecmp.cmp(tmp_file, target_file): + sys.stderr.write('%s needs to be regenerated\n' % target_file) + res = 1 + os.unlink(tmp_file) + else: + os.rename(tmp_file, target_file) + return res + + +class ODRChecker(object): + """Detects ODR violations in linker units + + When we turn GN source sets into Soong & Bazel file groups, there is the risk + to create ODR violations by including the same file group into different + linker unit (this is because other build systems don't have a concept + equivalent to GN's source_set). This class navigates the transitive + dependencies (mostly static libraries) of a target and detects if multiple + paths end up including the same file group. This is to avoid situations like: + + traced.exe -> base(file group) + traced.exe -> libperfetto(static lib) -> base(file group) + """ + + def __init__(self, gn, target_name): + self.gn = gn + self.root = gn.get_target(target_name) + self.source_sets = collections.defaultdict(set) + self.deps_visited = set() + self.source_set_hdr_only = {} + + self._visit(target_name) + num_violations = 0 + if target_name in ODR_VIOLATION_IGNORE_TARGETS: + return + for sset, paths in self.source_sets.items(): + if self.is_header_only(sset): + continue + if len(paths) != 1: + num_violations += 1 + print( + 'ODR violation in target %s, multiple paths include %s:\n %s' % + (target_name, sset, '\n '.join(paths)), + file=sys.stderr) + if num_violations > 0: + raise Exception('%d ODR violations detected. Build generation aborted' % + num_violations) + + def _visit(self, target_name, parent_path=''): + target = self.gn.get_target(target_name) + path = ((parent_path + ' > ') if parent_path else '') + target_name + if not target: + raise Exception('Cannot find target %s' % target_name) + for ssdep in target.source_set_deps: + name_and_path = '%s (via %s)' % (target_name, path) + self.source_sets[ssdep].add(name_and_path) + deps = set(target.deps).union( + target.transitive_proto_deps) - self.deps_visited + for dep_name in deps: + dep = self.gn.get_target(dep_name) + if dep.type == 'executable': + continue # Execs are strong boundaries and don't cause ODR violations. + # static_library dependencies should reset the path. It doesn't matter if + # we get to a source file via: + # source_set1 > static_lib > source.cc OR + # source_set1 > source_set2 > static_lib > source.cc + # This is NOT an ODR violation because source.cc is linked from the same + # static library + next_parent_path = path if dep.type != 'static_library' else '' + self.deps_visited.add(dep_name) + self._visit(dep_name, next_parent_path) + + def is_header_only(self, source_set_name): + cached = self.source_set_hdr_only.get(source_set_name) + if cached is not None: + return cached + target = self.gn.get_target(source_set_name) + if target.type != 'source_set': + raise TypeError('%s is not a source_set' % source_set_name) + res = all(src.endswith('.h') for src in target.sources) + self.source_set_hdr_only[source_set_name] = res + return res + + +class GnParser(object): + """A parser with some cleverness for GN json desc files + + The main goals of this parser are: + 1) Deal with the fact that other build systems don't have an equivalent + notion to GN's source_set. Conversely to Bazel's and Soong's filegroups, + GN source_sets expect that dependencies, cflags and other source_set + properties propagate up to the linker unit (static_library, executable or + shared_library). This parser simulates the same behavior: when a + source_set is encountered, some of its variables (cflags and such) are + copied up to the dependent targets. This is to allow gen_xxx to create + one filegroup for each source_set and then squash all the other flags + onto the linker unit. + 2) Detect and special-case protobuf targets, figuring out the protoc-plugin + being used. + """ + + class Target(object): + """Reperesents A GN target. + + Maked properties are propagated up the dependency chain when a + source_set dependency is encountered. + """ + + def __init__(self, name, type): + self.name = name # e.g. //src/ipc:ipc + + VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group', + 'action', 'source_set', 'proto_library') + assert (type in VALID_TYPES) + self.type = type + self.testonly = False + self.toolchain = None + + # These are valid only for type == proto_library. + # This is typically: 'proto', 'protozero', 'ipc'. + self.proto_plugin = None + self.proto_paths = set() + self.proto_exports = set() + + self.sources = set() + # TODO(primiano): consider whether the public section should be part of + # bubbled-up sources. + self.public_headers = set() # 'public' + + # These are valid only for type == 'action' + self.inputs = set() + self.outputs = set() + self.script = None + self.args = [] + + # These variables are propagated up when encountering a dependency + # on a source_set target. + self.cflags = set() + self.defines = set() + self.deps = set() + self.libs = set() + self.include_dirs = set() + self.ldflags = set() + self.source_set_deps = set() # Transitive set of source_set deps. + self.proto_deps = set() + self.transitive_proto_deps = set() + + # Deps on //gn:xxx have this flag set to True. These dependencies + # are special because they pull third_party code from buildtools/. + # We don't want to keep recursing into //buildtools in generators, + # this flag is used to stop the recursion and create an empty + # placeholder target once we hit //gn:protoc or similar. + self.is_third_party_dep_ = False + + def __lt__(self, other): + if isinstance(other, self.__class__): + return self.name < other.name + raise TypeError( + '\'<\' not supported between instances of \'%s\' and \'%s\'' % + (type(self).__name__, type(other).__name__)) + + def __repr__(self): + return json.dumps({ + k: (list(sorted(v)) if isinstance(v, set) else v) + for (k, v) in iteritems(self.__dict__) + }, + indent=4, + sort_keys=True) + + def update(self, other): + for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags', + 'source_set_deps', 'proto_deps', 'transitive_proto_deps', + 'libs', 'proto_paths'): + self.__dict__[key].update(other.__dict__.get(key, [])) + + def __init__(self, gn_desc): + self.gn_desc_ = gn_desc + self.all_targets = {} + self.linker_units = {} # Executables, shared or static libraries. + self.source_sets = {} + self.actions = {} + self.proto_libs = {} + + def get_target(self, gn_target_name): + """Returns a Target object from the fully qualified GN target name. + + It bubbles up variables from source_set dependencies as described in the + class-level comments. + """ + target = self.all_targets.get(gn_target_name) + if target is not None: + return target # Target already processed. + + desc = self.gn_desc_[gn_target_name] + target = GnParser.Target(gn_target_name, desc['type']) + target.testonly = desc.get('testonly', False) + target.toolchain = desc.get('toolchain', None) + self.all_targets[gn_target_name] = target + + # We should never have GN targets directly depend on buidtools. They + # should hop via //gn:xxx, so we can give generators an opportunity to + # override them. + assert (not gn_target_name.startswith('//buildtools')) + + # Don't descend further into third_party targets. Genrators are supposed + # to either ignore them or route to other externally-provided targets. + if gn_target_name.startswith('//gn'): + target.is_third_party_dep_ = True + return target + + proto_target_type, proto_desc = self.get_proto_target_type(target) + if proto_target_type is not None: + self.proto_libs[target.name] = target + target.type = 'proto_library' + target.proto_plugin = proto_target_type + target.proto_paths.update(self.get_proto_paths(proto_desc)) + target.proto_exports.update(self.get_proto_exports(proto_desc)) + target.sources.update(proto_desc.get('sources', [])) + assert (all(x.endswith('.proto') for x in target.sources)) + elif target.type == 'source_set': + self.source_sets[gn_target_name] = target + target.sources.update(desc.get('sources', [])) + elif target.type in LINKER_UNIT_TYPES: + self.linker_units[gn_target_name] = target + target.sources.update(desc.get('sources', [])) + elif target.type == 'action': + self.actions[gn_target_name] = target + target.inputs.update(desc.get('inputs', [])) + target.sources.update(desc.get('sources', [])) + outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']] + target.outputs.update(outs) + target.script = desc['script'] + # Args are typically relative to the root build dir (../../xxx) + # because root build dir is typically out/xxx/). + target.args = [re.sub('^../../', '//', x) for x in desc['args']] + + # Default for 'public' is //* - all headers in 'sources' are public. + # TODO(primiano): if a 'public' section is specified (even if empty), then + # the rest of 'sources' is considered inaccessible by gn. Consider + # emulating that, so that generated build files don't end up with overly + # accessible headers. + public_headers = [x for x in desc.get('public', []) if x != '*'] + target.public_headers.update(public_headers) + + target.cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', [])) + target.libs.update(desc.get('libs', [])) + target.ldflags.update(desc.get('ldflags', [])) + target.defines.update(desc.get('defines', [])) + target.include_dirs.update(desc.get('include_dirs', [])) + + # Recurse in dependencies. + for dep_name in desc.get('deps', []): + dep = self.get_target(dep_name) + if dep.is_third_party_dep_: + target.deps.add(dep_name) + elif dep.type == 'proto_library': + target.proto_deps.add(dep_name) + target.transitive_proto_deps.add(dep_name) + target.proto_paths.update(dep.proto_paths) + target.transitive_proto_deps.update(dep.transitive_proto_deps) + elif dep.type == 'source_set': + target.source_set_deps.add(dep_name) + target.update(dep) # Bubble up source set's cflags/ldflags etc. + elif dep.type == 'group': + target.update(dep) # Bubble up groups's cflags/ldflags etc. + elif dep.type == 'action': + if proto_target_type is None: + target.deps.add(dep_name) + elif dep.type in LINKER_UNIT_TYPES: + target.deps.add(dep_name) + + return target + + def get_proto_exports(self, proto_desc): + # exports in metadata will be available for source_set targets. + metadata = proto_desc.get('metadata', {}) + return metadata.get('exports', []) + + def get_proto_paths(self, proto_desc): + # import_dirs in metadata will be available for source_set targets. + metadata = proto_desc.get('metadata', {}) + return metadata.get('import_dirs', []) + + def get_proto_target_type(self, target): + """ Checks if the target is a proto library and return the plugin. + + Returns: + (None, None): if the target is not a proto library. + (plugin, proto_desc) where |plugin| is 'proto' in the default (lite) + case or 'protozero' or 'ipc' or 'descriptor'; |proto_desc| is the GN + json desc of the target with the .proto sources (_gen target for + non-descriptor types or the target itself for descriptor type). + """ + parts = target.name.split('(', 1) + name = parts[0] + toolchain = '(' + parts[1] if len(parts) > 1 else '' + + # Descriptor targets don't have a _gen target; instead we look for the + # characteristic flag in the args of the target itself. + desc = self.gn_desc_.get(target.name) + if '--descriptor_set_out' in desc.get('args', []): + return 'descriptor', desc + + # Source set proto targets have a non-empty proto_library_sources in the + # metadata of the description. + metadata = desc.get('metadata', {}) + if 'proto_library_sources' in metadata: + return 'source_set', desc + + # In all other cases, we want to look at the _gen target as that has the + # important information. + gen_desc = self.gn_desc_.get('%s_gen%s' % (name, toolchain)) + if gen_desc is None or gen_desc['type'] != 'action': + return None, None + args = gen_desc.get('args', []) + if '/protoc' not in args[0]: + return None, None + plugin = 'proto' + for arg in (arg for arg in args if arg.startswith('--plugin=')): + # |arg| at this point looks like: + # --plugin=protoc-gen-plugin=gcc_like_host/protozero_plugin + # or + # --plugin=protoc-gen-plugin=protozero_plugin + plugin = arg.split('=')[-1].split('/')[-1].replace('_plugin', '') + return plugin, gen_desc diff --git a/tools/gn2bp/run_buildtools_binary.py b/tools/gn2bp/run_buildtools_binary.py new file mode 100755 index 0000000000..7a6fe818f2 --- /dev/null +++ b/tools/gn2bp/run_buildtools_binary.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" A wrapper to run gn, ninja and other buildtools/ for all platforms. """ + +from __future__ import print_function + +import os +import subprocess +import sys + +from platform import system, machine + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def run_buildtools_binary(args): + if len(args) < 1: + print('Usage %s command [args]\n' % sys.argv[0]) + return 1 + + sys_name = system().lower() + os_dir = None + ext = '' + if sys_name == 'windows': + os_dir = 'win' + ext = '.exe' + elif sys_name == 'darwin': + os_dir = 'mac' + elif sys_name == 'linux': + os_dir = 'linux64' + else: + print('OS not supported: %s\n' % sys_name) + return 1 + + cmd = args[0] + args = args[1:] + exe_path = os.path.join(ROOT_DIR, 'buildtools', os_dir, cmd) + ext + if sys_name == 'windows': + # execl() behaves oddly on Windows: the spawned process doesn't seem to + # receive CTRL+C. Use subprocess instead. + return subprocess.call([exe_path] + args) + else: + os.execl(exe_path, os.path.basename(exe_path), *args) + + +if __name__ == '__main__': + sys.exit(run_buildtools_binary(sys.argv[1:]))