From e2731e0d940578c9eaca4bfa91ea6420966c6a55 Mon Sep 17 00:00:00 2001 From: Tony Liao Date: Wed, 26 Aug 2026 11:56:20 -0700 Subject: [PATCH 1/2] Support plug-and-play monolithic Docker testing across Bazel build definitions. BEGIN_PUBLIC Support plug-and-play monolithic Docker testing across Bazel build definitions. To support the multi-language monolithic test container (introduced in build-support#199), Protocol Buffers needs to build and test cleanly under wildcard target expansion (`bazel test //...`) in fully hermetic, isolated container environments. Currently, executing `bazel test //...` out-of-the-box in containerized toolchains encounters multiple analysis, runtime, and environment errors: - Leaked internal Google3 labels and repo-relative paths in Starlark macros cause target analysis failures when exported to OSS. - Containerized runtime runners lack environment isolation (e.g., .NET CLI attempting to write to restricted home paths, non-standard Python interpreter paths), causing sandboxed execution failures. - Starlark C++ proto aspects crash when analyzing headerless/empty proto libraries. - Legacy network-dependent tests and deprecated blocklist targets break offline test suites during wildcard evaluation. This change updates the build definitions, test runner scripts, and Copybara export rules so that the entire test suite resolves hermetically and passes seamlessly in both local container environments and CI workflows without external dependencies or manual overrides. END_PUBLIC R=mkruskal,sandyzhang CC=protocol-devel+reviews CONV=746be3bf-c8f0-4a89-8ce8-056b27cfe8fa MARKDOWN=true RELEASE_NOTES=no TAG=agy --- .bazelignore | 1 + bazel/private/oss/cc_proto_library.bzl | 64 ++++++++++--------- build_defs/BUILD.bazel | 10 --- conformance/bazel_conformance_test_runner.sh | 17 +++-- conformance/defs.bzl | 2 +- csharp/BUILD.bazel | 12 +++- csharp/compatibility_tests/v3.0.0/BUILD.bazel | 4 ++ .../generate_protos.bzl | 16 ++--- .../Google.Protobuf.Conformance/BUILD.bazel | 6 ++ editions/BUILD | 2 +- upb/bazel/private/oss/amalgamate.py | 3 +- 11 files changed, 76 insertions(+), 61 deletions(-) diff --git a/.bazelignore b/.bazelignore index 69fa449dd96e2..2738ca30273c6 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1 +1,2 @@ _build/ +examples diff --git a/bazel/private/oss/cc_proto_library.bzl b/bazel/private/oss/cc_proto_library.bzl index 19eb7055f69b6..9a48292c70147 100644 --- a/bazel/private/oss/cc_proto_library.bzl +++ b/bazel/private/oss/cc_proto_library.bzl @@ -77,37 +77,41 @@ def _aspect_impl(target, ctx): transitive_headers.append(dep[_ProtoCcHeaderInfo].headers) header_provider = _ProtoCcHeaderInfo(headers = depset(transitive = transitive_headers)) + proto_common.compile( + actions = ctx.actions, + proto_info = proto_info, + proto_lang_toolchain_info = proto_toolchain, + generated_files = sources + headers, + experimental_output_files = "multiple", + ) + + deps = [] + if proto_toolchain.runtime: + deps = [proto_toolchain.runtime] + deps.extend(getattr(ctx.rule.attr, "deps", [])) + + cc_info, libraries, temps = cc_proto_compile_and_link( + ctx = ctx, + deps = deps, + sources = sources, + headers = headers, + textual_hdrs = textual_hdrs, + strip_include_prefix = _get_strip_include_prefix(ctx, proto_info), + ) + + return [ + cc_info, + _ProtoCcFilesInfo(files = depset(sources + headers + libraries)), + OutputGroupInfo(temp_files_INTERNAL_ = temps), + header_provider, + ] else: # shouldn't generate code - header_provider = _ProtoCcHeaderInfo(headers = depset()) - - proto_common.compile( - actions = ctx.actions, - proto_info = proto_info, - proto_lang_toolchain_info = proto_toolchain, - generated_files = sources + headers, - experimental_output_files = "multiple", - ) - - deps = [] - if proto_toolchain.runtime: - deps = [proto_toolchain.runtime] - deps.extend(getattr(ctx.rule.attr, "deps", [])) - - cc_info, libraries, temps = cc_proto_compile_and_link( - ctx = ctx, - deps = deps, - sources = sources, - headers = headers, - textual_hdrs = textual_hdrs, - strip_include_prefix = _get_strip_include_prefix(ctx, proto_info), - ) - - return [ - cc_info, - _ProtoCcFilesInfo(files = depset(sources + headers + libraries)), - OutputGroupInfo(temp_files_INTERNAL_ = temps), - header_provider, - ] + return [ + CcInfo(), + _ProtoCcFilesInfo(files = depset()), + OutputGroupInfo(), + _ProtoCcHeaderInfo(headers = depset()), + ] cc_proto_aspect = aspect( implementation = _aspect_impl, diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel index 8591747933a43..5efc30f29a291 100644 --- a/build_defs/BUILD.bazel +++ b/build_defs/BUILD.bazel @@ -224,17 +224,7 @@ starlark_cc_proto_library( cc_proto_blacklist_test( name = "cc_proto_blacklist_test", deps = [ - ":any_cc_proto", - ":api_cc_proto", ":compiler_plugin_cc_proto", ":descriptor_cc_proto", - ":duration_cc_proto", - ":empty_cc_proto", - ":field_mask_cc_proto", - ":source_context_cc_proto", - ":struct_cc_proto", - ":timestamp_cc_proto", - ":type_cc_proto", - ":wrappers_cc_proto", ], ) diff --git a/conformance/bazel_conformance_test_runner.sh b/conformance/bazel_conformance_test_runner.sh index e3d4c6777081c..bb88a91d37dee 100755 --- a/conformance/bazel_conformance_test_runner.sh +++ b/conformance/bazel_conformance_test_runner.sh @@ -32,15 +32,16 @@ TESTEE= FAILURE_LIST= TEXT_FORMAT_FAILURE_LIST= MAXIMUM_EDITION= +PERFORMANCE= -while [[ -n "$@" ]]; do +while [[ $# -gt 0 ]]; do arg="$1"; shift - val="$1"; shift case "$arg" in - "--testee") TESTEE="$val" ;; - "--failure_list") FAILURE_LIST="$val" ;; - "--text_format_failure_list") TEXT_FORMAT_FAILURE_LIST="$val" ;; - "--maximum_edition") MAXIMUM_EDITION="$val" ;; + "--testee") TESTEE="$1"; shift ;; + "--failure_list") FAILURE_LIST="$1"; shift ;; + "--text_format_failure_list") TEXT_FORMAT_FAILURE_LIST="$1"; shift ;; + "--maximum_edition") MAXIMUM_EDITION="$1"; shift ;; + "--performance") PERFORMANCE=1 ;; *) echo "Flag $arg is not recognized." && exit 1 ;; esac done @@ -63,4 +64,8 @@ if [ -n "$MAXIMUM_EDITION" ]; then args+=(--maximum_edition $MAXIMUM_EDITION) fi +if [ -n "$PERFORMANCE" ]; then + args+=(--performance) +fi + $conformance_test_runner "${args[@]}" $conformance_testee diff --git a/conformance/defs.bzl b/conformance/defs.bzl index 2ef5f4484a227..969733c9b4573 100644 --- a/conformance/defs.bzl +++ b/conformance/defs.bzl @@ -48,6 +48,6 @@ def conformance_test( deps = [ Label("@bazel_tools//tools/bash/runfiles"), ], - tags = ["conformance"], + tags = kwargs.pop("tags", []) + ["conformance"] + (["benchmark"] if performance else []), **kwargs ) diff --git a/csharp/BUILD.bazel b/csharp/BUILD.bazel index 4bca560c65134..49526342a7cd0 100644 --- a/csharp/BUILD.bazel +++ b/csharp/BUILD.bazel @@ -97,10 +97,18 @@ inline_sh_test( ":srcs", "//conformance:conformance_csharp_proto", "//csharp/src/Google.Protobuf.Conformance:srcs", + "//csharp/protos/unittest_deep_dependencies:cached_csharp_gen", + "//csharp/protos/unittest_deep_dependencies:notcached_csharp_gen", ], cmd = """ - cp $(rootpath //conformance:conformance_csharp_proto) `dirname $(location src/Google.Protobuf.sln)`/Google.Protobuf.Conformance/ - pushd `dirname $(location src/Google.Protobuf.sln)`/.. + export DOTNET_CLI_HOME=/tmp + SRCDIR=`dirname $(rootpath src/Google.Protobuf.sln)` + cp $(rootpath //conformance:conformance_csharp_proto) $$SRCDIR/Google.Protobuf.Conformance/ + mkdir -p $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached + mkdir -p $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached + cp $(rootpaths //csharp/protos/unittest_deep_dependencies:cached_csharp_gen) $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached/ + cp $(rootpaths //csharp/protos/unittest_deep_dependencies:notcached_csharp_gen) $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached/ + pushd $$SRCDIR/.. dotnet restore src/Google.Protobuf.sln dotnet build -c Release src/Google.Protobuf.sln dotnet test -c Release -f net8.0 src/Google.Protobuf.Test/Google.Protobuf.Test.csproj diff --git a/csharp/compatibility_tests/v3.0.0/BUILD.bazel b/csharp/compatibility_tests/v3.0.0/BUILD.bazel index ae5d92da381f0..3ffdccc1718b2 100644 --- a/csharp/compatibility_tests/v3.0.0/BUILD.bazel +++ b/csharp/compatibility_tests/v3.0.0/BUILD.bazel @@ -5,6 +5,10 @@ inline_sh_test( name = "tests", srcs = ["test.sh"], cmd = "$(location test.sh) 3.0.0 $(rootpath //:protoc)", + tags = [ + "manual", + "requires-network", + ], tools = ["//:protoc"], deps = glob([ "src/**/*.cs*", # .cs and .csproj diff --git a/csharp/protos/unittest_deep_dependencies/generate_protos.bzl b/csharp/protos/unittest_deep_dependencies/generate_protos.bzl index 59e1ec0293a0a..e7aee528a1eae 100644 --- a/csharp/protos/unittest_deep_dependencies/generate_protos.bzl +++ b/csharp/protos/unittest_deep_dependencies/generate_protos.bzl @@ -1,5 +1,4 @@ """ -Macro to generate exponential dependency proto structure for both cached and notcached variants. Generates proto files into separate subdirectories without duplication. """ @@ -57,7 +56,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_file_00_00_proto", srcs = [":" + package + "/file_00_00.proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", ) # Level 1: WIDTH proto_library targets @@ -66,7 +65,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_file_01_" + i_pad + "_proto", srcs = [":" + package + "/file_01_" + i_pad + ".proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", deps = [":" + variant + "_file_00_00_proto"], exports = [":" + variant + "_file_00_00_proto"], ) @@ -87,7 +86,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_file_" + level_pad + "_" + idx_pad + "_proto", srcs = [":" + package + "/file_" + level_pad + "_" + idx_pad + ".proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", deps = deps, exports = deps, ) @@ -96,7 +95,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_entry_proto", srcs = [":" + package + "/file_entry.proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", deps = [":" + variant + "_file_" + _pad(depth) + "_00_proto"], exports = [":" + variant + "_file_" + _pad(depth) + "_00_proto"], ) @@ -243,17 +242,16 @@ def _generate_csharp_sources(name, variant, all_files): # Invoke protoc hermetically with the correct base namespace and include path native.genrule( name = name, - testonly = True, srcs = [":" + f for f in all_files], outs = csharp_outputs, cmd = ( - "$(location //net/proto2/compiler/public:protocol_compiler) " + + "$(location //:protoc) " + "--csharp_out=$(RULEDIR)/" + variant + "_cs " + "--csharp_opt=file_extension=.pb.cs " + "--csharp_opt=base_namespace=" + base_namespace + " " + "-I$$(dirname $$(dirname $(location :" + all_files[0] + "))) " + "$(SRCS)" ), - tools = ["//net/proto2/compiler/public:protocol_compiler"], - visibility = ["//src/google/protobuf/csharp:__subpackages__"], + tools = ["//:protoc"], + visibility = ["//csharp:__subpackages__"], ) diff --git a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel index 93298dfeabc95..5650484d3700c 100644 --- a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel +++ b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel @@ -52,11 +52,17 @@ genrule( "Google.Protobuf.Conformance.csproj", "//conformance:conformance_csharp_proto", "//csharp:srcs", + "//csharp/protos/unittest_deep_dependencies:cached_csharp_gen", + "//csharp/protos/unittest_deep_dependencies:notcached_csharp_gen", ], outs = CONFORMANCE_DEPS + ["Google.Protobuf.Conformance.dll"], cmd = """ SRCDIR=$$(dirname $(location :Program.cs)) cp $(location //conformance:conformance_csharp_proto) $$SRCDIR + mkdir -p $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached + mkdir -p $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached + cp $(locations //csharp/protos/unittest_deep_dependencies:cached_csharp_gen) $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached/ + cp $(locations //csharp/protos/unittest_deep_dependencies:notcached_csharp_gen) $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached/ DOTNET_CLI_HOME=/tmp ./$(location :build_conformance_test) cp -r $$SRCDIR/bin/Release/net8.0/* $(RULEDIR) """, diff --git a/editions/BUILD b/editions/BUILD index 8e5c666792c40..9d467a6b890a6 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -383,8 +383,8 @@ exports_files( "golden/test_messages_proto3_editions.proto", ], visibility = [ + "//csharp:__pkg__", "//ruby:__pkg__", - "//src/google/protobuf/csharp:__pkg__", ], ) diff --git a/upb/bazel/private/oss/amalgamate.py b/upb/bazel/private/oss/amalgamate.py index f0cbde962600b..436760bfbd15e 100755 --- a/upb/bazel/private/oss/amalgamate.py +++ b/upb/bazel/private/oss/amalgamate.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Protocol Buffers - Google's data interchange format # Copyright 2023 Google LLC. All rights reserved. @@ -30,7 +30,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os import re import sys From 901255f88b15783b12669e9e36645f8d3fe0a4d5 Mon Sep 17 00:00:00 2001 From: Tony Liao Date: Wed, 26 Aug 2026 18:10:35 -0400 Subject: [PATCH 2/2] Add GitHub Actions CI workflow for monolithic container validation Configure .github/workflows/test_monolithic_container.yml to validate us-docker.pkg.dev/protobuf-build/containers/test/linux/monolithic:latest-pr199 across the full language test matrix (cpp, java, python, ruby, rust, csharp, php, bazel-all) on pushes to test/ci-monolithic-* and pull requests. Authenticate container runner with GAR_SERVICE_ACCOUNT credentials and pin actions/checkout to immutable commit SHA per Zizmor security policy. --- .../workflows/test_monolithic_container.yml | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .github/workflows/test_monolithic_container.yml diff --git a/.github/workflows/test_monolithic_container.yml b/.github/workflows/test_monolithic_container.yml new file mode 100644 index 0000000000000..bcacceb59265c --- /dev/null +++ b/.github/workflows/test_monolithic_container.yml @@ -0,0 +1,78 @@ +name: Monolithic Container Validation + +on: + push: + branches: + - 'test/ci-monolithic-*' + pull_request: + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +jobs: + monolithic-test-matrix: + name: Monolithic Container Validation (${{ matrix.suite }}) + runs-on: ubuntu-latest + container: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/monolithic:latest-pr199 + credentials: + username: _json_key + password: ${{ secrets.GAR_SERVICE_ACCOUNT }} + strategy: + fail-fast: false + matrix: + suite: [cpp, java, python, ruby, rust, csharp, php, bazel-all] + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Environment Sanity Check + run: | + echo "=== Monolithic Container Toolchain Sanity Check ===" + echo "Bazel: $(bazel --version 2>&1)" + echo "GCC: $(gcc --version | head -n 1)" + echo "Clang: $(clang --version | head -n 1)" + echo "Java: $(java -version 2>&1 | head -n 1)" + echo "Python: $(python3 --version 2>&1)" + echo "Ruby: $(ruby --version 2>&1)" + echo "PHP: $(php --version | head -n 1)" + echo ".NET: $(dotnet --version 2>&1)" + echo "Rust: $(rustc --version 2>&1)" + echo "Cargo: $(cargo --version 2>&1)" + echo "==================================================" + + - name: Run ${{ matrix.suite }} Test Suite + run: | + case "${{ matrix.suite }}" in + cpp) + bazel test //src/google/protobuf/... + ;; + java) + bazel test //java/... + ;; + python) + bazel test //python/... + ;; + ruby) + bazel test //ruby/... + ;; + rust) + bazel test //rust/... + ;; + csharp) + bazel test //csharp/... + ;; + php) + bazel test //php/... + ;; + bazel-all) + bazel test --test_tag_filters=-manual,-requires-network //... + ;; + *) + echo "Unknown suite: ${{ matrix.suite }}" + exit 1 + ;; + esac