diff --git a/.bazelignore b/.bazelignore index 69fa449dd96e2..2738ca30273c6 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1 +1,2 @@ _build/ +examples diff --git a/.github/workflows/test_monolithic_container.yml b/.github/workflows/test_monolithic_container.yml new file mode 100644 index 0000000000000..bcacceb59265c --- /dev/null +++ b/.github/workflows/test_monolithic_container.yml @@ -0,0 +1,78 @@ +name: Monolithic Container Validation + +on: + push: + branches: + - 'test/ci-monolithic-*' + pull_request: + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +jobs: + monolithic-test-matrix: + name: Monolithic Container Validation (${{ matrix.suite }}) + runs-on: ubuntu-latest + container: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/monolithic:latest-pr199 + credentials: + username: _json_key + password: ${{ secrets.GAR_SERVICE_ACCOUNT }} + strategy: + fail-fast: false + matrix: + suite: [cpp, java, python, ruby, rust, csharp, php, bazel-all] + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Environment Sanity Check + run: | + echo "=== Monolithic Container Toolchain Sanity Check ===" + echo "Bazel: $(bazel --version 2>&1)" + echo "GCC: $(gcc --version | head -n 1)" + echo "Clang: $(clang --version | head -n 1)" + echo "Java: $(java -version 2>&1 | head -n 1)" + echo "Python: $(python3 --version 2>&1)" + echo "Ruby: $(ruby --version 2>&1)" + echo "PHP: $(php --version | head -n 1)" + echo ".NET: $(dotnet --version 2>&1)" + echo "Rust: $(rustc --version 2>&1)" + echo "Cargo: $(cargo --version 2>&1)" + echo "==================================================" + + - name: Run ${{ matrix.suite }} Test Suite + run: | + case "${{ matrix.suite }}" in + cpp) + bazel test //src/google/protobuf/... + ;; + java) + bazel test //java/... + ;; + python) + bazel test //python/... + ;; + ruby) + bazel test //ruby/... + ;; + rust) + bazel test //rust/... + ;; + csharp) + bazel test //csharp/... + ;; + php) + bazel test //php/... + ;; + bazel-all) + bazel test --test_tag_filters=-manual,-requires-network //... + ;; + *) + echo "Unknown suite: ${{ matrix.suite }}" + exit 1 + ;; + esac diff --git a/bazel/private/oss/cc_proto_library.bzl b/bazel/private/oss/cc_proto_library.bzl index 19eb7055f69b6..9a48292c70147 100644 --- a/bazel/private/oss/cc_proto_library.bzl +++ b/bazel/private/oss/cc_proto_library.bzl @@ -77,37 +77,41 @@ def _aspect_impl(target, ctx): transitive_headers.append(dep[_ProtoCcHeaderInfo].headers) header_provider = _ProtoCcHeaderInfo(headers = depset(transitive = transitive_headers)) + proto_common.compile( + actions = ctx.actions, + proto_info = proto_info, + proto_lang_toolchain_info = proto_toolchain, + generated_files = sources + headers, + experimental_output_files = "multiple", + ) + + deps = [] + if proto_toolchain.runtime: + deps = [proto_toolchain.runtime] + deps.extend(getattr(ctx.rule.attr, "deps", [])) + + cc_info, libraries, temps = cc_proto_compile_and_link( + ctx = ctx, + deps = deps, + sources = sources, + headers = headers, + textual_hdrs = textual_hdrs, + strip_include_prefix = _get_strip_include_prefix(ctx, proto_info), + ) + + return [ + cc_info, + _ProtoCcFilesInfo(files = depset(sources + headers + libraries)), + OutputGroupInfo(temp_files_INTERNAL_ = temps), + header_provider, + ] else: # shouldn't generate code - header_provider = _ProtoCcHeaderInfo(headers = depset()) - - proto_common.compile( - actions = ctx.actions, - proto_info = proto_info, - proto_lang_toolchain_info = proto_toolchain, - generated_files = sources + headers, - experimental_output_files = "multiple", - ) - - deps = [] - if proto_toolchain.runtime: - deps = [proto_toolchain.runtime] - deps.extend(getattr(ctx.rule.attr, "deps", [])) - - cc_info, libraries, temps = cc_proto_compile_and_link( - ctx = ctx, - deps = deps, - sources = sources, - headers = headers, - textual_hdrs = textual_hdrs, - strip_include_prefix = _get_strip_include_prefix(ctx, proto_info), - ) - - return [ - cc_info, - _ProtoCcFilesInfo(files = depset(sources + headers + libraries)), - OutputGroupInfo(temp_files_INTERNAL_ = temps), - header_provider, - ] + return [ + CcInfo(), + _ProtoCcFilesInfo(files = depset()), + OutputGroupInfo(), + _ProtoCcHeaderInfo(headers = depset()), + ] cc_proto_aspect = aspect( implementation = _aspect_impl, diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel index 8591747933a43..5efc30f29a291 100644 --- a/build_defs/BUILD.bazel +++ b/build_defs/BUILD.bazel @@ -224,17 +224,7 @@ starlark_cc_proto_library( cc_proto_blacklist_test( name = "cc_proto_blacklist_test", deps = [ - ":any_cc_proto", - ":api_cc_proto", ":compiler_plugin_cc_proto", ":descriptor_cc_proto", - ":duration_cc_proto", - ":empty_cc_proto", - ":field_mask_cc_proto", - ":source_context_cc_proto", - ":struct_cc_proto", - ":timestamp_cc_proto", - ":type_cc_proto", - ":wrappers_cc_proto", ], ) diff --git a/conformance/bazel_conformance_test_runner.sh b/conformance/bazel_conformance_test_runner.sh index e3d4c6777081c..bb88a91d37dee 100755 --- a/conformance/bazel_conformance_test_runner.sh +++ b/conformance/bazel_conformance_test_runner.sh @@ -32,15 +32,16 @@ TESTEE= FAILURE_LIST= TEXT_FORMAT_FAILURE_LIST= MAXIMUM_EDITION= +PERFORMANCE= -while [[ -n "$@" ]]; do +while [[ $# -gt 0 ]]; do arg="$1"; shift - val="$1"; shift case "$arg" in - "--testee") TESTEE="$val" ;; - "--failure_list") FAILURE_LIST="$val" ;; - "--text_format_failure_list") TEXT_FORMAT_FAILURE_LIST="$val" ;; - "--maximum_edition") MAXIMUM_EDITION="$val" ;; + "--testee") TESTEE="$1"; shift ;; + "--failure_list") FAILURE_LIST="$1"; shift ;; + "--text_format_failure_list") TEXT_FORMAT_FAILURE_LIST="$1"; shift ;; + "--maximum_edition") MAXIMUM_EDITION="$1"; shift ;; + "--performance") PERFORMANCE=1 ;; *) echo "Flag $arg is not recognized." && exit 1 ;; esac done @@ -63,4 +64,8 @@ if [ -n "$MAXIMUM_EDITION" ]; then args+=(--maximum_edition $MAXIMUM_EDITION) fi +if [ -n "$PERFORMANCE" ]; then + args+=(--performance) +fi + $conformance_test_runner "${args[@]}" $conformance_testee diff --git a/conformance/defs.bzl b/conformance/defs.bzl index 2ef5f4484a227..969733c9b4573 100644 --- a/conformance/defs.bzl +++ b/conformance/defs.bzl @@ -48,6 +48,6 @@ def conformance_test( deps = [ Label("@bazel_tools//tools/bash/runfiles"), ], - tags = ["conformance"], + tags = kwargs.pop("tags", []) + ["conformance"] + (["benchmark"] if performance else []), **kwargs ) diff --git a/csharp/BUILD.bazel b/csharp/BUILD.bazel index 4bca560c65134..49526342a7cd0 100644 --- a/csharp/BUILD.bazel +++ b/csharp/BUILD.bazel @@ -97,10 +97,18 @@ inline_sh_test( ":srcs", "//conformance:conformance_csharp_proto", "//csharp/src/Google.Protobuf.Conformance:srcs", + "//csharp/protos/unittest_deep_dependencies:cached_csharp_gen", + "//csharp/protos/unittest_deep_dependencies:notcached_csharp_gen", ], cmd = """ - cp $(rootpath //conformance:conformance_csharp_proto) `dirname $(location src/Google.Protobuf.sln)`/Google.Protobuf.Conformance/ - pushd `dirname $(location src/Google.Protobuf.sln)`/.. + export DOTNET_CLI_HOME=/tmp + SRCDIR=`dirname $(rootpath src/Google.Protobuf.sln)` + cp $(rootpath //conformance:conformance_csharp_proto) $$SRCDIR/Google.Protobuf.Conformance/ + mkdir -p $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached + mkdir -p $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached + cp $(rootpaths //csharp/protos/unittest_deep_dependencies:cached_csharp_gen) $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached/ + cp $(rootpaths //csharp/protos/unittest_deep_dependencies:notcached_csharp_gen) $$SRCDIR/Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached/ + pushd $$SRCDIR/.. dotnet restore src/Google.Protobuf.sln dotnet build -c Release src/Google.Protobuf.sln dotnet test -c Release -f net8.0 src/Google.Protobuf.Test/Google.Protobuf.Test.csproj diff --git a/csharp/compatibility_tests/v3.0.0/BUILD.bazel b/csharp/compatibility_tests/v3.0.0/BUILD.bazel index ae5d92da381f0..3ffdccc1718b2 100644 --- a/csharp/compatibility_tests/v3.0.0/BUILD.bazel +++ b/csharp/compatibility_tests/v3.0.0/BUILD.bazel @@ -5,6 +5,10 @@ inline_sh_test( name = "tests", srcs = ["test.sh"], cmd = "$(location test.sh) 3.0.0 $(rootpath //:protoc)", + tags = [ + "manual", + "requires-network", + ], tools = ["//:protoc"], deps = glob([ "src/**/*.cs*", # .cs and .csproj diff --git a/csharp/protos/unittest_deep_dependencies/generate_protos.bzl b/csharp/protos/unittest_deep_dependencies/generate_protos.bzl index 59e1ec0293a0a..e7aee528a1eae 100644 --- a/csharp/protos/unittest_deep_dependencies/generate_protos.bzl +++ b/csharp/protos/unittest_deep_dependencies/generate_protos.bzl @@ -1,5 +1,4 @@ """ -Macro to generate exponential dependency proto structure for both cached and notcached variants. Generates proto files into separate subdirectories without duplication. """ @@ -57,7 +56,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_file_00_00_proto", srcs = [":" + package + "/file_00_00.proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", ) # Level 1: WIDTH proto_library targets @@ -66,7 +65,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_file_01_" + i_pad + "_proto", srcs = [":" + package + "/file_01_" + i_pad + ".proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", deps = [":" + variant + "_file_00_00_proto"], exports = [":" + variant + "_file_00_00_proto"], ) @@ -87,7 +86,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_file_" + level_pad + "_" + idx_pad + "_proto", srcs = [":" + package + "/file_" + level_pad + "_" + idx_pad + ".proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", deps = deps, exports = deps, ) @@ -96,7 +95,7 @@ def _generate_variant(variant, width, depth, package): proto_library( name = variant + "_entry_proto", srcs = [":" + package + "/file_entry.proto"], - strip_import_prefix = "/third_party/protobuf/csharp/protos/unittest_deep_dependencies", + strip_import_prefix = "/csharp/protos/unittest_deep_dependencies", deps = [":" + variant + "_file_" + _pad(depth) + "_00_proto"], exports = [":" + variant + "_file_" + _pad(depth) + "_00_proto"], ) @@ -243,17 +242,16 @@ def _generate_csharp_sources(name, variant, all_files): # Invoke protoc hermetically with the correct base namespace and include path native.genrule( name = name, - testonly = True, srcs = [":" + f for f in all_files], outs = csharp_outputs, cmd = ( - "$(location //net/proto2/compiler/public:protocol_compiler) " + + "$(location //:protoc) " + "--csharp_out=$(RULEDIR)/" + variant + "_cs " + "--csharp_opt=file_extension=.pb.cs " + "--csharp_opt=base_namespace=" + base_namespace + " " + "-I$$(dirname $$(dirname $(location :" + all_files[0] + "))) " + "$(SRCS)" ), - tools = ["//net/proto2/compiler/public:protocol_compiler"], - visibility = ["//src/google/protobuf/csharp:__subpackages__"], + tools = ["//:protoc"], + visibility = ["//csharp:__subpackages__"], ) diff --git a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel index 93298dfeabc95..5650484d3700c 100644 --- a/csharp/src/Google.Protobuf.Conformance/BUILD.bazel +++ b/csharp/src/Google.Protobuf.Conformance/BUILD.bazel @@ -52,11 +52,17 @@ genrule( "Google.Protobuf.Conformance.csproj", "//conformance:conformance_csharp_proto", "//csharp:srcs", + "//csharp/protos/unittest_deep_dependencies:cached_csharp_gen", + "//csharp/protos/unittest_deep_dependencies:notcached_csharp_gen", ], outs = CONFORMANCE_DEPS + ["Google.Protobuf.Conformance.dll"], cmd = """ SRCDIR=$$(dirname $(location :Program.cs)) cp $(location //conformance:conformance_csharp_proto) $$SRCDIR + mkdir -p $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached + mkdir -p $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached + cp $(locations //csharp/protos/unittest_deep_dependencies:cached_csharp_gen) $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesCached/ + cp $(locations //csharp/protos/unittest_deep_dependencies:notcached_csharp_gen) $$SRCDIR/../Google.Protobuf.Test.TestProtos/UnittestDeepDependenciesNotCached/ DOTNET_CLI_HOME=/tmp ./$(location :build_conformance_test) cp -r $$SRCDIR/bin/Release/net8.0/* $(RULEDIR) """, diff --git a/editions/BUILD b/editions/BUILD index 8e5c666792c40..9d467a6b890a6 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -383,8 +383,8 @@ exports_files( "golden/test_messages_proto3_editions.proto", ], visibility = [ + "//csharp:__pkg__", "//ruby:__pkg__", - "//src/google/protobuf/csharp:__pkg__", ], ) diff --git a/upb/bazel/private/oss/amalgamate.py b/upb/bazel/private/oss/amalgamate.py index f0cbde962600b..436760bfbd15e 100755 --- a/upb/bazel/private/oss/amalgamate.py +++ b/upb/bazel/private/oss/amalgamate.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # # Protocol Buffers - Google's data interchange format # Copyright 2023 Google LLC. All rights reserved. @@ -30,7 +30,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os import re import sys