Get Bazel building all of the APIs (istio#133)

* update bazel and go versions, add field_behavior_proto to the set of standard proto deps BUT NOT THE GO_PROTO_DEPS 🚨

* everything but install compiling with bazel

* add buildifier target

* run buildifier for the first time

* stashing state to explore making common-protos a sub-repo in bazel to fix path problems

revert unneeded changes

* get k8s.io building

* revert prefix strip change that broke the build, just pass it as kwargs for now

* run buildifier to clean up

* got istio.io building

* got install building - though currently the overall build fails as our protobuf validation is wrong. that'll be a follow-up

* rever the import change - this does break the bazel build here but I can't figure out how to keep it working in the old build too

* finally get the build working with both bazel and make!

* update operator build to get it _really really_ working - unfortunately this means basically an entirely hand-written build file for just this one spot

* buildifier

Author: ZackButcher

.bazelversion

@@ -1 +1 @@
-2.2.0
+3.2.0

.gitignore

@@ -10,3 +10,4 @@ gen.warnings
 private/iam/v1/*.pb.html
 tsb/private/cli/v2/*.pb.html
 tsb/private/v2/*.pb.html
+bazel-*

BUILD.bazel

@@ -0,0 +1,5 @@
+load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier")
+
+buildifier(
+    name = "buildifier",
+)

WORKSPACE

@@ -39,10 +39,10 @@ rules_proto_toolchains()
 
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "142dd33e38b563605f0d20e89d9ef9eda0fc3cb539a14be1bdb1350de2eda659",
+    sha256 = "a8d6b1b354d371a646d2f7927319974e0f9e52f73a2452d2b3877118169eb6bb",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.23.3/rules_go-v0.23.3.tar.gz",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.23.3/rules_go-v0.23.3.tar.gz",
     ],
 )
 
@@ -51,3 +51,11 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 go_rules_dependencies()
 
 go_register_toolchains()
+
+# buildifier
+http_archive(
+    name = "com_github_bazelbuild_buildtools",
+    sha256 = "21e3f4cc034f01ca74643133fc9986cbf0bf43b7a0a0b150f0967e72b22b9132",
+    strip_prefix = "buildtools-3.2.1",
+    urls = ["https://github.com/bazelbuild/buildtools/archive/3.2.1.tar.gz"],
+)

audit/v1/BUILD.bazel

@@ -0,0 +1,3 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package()

authz/v1/BUILD.bazel

@@ -0,0 +1,3 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package()

bazel/api_build_system.bzl

@@ -16,12 +16,31 @@ _COMMON_PROTO_DEPS = [
     "@com_google_protobuf//:struct_proto",
     "@com_google_protobuf//:timestamp_proto",
     "@com_google_protobuf//:wrappers_proto",
+    "@com_google_googleapis//google/api:annotations_proto",
     "@com_google_googleapis//google/api:http_proto",
     "@com_google_googleapis//google/api:httpbody_proto",
-    "@com_google_googleapis//google/api:annotations_proto",
+    "@com_google_googleapis//google/api:field_behavior_proto",
     "@com_google_googleapis//google/rpc:status_proto",
     "@com_envoyproxy_protoc_gen_validate//validate:validate_proto",
 ]
+_COMMON_GO_DEPS = [
+    "@com_github_golang_protobuf//ptypes:go_default_library",
+    "@com_github_golang_protobuf//ptypes/any:go_default_library",
+    "@com_github_golang_protobuf//ptypes/duration:go_default_library",
+    "@com_github_golang_protobuf//ptypes/struct:go_default_library",
+    "@com_github_golang_protobuf//ptypes/timestamp:go_default_library",
+    "@com_github_golang_protobuf//ptypes/wrappers:go_default_library",
+    "@com_envoyproxy_protoc_gen_validate//validate:go_default_library",
+    "@com_google_googleapis//google/api:annotations_go_proto",
+    # Do _not_ include :field_behavior_go_proto dep even though we include it as a
+    # dep in _COMMON_PROTO_DEPS: it breaks the build with an invalid import, which
+    # we get from :annotations_go_proto anyway.
+    # "@com_google_googleapis//google/api:field_behavior_go_proto",
+    "@com_google_googleapis//google/rpc:status_go_proto",
+]
+
+EXTERNAL_PROTO_GO_DEP_MAP = {
+}
 
 def _proto_mapping(dep, proto_dep_map, proto_suffix):
     mapped = proto_dep_map.get(dep)
@@ -31,15 +50,19 @@ def _proto_mapping(dep, proto_dep_map, proto_suffix):
     return mapped
 
 def _go_proto_mapping(dep):
-    return _proto_mapping(dep, EXTERNAL_PROTO_GO_BAZEL_DEP_MAP, _GO_PROTO_SUFFIX)
+    # merge our local map and the imported map from @envoy_api//bazel
+    full_map = dict(EXTERNAL_PROTO_GO_DEP_MAP.items() + EXTERNAL_PROTO_GO_BAZEL_DEP_MAP.items())
+    return _proto_mapping(dep, full_map, _GO_PROTO_SUFFIX)
 
 # API proto package
-def api_proto_package(
-        name = "pkg",
-        srcs = [],
-        deps = [],
-        has_services = False,
-        visibility = ["//visibility:public"]):
+def _api_proto_package(
+        name,
+        srcs,
+        proto_deps,
+        go_deps,
+        has_services,
+        visibility,
+        **kwargs):
     """Defines a API proto package.
 
     Args:
@@ -48,15 +71,19 @@ def api_proto_package(
       deps: dependencies
       has_services: whether .proto file has services
       visibility: Bazel visibility
+      kwargs: other arguments to pass to proto_library (they are not passed to go_proto_library)
     """
     if srcs == []:
         srcs = native.glob(["*.proto"])
 
+    # print("%s using sources %s" % (name, srcs))
+
     proto_library(
         name = name,
         srcs = srcs,
-        deps = deps + _COMMON_PROTO_DEPS,
+        deps = proto_deps,
         visibility = visibility,
+        **kwargs
     )
 
     compilers = ["@io_bazel_rules_go//proto:go_proto", "@envoy_api//bazel:pgv_plugin_go"]
@@ -66,17 +93,57 @@ def api_proto_package(
         name = name + _GO_PROTO_SUFFIX,
         compilers = compilers,
         importpath = _GO_IMPORTPATH_PREFIX + native.package_name(),
-        proto = name,
+        protos = [":" + name],
         visibility = ["//visibility:public"],
-        deps = [_go_proto_mapping(dep) for dep in deps] + [
-            "@com_github_golang_protobuf//ptypes:go_default_library",
-            "@com_github_golang_protobuf//ptypes/any:go_default_library",
-            "@com_github_golang_protobuf//ptypes/duration:go_default_library",
-            "@com_github_golang_protobuf//ptypes/struct:go_default_library",
-            "@com_github_golang_protobuf//ptypes/timestamp:go_default_library",
-            "@com_github_golang_protobuf//ptypes/wrappers:go_default_library",
-            "@com_envoyproxy_protoc_gen_validate//validate:go_default_library",
-            "@com_google_googleapis//google/api:annotations_go_proto",
-            "@com_google_googleapis//google/rpc:status_go_proto",
-        ],
+        deps = go_deps,
     )
+
+def api_proto_package(
+        name = "pkg",
+        srcs = [],
+        deps = [],
+        has_services = False,
+        visibility = ["//visibility:public"],
+        **kwargs):
+    """Defines a API proto package.
+
+    Args:
+      name: package name
+      srcs: source files if not specified all .proto files in the directory
+      deps: dependencies
+      has_services: whether .proto file has services
+      visibility: Bazel visibility
+      strip_import_prefix: prefix to remove for protobufs in this target (only applies to proto_library rules)
+    """
+    proto_deps = deps + _COMMON_PROTO_DEPS
+    go_deps = [_go_proto_mapping(dep) for dep in deps] + _COMMON_GO_DEPS
+    return _api_proto_package(name, srcs, proto_deps, go_deps, has_services, visibility, **kwargs)
+
+def k8s_api_proto_package(
+        name = "pkg",
+        srcs = [],
+        deps = [],
+        has_services = False,
+        visibility = ["//visibility:public"],
+        strip_import_prefix = "//common-protos"):
+    """Defines a API proto package for Kubernetes API resources. This assumes a few extra dependencies
+    compared to api_proto_package, as well as stripping of the `//common-protos` prefix for import.
+
+    Args:
+      name: package name
+      srcs: source files if not specified all .proto files in the directory
+      deps: dependencies
+      has_services: whether .proto file has services
+      visibility: Bazel visibility
+    """
+    _common_api_k8s_deps = [
+        "//common-protos/k8s.io/apimachinery/pkg/apis/meta/v1:pkg",
+        "//common-protos/k8s.io/apimachinery/pkg/runtime:pkg",
+        "//common-protos/k8s.io/apimachinery/pkg/runtime/schema:pkg",
+    ]
+
+    proto_deps = deps + _COMMON_PROTO_DEPS + _common_api_k8s_deps
+
+    # note we run the _common_api_k8s_deps through _go_proto_mapping too (but not _COMMON_GO_DEPS)
+    go_deps = [_go_proto_mapping(dep) for dep in deps + _common_api_k8s_deps] + _COMMON_GO_DEPS
+    return _api_proto_package(name, srcs, proto_deps, go_deps, has_services, visibility, strip_import_prefix = strip_import_prefix)

common-protos/gogoproto/BUILD.bazel

@@ -0,0 +1,5 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos",
+)

common-protos/istio.io/api/authentication/v1alpha1/BUILD.bazel

@@ -0,0 +1,5 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+)

common-protos/istio.io/api/envoy/config/filter/http/alpn/v2alpha1/BUILD.bazel

@@ -0,0 +1,3 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package()

common-protos/istio.io/api/envoy/config/filter/http/authn/v2alpha1/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/istio.io/api/authentication/v1alpha1:pkg",
+    ],
+)

common-protos/istio.io/api/envoy/config/filter/http/jwt_auth/v2alpha1/BUILD.bazel

@@ -0,0 +1,3 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package()

common-protos/istio.io/api/envoy/config/filter/network/tcp_cluster_rewrite/v2alpha1/BUILD.bazel

@@ -0,0 +1,3 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package()

common-protos/istio.io/api/mcp/v1alpha1/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/gogoproto:pkg",
+    ],
+)

common-protos/istio.io/api/mesh/v1alpha1/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/istio.io/api/networking/v1alpha3:pkg",
+    ],
+)

common-protos/istio.io/api/mixer/adapter/model/v1beta1/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/gogoproto:pkg",
+    ],
+)

common-protos/istio.io/api/mixer/v1/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/gogoproto:pkg",
+    ],
+)

common-protos/istio.io/api/mixer/v1/config/client/BUILD.bazel

@@ -0,0 +1,9 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/gogoproto:pkg",
+        "//common-protos/istio.io/api/mixer/v1:pkg",
+    ],
+)

common-protos/istio.io/api/networking/v1alpha3/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/gogoproto:pkg",
+    ],
+)

common-protos/istio.io/api/networking/v1beta1/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/gogoproto:pkg",
+    ],
+)

common-protos/istio.io/api/operator/v1alpha1/BUILD.bazel

@@ -0,0 +1,86 @@
+load("@rules_proto//proto:defs.bzl", "proto_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+
+# Beware, this file is written by hand and does not use api_proto_package.
+# We have to do that in this one place because these protos alone in the
+# entire codebase are imported with 2 names:
+#   internally, Istio API protos refer to eachother as `import "operator/compnent.proto"`
+#   externally, we reference those same protos as `import "istio.io/api/operator/component.proto"`
+# to support both of these at the same time, we need some refined targets to make the build work.
+# More notes inline, but the gist is that for the golang part of the build to succeed, we have
+# to have only a single go_proto_library rule (or we wind up with problems around go packages).
+
+proto_library(
+    name = "component_proto",
+    srcs = ["component.proto"],
+    strip_import_prefix = "//common-protos",  # enables import as "istio.io/api/operator/v1alpha1"
+    visibility = ["//visibility:public"],
+    deps = [
+        "//common-protos/istio.io/api/operator/v1alpha1:kubernetes_proto",
+        "//common-protos/k8s.io/api/core/v1:pkg",
+        "@com_google_protobuf//:any_proto",
+    ],
+)
+
+# this rule is the hand-written version of the go_proto_library that `api_proto_package`
+# generated for `component_proto` above, with the exception that we include
+# `:kubernetes_proto` here. This is because we can only have a single go_proto_library.
+go_proto_library(
+    name = "component_proto_go_proto",
+    compilers = [
+        "@io_bazel_rules_go//proto:go_proto",
+        "@envoy_api//bazel:pgv_plugin_go",
+    ],
+    importpath = "github.com/tetrateio/api-gocommon-protos/istio.io/api/operator/v1alpha1",
+    # this is the dirty part: we include ":kubernetes_proto" here for the golang build to work;
+    # we can do this because go_proto_library ignores `strip_import_prefix`. If we tried to
+    # pull in operator.proto, we'll have to rework how we attempt this (and probably just change
+    # the import in our proto to remove the need to do all this craziness).
+    protos = [
+        ":component_proto",
+        ":kubernetes_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//common-protos/k8s.io/api/core/v1:pkg_go_proto",
+        "//common-protos/k8s.io/apimachinery/pkg/api/resource:pkg_go_proto",
+        "//common-protos/k8s.io/apimachinery/pkg/apis/meta/v1:pkg_go_proto",
+        "//common-protos/k8s.io/apimachinery/pkg/util/intstr:pkg_go_proto",
+        "@com_github_golang_protobuf//ptypes:go_default_library",
+        "@com_github_golang_protobuf//ptypes/any:go_default_library",
+    ],
+)
+
+proto_library(
+    name = "kubernetes_proto",
+    srcs = ["kubernetes.proto"],
+    strip_import_prefix = "//common-protos/istio.io/api",  # enables import as "istio.io/api/operator/v1alpha1"
+    visibility = ["//visibility:private"],
+    deps = [
+        "//common-protos/k8s.io/apimachinery/pkg/api/resource:pkg",
+        "//common-protos/k8s.io/apimachinery/pkg/apis/meta/v1:pkg",
+        "//common-protos/k8s.io/apimachinery/pkg/util/intstr:pkg",
+    ],
+)
+
+proto_library(
+    name = "operator_proto",
+    srcs = ["operator.proto"],
+    strip_import_prefix = "//common-protos/istio.io/api",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//common-protos/istio.io/api/operator/v1alpha1:component_proto_internal",
+    ],
+)
+
+proto_library(
+    name = "component_proto_internal",
+    srcs = ["component.proto"],
+    strip_import_prefix = "//common-protos/istio.io/api",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//common-protos/istio.io/api/operator/v1alpha1:kubernetes_proto",
+        "//common-protos/k8s.io/api/core/v1:pkg",
+        "@com_google_protobuf//:any_proto",
+    ],
+)

common-protos/istio.io/api/policy/v1beta1/BUILD.bazel

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+    deps = [
+        "//common-protos/gogoproto:pkg",
+    ],
+)

common-protos/istio.io/api/rbac/v1alpha1/BUILD.bazel

@@ -0,0 +1,5 @@
+load("//bazel:api_build_system.bzl", "api_proto_package")
+
+api_proto_package(
+    strip_import_prefix = "//common-protos/istio.io/api",
+)