Skip to content

Commit dcc7a8b

Browse files
hawkinsphawkinsp
committed
Build protoc from source, rather than using a binary protoc.
Migrate to using the Bazel rules_proto and rules_python packages to build protocol buffers, rather than rolling our own protobuf generation code. Fixes #327
1 parent c00a74b commit dcc7a8b

File tree

5 files changed

+103
-70
lines changed

5 files changed

+103
-70
lines changed

‎WORKSPACE

+38-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Workspace file for lingvo."""
22

33
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
4-
load("//lingvo:repo.bzl", "cc_tf_configure", "icu", "lingvo_protoc_deps", "lingvo_testonly_deps")
4+
load("//lingvo:repo.bzl", "cc_tf_configure", "icu", "lingvo_testonly_deps")
55

66
http_archive(
77
name = "org_tensorflow",
@@ -14,10 +14,45 @@ load("@org_tensorflow//tensorflow/tools/toolchains/remote_config:configs.bzl", "
1414

1515
initialize_rbe_configs()
1616

17+
http_archive(
18+
name = "rules_python",
19+
sha256 = "5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036",
20+
strip_prefix = "rules_python-0.25.0",
21+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.25.0/rules_python-0.25.0.tar.gz",
22+
)
23+
24+
load("@rules_python//python:repositories.bzl", "py_repositories")
25+
26+
py_repositories()
27+
28+
http_archive(
29+
name = "com_google_protobuf",
30+
sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae",
31+
strip_prefix = "protobuf-21.7",
32+
urls = [
33+
"https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
34+
"https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
35+
],
36+
)
37+
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
38+
39+
protobuf_deps()
40+
41+
http_archive(
42+
name = "rules_proto",
43+
sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
44+
strip_prefix = "rules_proto-5.3.0-21.7",
45+
urls = [
46+
"https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz",
47+
],
48+
)
49+
load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
50+
rules_proto_dependencies()
51+
rules_proto_toolchains()
52+
53+
1754
cc_tf_configure()
1855

1956
lingvo_testonly_deps()
2057

21-
lingvo_protoc_deps()
22-
2358
icu()

‎lingvo/BUILD

+31-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ load(
55

66
# Placeholder: load py_library
77
# Placeholder: load py_test
8-
load("//lingvo:lingvo.bzl", "pytype_library", "pytype_strict_library")
8+
load("//lingvo:lingvo.bzl", "pytype_library", "pytype_strict_library",
9+
"WELL_KNOWN_PROTO_LIBS")
910

1011
package(default_visibility = ["//visibility:public"])
1112

@@ -319,10 +320,38 @@ lingvo_py_binary(
319320
],
320321
)
321322

323+
TF_PROTOS = [
324+
"tensorflow/core/framework/tensor.proto",
325+
"tensorflow/core/framework/versions.proto",
326+
"tensorflow/core/framework/op_def.proto",
327+
"tensorflow/core/framework/resource_handle.proto",
328+
"tensorflow/core/framework/function.proto",
329+
"tensorflow/core/framework/graph_debug_info.proto",
330+
"tensorflow/core/framework/node_def.proto",
331+
"tensorflow/core/framework/graph.proto",
332+
"tensorflow/core/framework/attr_value.proto",
333+
"tensorflow/core/framework/variable.proto",
334+
"tensorflow/core/framework/full_type.proto",
335+
"tensorflow/core/framework/tensor_shape.proto",
336+
"tensorflow/core/framework/types.proto",
337+
"tensorflow/core/protobuf/trackable_object_graph.proto",
338+
"tensorflow/core/protobuf/saver.proto",
339+
"tensorflow/core/protobuf/struct.proto",
340+
"tensorflow/core/protobuf/saved_object_graph.proto",
341+
"tensorflow/core/protobuf/meta_graph.proto",
342+
]
343+
344+
proto_library(
345+
name = "tf_protos",
346+
srcs = TF_PROTOS,
347+
strip_import_prefix = "/lingvo",
348+
deps = WELL_KNOWN_PROTO_LIBS,
349+
)
350+
322351
genrule(
323352
name = "tf_dot_protos",
324353
srcs = [],
325-
outs = ["tf_protos.tar"],
354+
outs = TF_PROTOS,
326355
cmd =
327356
"$(location //lingvo/tools:" +
328357
"generate_tf_dot_protos) $(location " +

‎lingvo/lingvo.bzl

+34-45
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
"""Implements custom rules for Lingvo."""
22

3+
load("@rules_python//python:proto.bzl", "py_proto_library")
4+
35
def tf_copts():
46
# TODO(drpng): autoconf this.
57
return ["-std=c++17", "-Wno-sign-compare", "-mavx"] + select({
@@ -107,65 +109,52 @@ def lingvo_cuda_py_test(name, tags = [], deps = [], **kwargs):
107109
**kwargs
108110
)
109111

110-
def _proto_gen_cc_src(name, basename):
111-
native.genrule(
112-
name = name,
113-
srcs = [basename + ".proto"],
114-
outs = [basename + ".pb.cc", basename + ".pb.h"],
115-
tools = [
116-
"@protobuf_protoc//:protoc_bin",
117-
"//lingvo:tf_dot_protos",
118-
],
119-
# TODO(drpng): only unpack if tf_proto dependency is requested.
120-
cmd = """
121-
mkdir -p $(@D)/tf_proto.$$$$;
122-
tar -C $(@D)/tf_proto.$$$$ -xf $(location //lingvo:tf_dot_protos);
123-
$(location @protobuf_protoc//:protoc_bin) --proto_path=$(@D)/tf_proto.$$$$ --proto_path=. --cpp_out=$(GENDIR) $(<);
124-
rm -rf $(@D)/tf_proto.$$$$
125-
""",
126-
)
127-
128-
def _proto_gen_py_src(name, basename):
129-
native.genrule(
130-
name = name,
131-
srcs = [basename + ".proto"],
132-
outs = [basename + "_pb2.py"],
133-
tools = [
134-
"@protobuf_protoc//:protoc_bin",
135-
"//lingvo:tf_dot_protos",
136-
],
137-
# TODO(drpng): only unpack if tf_proto dependency is requested.
138-
cmd = """
139-
mkdir -p $(@D)/tf_proto.$$$$;
140-
tar -C $(@D)/tf_proto.$$$$ -xf $(location //lingvo:tf_dot_protos);
141-
$(location @protobuf_protoc//:protoc_bin) --proto_path=$(@D)/tf_proto.$$$$ --proto_path=. --python_out=$(GENDIR) $(<);
142-
rm -rf $(@D)/tf_proto.$$$$
143-
""",
144-
)
112+
WELL_KNOWN_PROTO_LIBS = [
113+
"@com_google_protobuf//:any_proto",
114+
"@com_google_protobuf//:api_proto",
115+
"@com_google_protobuf//:compiler_plugin_proto",
116+
"@com_google_protobuf//:descriptor_proto",
117+
"@com_google_protobuf//:duration_proto",
118+
"@com_google_protobuf//:empty_proto",
119+
"@com_google_protobuf//:field_mask_proto",
120+
"@com_google_protobuf//:source_context_proto",
121+
"@com_google_protobuf//:struct_proto",
122+
"@com_google_protobuf//:timestamp_proto",
123+
"@com_google_protobuf//:type_proto",
124+
"@com_google_protobuf//:wrappers_proto",
125+
]
145126

146127
def lingvo_proto_cc(name, src, deps = []):
147128
# TODO(drpng): only works with proto with no deps within lingvo.
148129
_unused = [deps]
149130
basename = src.replace(".proto", "")
150-
_proto_gen_cc_src(name + "_gencc", basename)
151-
lingvo_cc_library(
131+
native.proto_library(
152132
name = name,
153-
srcs = [basename + ".pb.cc"],
154-
hdrs = [basename + ".pb.h"],
133+
srcs = [src],
134+
deps = [
135+
"//lingvo:tf_protos",
136+
] + WELL_KNOWN_PROTO_LIBS,
155137
)
156-
lingvo_cc_library(
157-
name = "%s_cc" % name,
158-
deps = [":%s" % name],
138+
native.cc_proto_library(
139+
name = name + "_cc",
140+
deps = [":" + name]
159141
)
160142

143+
161144
def lingvo_proto_py(name, src, deps = []):
162145
# TODO(drpng): only works with proto with no deps within lingvo.
163146
_unused = [deps]
164147
basename = src.replace(".proto", "")
165-
_proto_gen_py_src(name + "_genpy", basename)
166-
native.py_library(
148+
native.proto_library(
149+
name = name + "_pyproto",
150+
srcs = [src],
151+
deps = [
152+
"//lingvo:tf_protos",
153+
] + WELL_KNOWN_PROTO_LIBS,
154+
)
155+
py_proto_library(
167156
name = name,
168-
srcs = [basename + "_pb2.py"],
157+
deps = [name + "_pyproto"],
169158
)
170159

171160
# Placeholders to use until bazel supports pytype_{,strict_}{library,test,binary}.

‎lingvo/repo.bzl

-16
Original file line numberDiff line numberDiff line change
@@ -219,22 +219,6 @@ cc_library(
219219
strip_prefix = "googletest-release-1.10.0",
220220
)
221221

222-
def lingvo_protoc_deps():
223-
http_archive(
224-
name = "protobuf_protoc",
225-
build_file_content = """
226-
filegroup(
227-
name = "protoc_bin",
228-
srcs = ["bin/protoc"],
229-
visibility = ["//visibility:public"],
230-
)
231-
""",
232-
urls = [
233-
"https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip",
234-
],
235-
sha256 = "3cd951aff8ce713b94cde55e12378f505f2b89d47bf080508cf77e3934f680b6",
236-
)
237-
238222
def icu():
239223
third_party_http_archive(
240224
name = "icu",

‎lingvo/tools/generate_tf_dot_protos.sh

-4
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,3 @@ mkdir -p ${dest}/tensorflow/core/framework
3131
mkdir -p ${dest}/tensorflow/core/protobuf
3232

3333
${binary} ${dest}
34-
35-
# genrule requires statically determined outputs, so we package all
36-
# into a single file.
37-
tar -C ${dest} -cf ${dest}/tf_protos.tar tensorflow/core/{framework,protobuf}

0 commit comments

Comments
 (0)