chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
License Check / license-check (push) Failing after 2s

This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
+121
View File
@@ -0,0 +1,121 @@
load("@rules_python//python:proto.bzl", "py_proto_library")
load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
load(
"//tensorflow/core/platform:build_config.bzl",
"tf_proto_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow/tools/proto_splitter:__subpackages__"],
licenses = ["notice"],
)
exports_files([
"df-split-tree.cpb",
"df-split-tree.pbtxt",
"bf-split-tree.cpb",
"bf-split-tree.pbtxt",
"split-tree.pbtxt",
"split-large-nodes.cpb",
"split-large-nodes.pb",
"split-large-nodes.pbtxt",
"split-standard.cpb",
"split-standard.pb",
"split-standard.pbtxt",
"split-large-constant.cpb",
"split-large-constant.pbtxt",
"split-lots-nodes.cpb",
"split-lots-nodes.pb",
"split-lots-nodes.pbtxt",
"split-large-constant.cpb",
"split-large-constant.pb",
"split-large-constant.pbtxt",
"function-large-nodes.pb",
"function-lots-of-nodes.pb",
"graph-def-and-function.pb",
"many-field.cpb",
"many-field.pbtxt",
])
tf_proto_library(
name = "test_message_proto",
srcs = ["test_message.proto"],
make_default_target_header_only = True,
visibility = [
"//tensorflow/cc/saved_model:__subpackages__",
"//tensorflow/tools/proto_splitter:__subpackages__",
],
)
# copybara:uncomment_begin(google-only)
#
# py_proto_library(
# name = "test_message_proto_py_pb2",
# deps = [
# ":test_message_proto",
# ],
# )
#
# py_binary(
# name = "split_gen",
# srcs = ["split_gen.py"],
# strict_deps = True,
# deps = [
# ":test_message_proto_py_pb2",
# "//tensorflow/python/lib/io:file_io",
# "//tensorflow/tools/proto_splitter:chunk_proto_py_pb2",
# "//tensorflow/tools/proto_splitter:split",
# "//tensorflow/tools/proto_splitter:util",
# "@absl_py//absl:app",
# "@absl_py//absl/flags",
# "@absl_py//absl/logging",
# ],
# )
#
# py_binary(
# name = "split_graph_def_gen",
# srcs = ["split_graph_def_gen.py"],
# strict_deps = True,
# deps = [
# "//tensorflow/core:protos_all_py",
# "//tensorflow/python/lib/io:file_io",
# "//tensorflow/tools/proto_splitter:constants",
# "//tensorflow/tools/proto_splitter:split_graph_def",
# "//tensorflow/tools/proto_splitter/python:test_util",
# "@absl_py//absl:app",
# "@absl_py//absl/flags",
# "@absl_py//absl/logging",
# ],
# )
#
# py_binary(
# name = "split_saved_model_gen",
# srcs = ["split_saved_model_gen.py"],
# strict_deps = True,
# deps = [
# "//tensorflow/core:protos_all_py",
# "//tensorflow/python/lib/io:file_io",
# "//tensorflow/tools/proto_splitter:constants",
# "//tensorflow/tools/proto_splitter/python:saved_model",
# "//tensorflow/tools/proto_splitter/python:test_util",
# "@absl_py//absl:app",
# "@absl_py//absl/flags",
# "@absl_py//absl/logging",
# ],
# )
#
# py_binary(
# name = "many_field_gen",
# srcs = ["many_field_gen.py"],
# strict_deps = True,
# deps = [
# ":test_message_proto_py_pb2",
# "//tensorflow/python/lib/io:file_io",
# "//tensorflow/tools/proto_splitter:split",
# "@absl_py//absl:app",
# "@absl_py//absl/flags",
# ],
# )
#
# copybara:uncomment_end
Binary file not shown.
@@ -0,0 +1,100 @@
chunk_index: 0
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 1
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 2
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 4
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 1
}
message {
chunk_index: 5
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 2
}
message {
chunk_index: 6
}
}
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 1
}
message {
chunk_index: 3
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 7
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 1
}
message {
chunk_index: 8
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 2
}
message {
chunk_index: 9
}
}
}
}
}
}
Binary file not shown.
@@ -0,0 +1,100 @@
chunk_index: 0
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 1
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 2
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 3
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 1
}
message {
chunk_index: 4
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 2
}
message {
chunk_index: 5
}
}
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 1
}
message {
chunk_index: 6
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 0
}
message {
chunk_index: 7
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 1
}
message {
chunk_index: 8
}
}
chunked_fields {
field_tag {
field: 2
}
field_tag {
index: 2
}
message {
chunk_index: 9
}
}
}
}
}
}
@@ -0,0 +1,17 @@
÷
ô

fn3
Const_0Const*!
valueB" C3D?7 >ÿt²>_
Const_1Const*M
valueDBB"8>h>ñÞ²>wd>|(˜<8Ú>¨éL?}\°=¾Ãu>»œà>Ûm€>D•”>]sE?1,?
ü>_
Const_2Const*M
valueDBB"8”–×>R\â>7Ö<OwE>cý ?KÐÖ=¥?f\?ˆð8?ˆ&°<õZ²>&èö>×õ¤>_
Const_3Const*M
valueDBB"8=‡?ÙÅÐ=_"?JlG?N4 ?oôJ?EF}?Fã=O v?N;m>Â\?!(>?O7 ?Kº–>3
Const_4Const*!
valueB" K÷¹>H?Û+f?_
Const_5Const*M
valueDBB"8=aý>øÈ¡> \G?b?IY"?ëuG>L±{<9\?V·î= ñ®=v.x?G“>òc?ì…z?
Binary file not shown.
@@ -0,0 +1,31 @@
field_one {
repeated_field {
}
repeated_field {
string_field: "inner_inner_string"
map_field_uint32 {
key: 324
value: "map_value_324"
}
map_field_uint32 {
key: 543
value: "map_value_543"
}
}
}
map_field_int64 {
key: -1345
value: "map_value_-1345"
}
nested_map_bool {
key: false
value {
string_field: "string_false"
}
}
nested_map_bool {
key: true
value {
string_field: "string_true"
}
}
@@ -0,0 +1,102 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
r"""Generates ManyField test data for Merger.
Constructs chunked proto test data containing various field types for
Merger::Read and Merger::Merge.
Usage: bazel run tensorflow/tools/proto_splitter/testdata:many_field_gen -- \
--path=/tmp/many_field
"""
from collections.abc import Sequence
import os
from absl import app
from absl import flags
from tensorflow.python.lib.io import file_io
from tensorflow.tools.proto_splitter import split
from tensorflow.tools.proto_splitter.testdata import test_message_pb2
# Example path: /tmp/many_field
SPLITTER_TESTDATA_PATH = flags.DEFINE_string(
"path", None, help="Path to testdata directory."
)
class ManyFieldSplitter(split.ComposableSplitter):
"""Splitter for ManyField proto."""
def build_chunks(self):
self.add_chunk(
self._proto.field_one,
[
test_message_pb2.ManyFields.DESCRIPTOR.fields_by_name[
"field_one"
].number
],
)
self._proto.ClearField("field_one")
for map_key, map_value in self._proto.nested_map_bool.items():
self.add_chunk(
map_value,
[
test_message_pb2.ManyFields.DESCRIPTOR.fields_by_name[
"nested_map_bool"
].number,
map_key,
],
)
self._proto.ClearField("nested_map_bool")
def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError("Too many command-line arguments.")
proto = test_message_pb2.ManyFields(
field_one=test_message_pb2.ManyFields(
repeated_field=[
test_message_pb2.ManyFields(),
test_message_pb2.ManyFields(
string_field="inner_inner_string",
map_field_uint32={
324: "map_value_324",
543: "map_value_543",
},
),
]
),
map_field_int64={
-1345: "map_value_-1345",
},
nested_map_bool={
True: test_message_pb2.ManyFields(string_field="string_true"),
False: test_message_pb2.ManyFields(string_field="string_false"),
},
)
file_io.write_string_to_file(
os.path.join(SPLITTER_TESTDATA_PATH.value, "many-field.pbtxt"), str(proto)
)
ManyFieldSplitter(proto).write(
os.path.join(SPLITTER_TESTDATA_PATH.value, "many-field")
)
if __name__ == "__main__":
app.run(main)
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,90 @@
node {
name: "Const_0"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 3
}
}
tensor_content: "/O5?\342\020\271>\215\326{?"
}
}
}
}
node {
name: "Const_1"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 3
}
}
tensor_content: "\327\2177>\024)G?\007\027\224>"
}
}
}
}
node {
name: "Const_2"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 241
}
}
tensor_content: ">!\343>R\362[?\025a\003>u\224\007>9\237\n>R\264b?\330\310k?c\350\315=^\335Q?\332\254s?\"$a>\366\'M?4Yz>\363\036\017>\270\363U?\260\300\351>\350z+?\270\3270?\213\3651?>\024O?\247:K?W\"\211>\021\001P?v\346\262>=\244\307>*\t^?\336=O=\2265\206>\241\322\023?\'\3658?\256p\014?\344\253V?N\247o?j\013\311>\r\267??t\365*>\245\030\007=\334\300]?\005\221\'?\032W\266>Q\033\005?\375p\341>\324B=?\360 C?\375\317T?\016\001\265>\007\347\357>\325k\235=\004]}?\266\212\253>f\266C?\326\032\"?|\032\371>\311\016\252=\240\355d?\340D~?\310\343\'>\237\305\023>P\035;?\327\003\223>\246\202w?\320\026\'?y\351)>\335J\300>\221\362l?\r>\032?\3556\234<\231\017\253>\274\326~?\316{Y>N/\304>2\274l?\272\332\255>\254\373\032?)\334!?#jX?\360\336&?N\263\010?T\251\037?\240\014\344=\371\\\371>\261X\270>M\257\026?\0278\365>]\200\220>\377\200\200>[\267t?\220\353\013>8>~?\276\035\020?\033\217E?\004\200}?\223\373>?n\320\244>\274T\010?\347\363\260>\000|\010=u[\264>\355AS?\216gV?\231\341r?\002\360\033?g\204\027?E\354??\025\246t>`\025\000?\312\035\206>\335}x?\305\206\253>\275\356f?\306\251S?i&\206>:U\313>\313\354^>\231\353\t>\035S4?m\376\323>\277\337\372>EXx?1\216\265>\310\220\003?\272\021\037?\010g\305>\304\270U>\270._>F\333D?\343\357\005>\250\030\310>\2521\"?#6~?\305\002\210>)\3320:f\025\t?V\022v>\321~\214>.v\017?\204p1?\363\240\216>\357\225\221>\n\332\234=\025\0033:\272F\264>!Mn>\335f\350>\243\334/?\023\236y>Q\220\"?\316st?I\023\355>\0002\331=\377H]?\364\010d?}\326(?\301\251)?|\354\016?K\212B?\002\237t?\200\"M=\250\016&>|\243\360>\372p\177?\031xC>\323\177\310>\323\215\226>\230k|?~\227\n;\344E\004?\252\357)>\257\353*?\\\325\'>\255\365\214=\022Q(>G}[?f@\177?y\310\032?\010o\267=%\362\326>4\361\341>\026n\020>\375RV?^\203\023?\222\336x>I\276P?\020\212\304>\342;\350>b\037\330>s\236\027?\306\346\336>\322z\030?m\247d?\320zB>c\351N?\001\365\r?\336g\027?\253\031\037?\333\235\\?\000\314E?Q\302y?\206K\243=r^\'?\201\236\243>@Kj?:\001\326>\330L\266>\322Yj?9D\261=\357\216w?c\276C?\201\204\326=\235-E>\325Z\254>76_>S\304m?\346*\250>\241\017D?a\301\014>\332\316\372>\230\007?=\241\'b?.\357s?N\350\022<\032p+?\345\256d?\023\231\023>I\257\017?\\~V>\'\327]>\377A\'?GGL?\371\372f?\323\341H?pB\207>\344\322\214>r\271(?\363R\265>%Rb?G\1770?X\351C?\313\223\261>i\255\204>\255\n\013?"
}
}
}
}
node {
name: "Const_3"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 3
}
}
tensor_content: ":\267\364>0\310\222>\260\224`?"
}
}
}
}
node {
name: "Const_4"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 241
}
}
tensor_content: "-\020\353>\212!k?.\264\007=r\342\323>=\023P?#\356\025>o\310q?\352\237x>\003\264p>\201\212\004?\262#J>\222\005\"=\304\272%?\021\\\362>\205\256z?d\3042?\"~\263>\300\303\342=\332\003*?\356\301O?\303\316l?\0073\336>\334\033\213>\013\371\033?\275\033\311>+\204P?\221\000\022?5.t:id\005?B\265\245>k\'\202>\203h\304>9\236\234>.\202@?-\303U?\215\0326?\270\346\005=ro\205>\242zu?\253m\342>_\234_=A\237\303<\3610H?\211\272\010?\235\307\216>\"?g?*\005\377=F\231\330>\2325\001?\221\006_=\025\2762?\206wZ?\211\265\342>\242C\007?\237T^>\306\211.>\2010\265=\370yf?hw\347<\357-)?8+b?\253\246\325>2d\222>\242\246\004?\031\022\260:z\030\365=K.\266>\302\224G?\311\330*?\325\301\307>\257\201\271>\242\216\355>D7\024?L\275S?\254\224\250=\014i\250>p)=?\264i\346>dGS?@\\v?\214J\310>\256\2523?\226\245\005?\357)R?(:d?T\265\232>/ZV?\214a4?\327\242V= \376K?bp\376>v\236%?\242\206??\"\276\257>\271\014+?\0224\002>\365\010\313>\214k\000?\236\265\023?\037UG?\237\274B?B`\372>\371u\033?\247x\324>\266\346\321=N\024f?\254\205\310>\203n\030>:\312h?\315\355\212>\266\244\026?\253r\375>[.\242=B\225\277<\314\202b>wN\374<\032\020\014?\300i\351>\245\372\275>O\242O=5fE>\t\333\316>\021^\020?e\270\305<\255\277k?\204Z\227;$2\337>@V\004?\371\375\\?\007\323\004>@\206\177?\205\203\346>l\036\025?\332\264\207>4#\307>[\3119?\344\342\024?C\177(?\305Sk?\377wl?\347\351P?\004\t\354>I\214\264=`\227\366=H=\031?c\251P?Q\305\030<\336\036\006>\326\214\311>{k\024>\032\333\263>\023\305\350=~$\314>\237\225\005?+vi?\027\267\314>\014\223P?l\315\375>\265OZ?\257\036\310>\014P1?\250\001\037>;\366\030?\005~M?mH<?\325\257\327>\352\244k>\311\217(?\243\305\367>$(>?`\234\235>\322[\n?\227\237h?\350Vd?E?\222>\244\377\226=\362\205\235=-\342W?\211u\001?\276V1?_\370\365>\361M\250>\004\335M?2Y2?\202\273.?#X\256>\245\344!?H\2061?\027\340\372>\325\311|?x\n\353>\217\307\310=\242\325E?B(\216=x\223;?D/\241>b\2407?i\024y>\311{`=\2037\354=k\342\220>\214\340H?\001\233\263> \021-?\345x$?m\242\211>\345i+?\305\276)>\224\266`=\377.3>\213\251m?\330\031\337>JTv?\343\264\361>\340\332J?{\265\245>\357v\334>-\327u?\225\013h?\316\224Y=\274`l?\244>\034?\264A\022?\263\203\257>\016hV=\020]\022?\302P\363>\"*\324>M\343m?^\016x?\204\362\331>\273\354:?\271\320.?\231\2100>*\272\003>\024i[?\262\347\364>Hq2?\024\247\306=\234\027\372>\321\3358?"
}
}
}
}
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,108 @@
node {
name: "Const_0"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 3
}
}
tensor_content: "\355,W?y\232{?\234G\234>"
}
}
}
}
node {
name: "Const_1"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\222\241\363>c5r>>\244P?\345&\324>^i ?\205\'\242>2*\221>\014\305z>a\243@?\354Pj?ZH\210>\215\225\001>d\203/?\255\372k?"
}
}
}
}
node {
name: "Const_2"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "4\322\026?\2046K?\265\367]>\3672\305>\272\345w?\003\323\370=x\000\246>\301\255\027?\215)6?;\302W?\212#o?\0376\004?\335\356\016>\260Va?"
}
}
}
}
node {
name: "Const_3"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "v\334H?b\366M?\345\200\021?H\364\247>v\0231?\311\203\036?H\2777?K\227R?\351VI?\343\240\256>\341\333@?\364\251\244>lnY?\201\336d?"
}
}
}
}
node {
name: "Const_4"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 3
}
}
tensor_content: "n\344T?2\220m?v\001\252="
}
}
}
}
node {
name: "Const_5"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\200\216\007?\377\350o?\337\363\316>\275e\337=\313\263\t=\326Z\001?\263lI>\322\021_?\377\340\007?\354\321p>}\264\222>\036\177\037?\001\234#>\021\243<?"
}
}
}
}
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,270 @@
node {
name: "Const_0"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\016\300\252>\\\025$?\205M\365>~>\036?X\237\016=\232=]?Y\002*? \263\216>V\002x=\372C\355=\243\356\376>\213M\023;_fv?\'I~>"
}
}
}
}
node {
name: "Const_1"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\235\325\005?OO\374>m\206\004?0\244\320<\201\036\317>\304\264X?\320Yb?\350\313\013?\202\241\352=d\215*?\306\336^?\3438y?W\037~?K\214h?"
}
}
}
}
node {
name: "Const_2"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\342&\035?\246\343\250>\324k\236>E\243Q?\253/\026?+!e?Q\353\302>\203mc?^7\311> \213[?\222\004??H4\277=\3065t?\374\346N>"
}
}
}
}
node {
name: "Const_3"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\270\255M?\033\203\312>\225\263`?%\'\350>\177P+?\2633\030? \2630?\262\243(>>\221\373>\203\203|>\314\302\301=\026!\227=\2329 ?\t\255\326="
}
}
}
}
node {
name: "Const_4"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\320\333\265;\276r.?P\271\340=\302?\035?\314\2163?j\n{?#\216\255>\256\r\374>7\266\337>\024+4>yH\315=\244vW?\354\007\262<p%=?"
}
}
}
}
node {
name: "Const_5"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\375\233\202=\2257\254>8\337\004=\234\227h>I\327j?\177S$?L\375\005?G\306s?z\2479>\257\330,>~,\037?m\313R>}\210\r>\330\224w?"
}
}
}
}
node {
name: "Const_6"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\360\020\353>\351\206(?\360f\244>\033\257\034?\377\321>?J[\335>\363\372\036?1\336e>\263\207\232> [\010?\312d\274>\351\317v>\031\320\320>\377}\017?"
}
}
}
}
node {
name: "Const_7"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: ">\242\232=4/\036?\250\203\344>\010-0?\346\022\212>\211\022(?w{\025>:\025c?\035\374\246>\374\255[?\035_\246>fj^>\346\010\225>\230\3539="
}
}
}
}
node {
name: "Const_8"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: ">\211n?pZ\001?*\3054>\276\347W?\346\374\251=\251\257\350=\003\245\362>\nvs>\360\251g?\2039(?\235\202\370>L\301>?\373J\034?\026\225\225>"
}
}
}
}
node {
name: "Const_9"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\222\342\226>pN\212>\256c7?NS\263>\001\312\225>y0`?w/\301>3\3369>\312,\243=\353\036\030?\267\022\351>\314\246l?\004O\245>\303\035\322>"
}
}
}
}
node {
name: "Const_10"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\363\030R?\215\367j>I\364\301>8\331\022?0e\321>\007\2240?z\275|?\327u\250>z\225U?t\301{?\223$\010>0\006\025?\267\033\n>\336e\351="
}
}
}
}
node {
name: "Const_11"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\213\336\315>d\t5?\364\331\010?\267G\205<\277\002\204=\036\343j>\017\267\274>I\275F>\005\027\222>\\\226 =\254\267m>\245\373!>=\\\014?\307\314\240<"
}
}
}
}
node {
name: "Const_12"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "\226rG?\232\247Y>A\027n?.\321H?B\3638?B\244*?\340\\\030?\227[7?\036\375\032>\nv$?)\313\227>\\\255l?\033G:?,\r<>"
}
}
}
}
node {
name: "Const_13"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "a\332\223=\006z\'?\n\225\021=\263&\013?\355\222\267>\0307\026?\356\200\270=\227\361e?1(g?\237\032\272;\2447\252>\213a8?)\276\"?]!\021?"
}
}
}
}
node {
name: "Const_14"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 14
}
}
tensor_content: "v\261k=\213S\216>\346\257f?!\247r?X\330\314>\356\001~>\005\177`<\316C\">\357\250\032?\020h\027=\260\002\007?\261\2609?Q+\216<\323+\t>"
}
}
}
}
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,332 @@
meta_graphs {
graph_def {
node {
name: "Const_0"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 16
}
}
tensor_content: "\235d\331>N\217`?M!\241>\023\'|?P\006$?\017\225*?\332\016\265>\311\221\233=2\205\370>\303\216i?Y\321\271>\221\032\233>\343\335&?;\031\305>&=\317>\360\220T?"
}
}
}
}
node {
name: "Const_1"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 16
}
}
tensor_content: "{\260\007>%\n1?\200\205\273> \306N?\337\201\363>B\2348?C\366\306>A{e?g2}?\225_I?\356@!?^G\265<&\013\260>Ca\021?\300\355k>\363y\361>"
}
}
}
}
node {
name: "Const_2"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 241
}
}
tensor_content: "*\272+>\203#i>6r[?\254\206/>l\002S?\242\213K?K\214\205>\023\275\312>\256EM=y\321\n?\014\274^>\370\037{?\367\016t?\274\327\241>[\376\277>\225\036}?x:]?\272tO?\272\364\032?9,;?`I(?\377t\222=~;\355>o:\356>\225<\022?t\350\266>\000:O?&>\201>n\230\273>n\266\307>\370\034\260>\301\375n?\230\357\013?\346\312\021?\274D\264>\357\016\237>m\337\013>\373\3025=\246\262\317>\315\351z?\001\376#?\001\370\374>\305\277^?\031\231\342>\307\266\r?\364\236\032?\000\r\211>\223\210v?bq\307>\334\341\264=\006\274\265>\310\232\017?\013D\372>[\004\014>\032!^?R\212\301>\227\"\324>\024\311.?(\371\375>\207\007#?\031#k=BR=?\030Bj?\001\215)=e\351K?s\376\350>\330)\247;\337\030]?|d\275>\276\307\273=?\372r?u\262\335>\353\205z?\322\362\305>\313y*?r\316\270>Zf\325>\377\341\023?}+\234>\023\265\000?8>A=-\302\n?\323s\352=\331\371\215>G\000\247>`\001L>_\004F?\322\317-?\003dQ?h\007\005?o@S=\025\221c?U\203\245;\270\223d?\331\364\001?\3248V?\207\334H>\r\245\342>\346\331\311>\244\024\274=\300\023m>D++?\346\237M?w\020\304>\355$}?<A\032>\371h\333>\344\274|?\233\247M>\277\377\217=\221\031\275>V\214\234>\222\347:?\203\001\007?\312\241\025>\274\273(?\342\344\035?\311.\177?\346\246\312>\267\205\304>\370/H>k\213`?\0041\313>\234\333\264>^\257\032>\023<\022>[\323\026?\230\254S>\233\360B<p\321\002?b\346\341>9\244P>\313\351\226>\217\365\020?E(`>\254%A?\320q@?M_<?\363\037\010<\200\203d?\036h\212>\306\264\036?g\206\241=\327\341\036=P\033\366>\266\032\266>\254\224X?\361:a?#\017T?\017\220\016?\r\252\261=.\366\245>*r\303>\031\370\222>@T<?\027\235\021?\0214T?8-.?`~4?\375;j?u?\266>\267\215\207=\031\323{?\377y\'>\001\240f?Iv\276>s\214u>\356\354K>\315\256z?\210+\361=\250\372\254>\346\246\253>\223R\016?\201bR?t\227\215=\335j}?V\365\211>I\"]?~\325\354>\213\275*?:\3371?\263\267:?8\033\321>\352C\013> 7\023?\255\214\336>\362\244\243=\360.\355=1Z6?\201\177L?\332\277<>\273\264\010=\036A\226>\277O\254>\307\024G?\347\030,?\355b\327=\257SV?\037\336\017?\262\232\001?\034<k?\'\266\006?\252Wv?\013`\027?B\n/>\210\237\";|(\021>\022\023\336>\362\205?>M\220m>m\304\364=z\030\220>|\375g?\335\013h?\"\321\215>s-x?\342\334\036?\244\037\367>\334\245L?,\317d?\301\317r?\335\371\265=v\331\025?\035qT?a\253\004>6&w>\3238q?J\252\321>h2O?\316ra?\"G\206>\013\275\003?u\364\n?9:J?\t\305K?3\023\365>\241-\037?\225\354~>>\264\020?C\2066?\333\3446?"
}
}
}
}
node {
name: "Const_3"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 16
}
}
tensor_content: "#\375.>\021\223\021?\313\367K?\327.\033?w\206m?\273q\027?\250\213I?\201mV>\266\377b>\246\212\003?h\004]?\214H\010>\020\274\001>\033\035\010>\234\373x>r\312i?"
}
}
}
}
node {
name: "Const_4"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 241
}
}
tensor_content: "c\203;?\335\037C>\007\316E>\245\010\332=k\344\306>\201r\025>\330\274\020?h\272t?n\001\\?\264\226\314>\014z\014?\233\014J?KAU?\327\225V?[\007\240>\005\277\253>\341\250 ?5\362^?\355\366\246>\324&h?)\211\005?\343\371\347>\347sb?\234\236\325>E\004\037?\rkz=S<_>+\350\277=$\212M?\211S{?\222q1?q\352\233=By\245>\"r\354=\214\345\236>\177}\037?y\024\306>Ki\037>\021\216\007>\227\204\026?\326\371\022?\352\364~>\221\215\272=\026\372x?\031%c?/\336D>\277\363*>mH\r=G\222\014?4P&?\250AR<\"\307g?^\303{?\312\350\340>\270\234\264=\323\000\354>\0050+>\244\206(?\341\000^=\270\320\226>z\365\221>\014\007\347>\277^\030>w\035\340>G=*?\310c\236>2\037%=\361Qq?\215\261j?\033\204\345={T\217>\253N\333=\211]\004:\t\301\221>\357\031\016>eG\272>\"\370L?&\004j?\230\275\275<\300=5?\366\036a>\'\254X>\272zV?fY\220=\231\025w?\322\037&>m\220\335>\177\300\372=\210B\010>\326\313p?\305\340\325=\362\343k?\227\252`?\372Z\326>P9Q?\004\317X?\363l\013=\233p\177?\342T\021?f\224\365>\370\270A?\366\327\220=\343[\016?!\2033?\205\344\320=\336\0007?\360\n\t?\2012\000?\327(B>\007\252Z?\253\202\367>\225\356`>Z\2026?|J/?\311>:>r\023\006>\350qf?\265\317@?\243\017p?\016)T=\324\374\010=g\341J>)\243y>\257\004\252=\223\277|?1\237u? <%?\313|\\>\2164\216=\206\0222?XY,?\336b$?@\033G?\313\206\255>\315\372(>B\345e?\227\3123?\310\231\251=\1772\036?l%|?\312\032\334<\253Z\221>\357\020<?\256\201-?\220\342(?\035\r\324=\025a\326>\270\2779?C1S?\\\007\003>\374E\363>\222\223<?P\334\"?\204\177%?\350\207\371>)\356E=\37321>\240\220}>\356\000A?}\366*>\264Of?\030lm?\372\224\r?\351\211\252>L\031|?\215\237d?\337yb>\25277?mn??S\267\231>:\317]?\364\350\334>\324\247\007?\232] ?\n\026\311>\323\024E?r\372%?\017\362E>\250\214\317>\353l->\337IT?_\210o?o\037\361>$ \032?zxV>\212\327\037?W\317\177?o\031\006?\331\261\036?V\254y>\333\300P?\315\302\000?6\312\235>\034\373\304>\2003 =\344Oo?\210\030 >\265\\\353<P\350r?g\336\336<\342f\325>\253!\207>py\023?u\014\036?o\017<?\271\267T?\217\234\314>\243@\323>\241=\373>\212)\225=\255\234,?o\354\213=\204\235S?Y5c?>\310\005?\014*\024?\304\331;?\002|\226>\231\214\376>\341S\214=g4N?)\272\303>\225\036P>\213\324\\?\353i\341>\241\230j?\302\367m=\256\215R?\360\231\226>\2042\321>\025{\311>\332@\265>$E\017?\216\323j?\260\365\263=\276\353\276>\207\272r?g\315\212=\303\302\335>\347\026\376=\254@;?"
}
}
}
}
node {
name: "Const_5"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 116
}
}
tensor_content: "\036\014n?H/\027>\375be?\247\2065?v\303\366>\266\357\025?\266G#?\364c\304>\245j\035>{\273 ?\312\030~?\017o\031?/\345L>\272\254^>QQ\005??\366\201>\217N.?\274\013j?\0300{=\243\'0?\343=\325>\237v\010>;UQ?\322E\002?G^<?\tg4=\271\345^?y,\363>\204\206\251>\036+!=\275/S?\005\002\260=\203\3374>\\\300G?y\204\177?\257\254\007?\235\230+>\264\365\003?\014\261\261>\025\322\026?\2375\225>\031\263T?\313N\033>\237\325\016?ZtH=\204\331>?\215\343\010?\253\354L>\360\325>?5\224;?\232D\256;?\031B?\337kR?\3677s?`0\367>\332@\n?\024\245a?\352g\373=\303l7?\224\001\313>n\364\025?\006c^?#N\022<zs7?fD\007?$\021W?\"\027!>\233]\207>\315\356v?)\2075>\217}L?\345H\364>\333\033\374;p\271\034?\027Ke>@l\n?=\235\225>\t\264L?\256\343b>u\203\325>\234}\273=\340\\\333>\370k\245>\0021\n?\332\027\330>X\336S=7N\"<\227\335O>\375\025\037?\213Ee?\244\262\036?\210r*?\343zU?\022\347\025?\021rf?f\037a?\354\240\353=\216z#?Vr\256>\016\033&>\301\252\307>C\271\311>`\320\253>\307p\235>K\302f>\362s\272>\376~\377>\204Ct>\203\t\036?VjO>\023{&?Gf\036?j\017*?6pw>\244\272r?\032\"\021?"
}
}
}
}
node {
name: "Const_6"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 16
}
}
tensor_content: "\212\220q>\213\277y>4?\006<1\226\177>o}\271>b\374\016?\237\366,?\357h->P\312\230>\010\033_?\232\240\013?\004j\206>\023\021\007?%\024Q?\277\3634?\004\317*?"
}
}
}
}
node {
name: "Const_7"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 16
}
}
tensor_content: "\206\226v?4\211F?c\250\353>\'G\t?:\241%?\272y|=\020o\n>NN\265>9\3301>\313\377\325>&\302\013>3\035%?x\217\017?\037\255\345>P(\334>g\263<?"
}
}
}
}
node {
name: "Const_8"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 16
}
}
tensor_content: ".\356\201>\t\246\371>\234\014p?D\3626?j;\013=V\232h>\032\327w>I\000B?\320\336Z?(\364\234>\020$\223=~q\211>v\345&?a\257\000?\030\221\360>\350pX?"
}
}
}
}
library {
function {
signature {
name: "fn1"
}
node_def {
name: "Const_9"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 16
}
}
tensor_content: "\021\"/?\032\335\354;\210\352Q>!\344\341>\243[\204>\245\227h>\020\177\006?^\357\203;\273\305T?\257\213\370=p\352\230>\261\n\220>\256\375m=KO\267>\\|w>\241\324q?"
}
}
}
}
node_def {
name: "Const_10"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 15
}
}
tensor_content: "CT8?\254\211Q?HHd?\240\210\351>\352BJ?\013\253\265=\0305\357=&\225\016>\362\267\211>?G\332>*\3031?\356\315\255;Z\344$?\023$X=\344\272\271>"
}
}
}
}
node_def {
name: "Const_11"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 15
}
}
tensor_content: "\317\350l?A\030\357=b\313\246>\304@*>]~\205=t6\010>\251]\000>\205ms=\236\252\006>/\343\236;T\226\364=/b\273=v|E?\374\375\014?\303\317\027?"
}
}
}
}
}
function {
signature {
name: "fn2"
}
node_def {
name: "Const_12"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 15
}
}
tensor_content: "\205\324l?\274|H?)\235!?\242[??\3551\307>\355\2040?%\310\342>\034\025E?\302j\'?\237Q\350>*\r^=a\317;?p\271=?\272MA?\267&\346>"
}
}
}
}
node_def {
name: "Const_13"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 115
}
}
tensor_content: "N\265\276>.X\304>\352%k?\225!\245>\305\304\235>\346\211{?\3272*?R\321f?\320\300i?Iy@>\271x\203>\002\322\274=ioI?\t\225\372>\362Bp?\2031W<\022\337\002>E3\361>\366\214\274<\2274\036?\337\220\371>,B\336>*\361C?\250[\263>\236\t\020?\346Y\261>\336\232U>v=\325>f\277f<\207C2?h#=?_\215h?\314\211\215>\032rn?\214CH>\'\232\342=\204\376r=\374\237\'?x\310\265>\005\223\315>\260\270\016?\2179\255>\370\t\025>\224RE?ay9?\004\372=?\237\024G?!\034\351=\0231e?M\327\234>`\357f?\375\355t?\365\274\177?n\220E?sN\267>\253tQ?\303\277K?h\351\272>\031\277w?\007\341.>\240\024S=\217\177y?\271z\010?\323\231\356>\306k\034?\031@\264>\024\371\031>\036\211\035>W\276K?\001\272K>\223\'v?\213\212\376>\372F$?6\276R?\257vb?V\341\202>\004\023\032?=P\371>W\010F?\007\222\376>]\232\321>\231\252\370>\'7.>,\233k?`_\376>I\337d?\363\220b?hS]?\203\377u?\177Y\007?\023\177\306=\234\2534?\322\257\262>\225R\010<\371 m>4\334P?\337\241\345>E\001\271>\352\226O?*|@?\203\352a??#\010?\277\243\005?]\324\361>3\007;?\"\3753>\3427\000?ZW\022>\0037G?\350\220\273>XZ\277<q\252m?\342\315\311>xJd?\n\343\035?"
}
}
}
}
}
function {
signature {
name: "fn3"
}
node_def {
name: "Const_14"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 15
}
}
tensor_content: "\367\356)?\020y\310=o\2048?\243 >?\312\203T?a\273W>n\212m?\232C6?\370\331/?/\374\253>\372R ?\257|\017?\230\256X=\204\330\"?\0007\366>"
}
}
}
}
}
function {
signature {
name: "fn4"
}
node_def {
name: "Const_15"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 15
}
}
tensor_content: "&\036\000?\271\260b?\375\307h?\277u\371=Vl[?rP\321>:\306\010?\274\001\002?\267\\k?\374e\006?\347\355;>RlC?\017\370\330>\362\002\276=\336p\357>"
}
}
}
}
node_def {
name: "Const_16"
op: "Const"
attr {
key: "value"
value {
tensor {
dtype: DT_FLOAT
tensor_shape {
dim {
size: 15
}
}
tensor_content: "J\241F>\232\"i>H(z?b\227\274>t\331\233>\224\361O>&\210\210>au\316>\231\210V?\224\241j>\377P&>T>\303=C\371 ?O\232;>\216\016|>"
}
}
}
}
}
}
}
}
@@ -0,0 +1,28 @@
val: "0"
child_nodes {
val: "010"
child_nodes {
val: "01020"
child_nodes {
val: "0102030"
}
child_nodes {
val: "0102031"
}
child_nodes {
val: "0102032"
}
}
child_nodes {
val: "01021"
child_nodes {
val: "0102130"
}
child_nodes {
val: "0102131"
}
child_nodes {
val: "0102132"
}
}
}
+179
View File
@@ -0,0 +1,179 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generates test data for Merger.
Constructs depth- and breadth-first tree-like chunked protos test data for
Merger::Read and Merger::Merge.
"""
from collections.abc import Sequence
import os
from typing import Optional, Union
from absl import app
from absl import flags
from absl import logging
from google.protobuf import message
from tensorflow.python.lib.io import file_io
from tensorflow.tools.proto_splitter import chunk_pb2
from tensorflow.tools.proto_splitter import split
from tensorflow.tools.proto_splitter import util
from tensorflow.tools.proto_splitter.testdata import test_message_pb2
SPLITTER_TESTDATA_PATH = flags.DEFINE_string(
"path", None, help="Path to testdata directory.")
_CHILD_NODES_FIELD_TAG = (
test_message_pb2.StringNode.DESCRIPTOR.fields_by_name[
"child_nodes"
].number
)
class StringNodeSplitter(split.ComposableSplitter):
"""Splits a StringNode proto with N strings into a tree with depth N."""
def __init__(self, proto: test_message_pb2.StringNode,
chunked_message: Optional[chunk_pb2.ChunkedMessage] = None,
**kwargs):
super().__init__(proto, **kwargs)
self._chunked_message = self._chunked_message or chunked_message
def add_chunk(
self, chunk: Union[message.Message, bytes], field_tags: util.FieldTypes
) -> None:
"""Adds a new chunk and updates the ChunkedMessage proto."""
assert self._chunked_message is not None
field = self._chunked_message.chunked_fields.add(
field_tag=util.get_field_tag(self._proto, field_tags)
)
field.message.chunk_index = self.total_chunks_len()
self.add_root_chunk(chunk)
def total_chunks_len(self) -> int:
"""Returns length of chunks stored in root splitter."""
if self._parent_splitter is not None:
return self._parent_splitter.total_chunks_len()
return len(self._chunks)
def add_root_chunk(self, chunk: Union[message.Message, bytes]) -> None:
"""Adds chunk to root splitter chunks."""
if self._parent_splitter is None:
assert self._chunks is not None
self._chunks.append(chunk)
else:
self._parent_splitter.add_root_chunk(chunk)
class DFStringNodeSplitter(StringNodeSplitter):
"""Depth-first string node splitter."""
def build_chunks(self) -> Sequence[Union[message.Message, bytes]]:
if not isinstance(self._proto, test_message_pb2.StringNode):
raise TypeError("Can only split TreeString type protos")
if not self._proto.child_nodes:
return
for i, node in enumerate(self._proto.child_nodes):
self.add_chunk(node, [_CHILD_NODES_FIELD_TAG, i])
DFStringNodeSplitter(
proto=node,
parent_splitter=self,
fields_in_parent=[_CHILD_NODES_FIELD_TAG],
chunked_message=self._chunked_message.chunked_fields[i].message
).build_chunks()
self._proto.ClearField("child_nodes")
if self._parent_splitter is None:
self._chunks.append(self._chunked_message)
file_io.write_string_to_file(
os.path.join(SPLITTER_TESTDATA_PATH.value, "df-split-tree.pbtxt"),
str(self._chunked_message))
return self._chunks
class BFStringNodeSplitter(StringNodeSplitter):
"""Breadth-first string node splitter."""
def build_chunks(self) -> Sequence[Union[message.Message, bytes]]:
if not isinstance(self._proto, test_message_pb2.StringNode):
raise TypeError("Can only split TreeString type protos")
if not self._proto.child_nodes:
return
for i, node in enumerate(self._proto.child_nodes):
self.add_chunk(node, [_CHILD_NODES_FIELD_TAG, i])
for i, node in enumerate(self._proto.child_nodes):
BFStringNodeSplitter(
proto=node,
parent_splitter=self,
fields_in_parent=[_CHILD_NODES_FIELD_TAG],
chunked_message=self._chunked_message.chunked_fields[i].message
).build_chunks()
self._proto.ClearField("child_nodes")
if self._parent_splitter is None:
self._chunks.append(self._chunked_message)
file_io.write_string_to_file(
os.path.join(SPLITTER_TESTDATA_PATH.value, "bf-split-tree.pbtxt"),
str(self._chunked_message))
return self._chunks
def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError("Too many command-line arguments.")
if SPLITTER_TESTDATA_PATH.value is None:
raise app.UsageError("'path' flag not specified.")
levels = 4
def make_string_tree(
string_tree: test_message_pb2.StringNode, level: int = 0, label: str = "0"
) -> test_message_pb2.StringNode:
string_tree.val = label
if level >= levels-1:
return string_tree
for i in range(level+1):
make_string_tree(string_tree.child_nodes.add(),
level+1, label+str(level+1)+str(i))
return string_tree
def copy_string_tree(string_tree: test_message_pb2.StringNode):
new_tree = test_message_pb2.StringNode()
new_tree.CopyFrom(string_tree)
return new_tree
string_tree = make_string_tree(test_message_pb2.StringNode())
logging.info("StringNode tree generated:\n%s", string_tree)
file_io.write_string_to_file(
os.path.join(SPLITTER_TESTDATA_PATH.value, "split-tree.pbtxt"),
str(string_tree))
# depth-first chunk ordering
DFStringNodeSplitter(copy_string_tree(string_tree)).write(
os.path.join(SPLITTER_TESTDATA_PATH.value, "df-split-tree"))
# breadth-first
BFStringNodeSplitter(copy_string_tree(string_tree)).write(
os.path.join(SPLITTER_TESTDATA_PATH.value, "bf-split-tree"))
if __name__ == "__main__":
app.run(main)
@@ -0,0 +1,198 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
r"""Generates GraphDef test data for Merger.
Constructs chunked protos test data containing GraphDefs with lots of nodes and
large nodes for Merger::Read and Merger::Merge.
Example command:
bazel run tensorflow/tools/proto_splitter/testdata:split_graph_def_gen -- \
--path /tmp \
--graph_type=split-lots-nodes,split-large-nodes,split-large-constant \
--export=pb,cpb
"""
from collections.abc import Sequence
import os
from absl import app
from absl import flags
from absl import logging
from tensorflow.core.framework import graph_pb2
from tensorflow.python.lib.io import file_io
from tensorflow.tools.proto_splitter import constants
from tensorflow.tools.proto_splitter import split_graph_def
from tensorflow.tools.proto_splitter.python import test_util
LOTS_NODES_SIZES = [95] * 15
LARGE_NODES_SIZES = [50, 95, 95, 95, 50, 95]
LARGE_CONSTANT_SIZES = [50, 50, 1000, 50, 1000]
def _split_and_write(
path: str,
graph_def: graph_pb2.GraphDef,
max_size: int,
export_files: Sequence[str],
):
"""Writes the .pb, .pbtxt and .cpb files for a GraphDef."""
constants.debug_set_max_size(max_size)
if "pbtxt" in export_files:
output_path = f"{path}.pbtxt"
file_io.write_string_to_file(output_path, str(graph_def))
logging.info(" %s written", output_path)
if "pb" in export_files:
output_path = f"{path}.pb"
file_io.write_string_to_file(output_path, graph_def.SerializeToString())
logging.info(" %s written", output_path)
if "cpb" in export_files:
splitter = split_graph_def.GraphDefSplitter(graph_def)
splitter.write(path)
chunks, _ = splitter.split()
if len(chunks) > 1:
logging.info(" %s.cpb written", path)
else:
raise RuntimeError(
"For some reason this graph was not chunked, so a .cpb file was not"
" produced. Raising an error since this should not be the case."
)
def split_lots_nodes(path: str, export_files: Sequence[str]):
"""GraphDef with lots of nodes."""
# The actual sizes in the generated graph has a slight deviation, but are
# between [90, 100] (tested in testMakeGraphDef with atol=5).
# Expected Chunks (Max Size = 500)
# -----------------------------
# Chunk #: Contents
# -----------------------------
# 0: GraphDef # (nodes [0:5])
# -----------------------------
# 1: GraphDef # (nodes [5:10])
# -----------------------------
# 2: GraphDef # (nodes [10:15])
# -----------------------------
# 3: ChunkedMessage
# -----------------------------
graph_def = test_util.make_graph_def_with_constant_nodes(LOTS_NODES_SIZES)
_split_and_write(path, graph_def, 500, export_files)
def split_large_nodes(path: str, export_files: Sequence[str]):
"""GraphDef with large nodes."""
# Large nodes are greedily split from the original proto if they are
# larger than max_size / 3.
# This should create 6 chunks:
# [parent GraphDef, node[1], node[2], node[3], node[5], ChunkedMessage]
graph_def = test_util.make_graph_def_with_constant_nodes(LARGE_NODES_SIZES)
_split_and_write(path, graph_def, 200, export_files)
def split_large_constant(path: str, export_files: Sequence[str]):
"""GraphDef with large constant nodes."""
# Expected Chunks (Max Size = 500)
# -----------------------------
# Chunk #: Contents
# -----------------------------
# 0: GraphDef
# -----------------------------
# 1: GraphDef.nodes[2].attr["value"].tensor.tensor_content
# -----------------------------
# 2: GraphDef.nodes[4].attr["value"].tensor.tensor_content
# -----------------------------
graph_def = test_util.make_graph_def_with_constant_nodes(LARGE_CONSTANT_SIZES)
_split_and_write(path, graph_def, 500, export_files)
def function_lots_of_nodes(path: str, export_files: Sequence[str]):
"""Generates a proto of GraphDef with a FunctionDef that have many nodes."""
graph_def = test_util.make_graph_def_with_constant_nodes(
[], fn=LOTS_NODES_SIZES
)
_split_and_write(path, graph_def, 500, export_files)
def function_large_nodes(path: str, export_files: Sequence[str]):
graph_def = test_util.make_graph_def_with_constant_nodes(
[], fn=LARGE_NODES_SIZES
)
_split_and_write(path, graph_def, 200, export_files)
def graph_def_and_function(path: str, export_files: Sequence[str]):
graph_def = test_util.make_graph_def_with_constant_nodes(
[50, 50, 50, 50, 50, 50], fn1=[50, 50, 50], fn2=[50], fn3=[50], fn4=[50]
)
_split_and_write(path, graph_def, 200, export_files)
VALID_GRAPH_TYPES = {
"split-lots-nodes": split_lots_nodes,
"split-large-nodes": split_large_nodes,
"split-large-constant": split_large_constant,
"function-lots-of-nodes": function_lots_of_nodes,
"function-large-nodes": function_large_nodes,
"graph-def-and-function": graph_def_and_function,
}
ALL_GRAPH_TYPES = ", ".join(VALID_GRAPH_TYPES.keys())
SPLITTER_TESTDATA_PATH = flags.DEFINE_string(
"path", None, help="Path to testdata directory."
)
GRAPH_TYPES = flags.DEFINE_multi_string(
"graph_type",
"all",
help=f"Type(s) of graph to export. Valid types: all, {ALL_GRAPH_TYPES}",
)
EXPORT_FILES = flags.DEFINE_multi_string(
"export",
"all",
help="List of files to export. Valid options: all, pb, pbtxt, cpb",
)
def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError("Too many command-line arguments.")
if "all" in EXPORT_FILES.value:
export_files = ["pb", "pbtxt", "cpb"]
else:
export_files = EXPORT_FILES.value
if "all" in GRAPH_TYPES.value:
graph_types = VALID_GRAPH_TYPES.keys()
else:
graph_types = GRAPH_TYPES.value
for v in graph_types:
if v not in VALID_GRAPH_TYPES:
raise ValueError(
f"Invalid flag passed to `graph_type`: {v}\nValid graph types:"
f" {ALL_GRAPH_TYPES}"
)
logging.info("Generating graph %s", v)
f = VALID_GRAPH_TYPES[v]
f(os.path.join(SPLITTER_TESTDATA_PATH.value, v), export_files)
if __name__ == "__main__":
app.run(main)
@@ -0,0 +1,145 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
r"""Generates SavedModel test data for Merger.
Constructs chunked proto test data containing a SavedModel.
Example command:
bazel run tensorflow/tools/proto_splitter/testdata:split_saved_model_gen -- \
--path /tmp \
--saved_model_type=split-standard \
--export=pb,cpb
"""
from collections.abc import Sequence
import os
from absl import app
from absl import flags
from absl import logging
from tensorflow.core.protobuf import saved_model_pb2
from tensorflow.python.lib.io import file_io
from tensorflow.tools.proto_splitter import constants
from tensorflow.tools.proto_splitter.python import saved_model as split_saved_model
from tensorflow.tools.proto_splitter.python import test_util
STANDARD_SIZES = [100, 100, 1000, 100, 1000, 500, 100, 100, 100]
def _split_and_write(
path: str,
saved_model: saved_model_pb2.SavedModel,
max_size: int,
export_files: Sequence[str],
):
"""Writes the .pb, .pbtxt and .cpb files for a SavedModel."""
constants.debug_set_max_size(max_size)
if "pbtxt" in export_files:
output_path = f"{path}.pbtxt"
file_io.write_string_to_file(output_path, str(saved_model))
logging.info(" %s written", output_path)
if "pb" in export_files:
output_path = f"{path}.pb"
file_io.write_string_to_file(output_path, saved_model.SerializeToString())
logging.info(" %s written", output_path)
if "cpb" in export_files:
splitter = split_saved_model.SavedModelSplitter(saved_model)
splitter.write(path)
chunks, _ = splitter.split()
if len(chunks) > 1:
logging.info(" %s.cpb written", path)
else:
raise RuntimeError(
"For some reason this graph was not chunked, so a .cpb file was not"
" produced. Raising an error since this should not be the case."
)
def split_standard(path: str, export_files: Sequence[str]):
"""Splits a standard SavedModel."""
fn1 = [100, 100, 100]
fn2 = [100, 500]
fn3 = [100]
fn4 = [100, 100]
max_size = 500
constants.debug_set_max_size(max_size)
graph_def = test_util.make_graph_def_with_constant_nodes(
STANDARD_SIZES, fn1=fn1, fn2=fn2, fn3=fn3, fn4=fn4
)
proto = saved_model_pb2.SavedModel()
proto.meta_graphs.add().graph_def.CopyFrom(graph_def)
_split_and_write(path, proto, max_size, export_files)
VALID_SAVED_MODEL_TYPES = {
"split-standard": split_standard,
}
ALL_SAVED_MODEL_TYPES = ", ".join(VALID_SAVED_MODEL_TYPES.keys())
SPLITTER_TESTDATA_PATH = flags.DEFINE_string(
"path", None, help="Path to testdata directory."
)
SAVED_MODEL_TYPES = flags.DEFINE_multi_string(
"saved_model_type",
"all",
help=(
"Type(s) of saved model to export. Valid types: all, "
f"{ALL_SAVED_MODEL_TYPES}"
),
)
EXPORT_FILES = flags.DEFINE_multi_string(
"export",
"all",
help="List of files to export. Valid options: all, pb, pbtxt, cpb",
)
def main(argv: Sequence[str]) -> None:
if len(argv) > 1:
raise app.UsageError("Too many command-line arguments.")
if "all" in EXPORT_FILES.value:
export_files = ["pb", "pbtxt", "cpb"]
else:
export_files = EXPORT_FILES.value
if "all" in SAVED_MODEL_TYPES.value:
saved_model_types = VALID_SAVED_MODEL_TYPES.keys()
else:
saved_model_types = SAVED_MODEL_TYPES.value
for v in saved_model_types:
if v not in VALID_SAVED_MODEL_TYPES:
raise ValueError(
"Invalid flag passed to `saved_model_type`: "
f"{v}\nValid saved model types:"
f" {ALL_SAVED_MODEL_TYPES}"
)
logging.info("Generating saved model %s", v)
f = VALID_SAVED_MODEL_TYPES[v]
f(os.path.join(SPLITTER_TESTDATA_PATH.value, v), export_files)
if __name__ == "__main__":
app.run(main)
@@ -0,0 +1,42 @@
// Copyright 2026 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================
syntax = "proto3";
package tensorflow.proto_splitter_testdata;
message RepeatedString {
repeated string strings = 1;
}
message RepeatedRepeatedString {
int32 filler_field = 1;
repeated RepeatedString rs = 2;
}
message ManyFields {
ManyFields field_one = 1;
repeated ManyFields repeated_field = 2;
string string_field = 3;
repeated string repeated_string_field = 4;
map<uint32, string> map_field_uint32 = 5;
map<int64, string> map_field_int64 = 6;
map<bool, ManyFields> nested_map_bool = 7;
}
message StringNode {
string val = 1;
repeated StringNode child_nodes = 2;
}