wehub-resource-sync
266 lines
9.9 KiB
Protocol Buffer
266 lines
9.9 KiB
Protocol Buffer
// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
syntax = "proto2";
|
|
package paddle;
|
|
option cc_generic_services = true;
|
|
option cc_enable_arenas=true;
|
|
|
|
message PSParameter {
|
|
optional string worker_class = 1;
|
|
optional string server_class = 2;
|
|
optional string instance_class = 3;
|
|
optional string init_gflags = 4 [ default = "" ];
|
|
optional WorkerParameter worker_param = 101;
|
|
optional ServerParameter server_param = 102;
|
|
repeated DownpourTrainerParameter trainer_param = 301;
|
|
optional FsClientParameter fs_client_param = 501;
|
|
}
|
|
|
|
message WorkerParameter {
|
|
optional DownpourWorkerParameter downpour_worker_param = 1;
|
|
}
|
|
|
|
message ServerParameter {
|
|
optional DownpourServerParameter downpour_server_param = 1;
|
|
}
|
|
|
|
message DownpourWorkerParameter {
|
|
repeated TableParameter downpour_table_param = 1;
|
|
}
|
|
|
|
message DownpourTrainerParameter {
|
|
repeated DenseTableParameter dense_table = 1;
|
|
repeated SparseTableParameter sparse_table = 2;
|
|
optional int32 push_sparse_per_batch = 3;
|
|
optional int32 push_dense_per_batch = 4;
|
|
repeated string skip_op = 5;
|
|
repeated ProgramConfig program_config = 6;
|
|
}
|
|
|
|
message ProgramConfig {
|
|
required string program_id = 1;
|
|
repeated int32 push_sparse_table_id = 2;
|
|
repeated int32 push_dense_table_id = 3;
|
|
repeated int32 pull_sparse_table_id = 4;
|
|
repeated int32 pull_dense_table_id = 5;
|
|
}
|
|
|
|
message DenseTableParameter {
|
|
optional int32 table_id = 1;
|
|
repeated string dense_variable_name = 2;
|
|
repeated string dense_gradient_variable_name = 3;
|
|
optional int32 fea_dim = 4;
|
|
}
|
|
|
|
message SparseTableParameter {
|
|
optional int32 table_id = 1;
|
|
optional int32 feature_dim = 2;
|
|
repeated string slot_key = 3;
|
|
repeated string slot_value = 4;
|
|
repeated string slot_gradient = 5;
|
|
}
|
|
|
|
message DownpourServerParameter {
|
|
repeated TableParameter downpour_table_param = 1;
|
|
optional ServerServiceParameter service_param = 2;
|
|
}
|
|
|
|
message ServerServiceParameter {
|
|
optional string server_class = 1 [ default = "DownpourBrpcPsServer" ];
|
|
optional string client_class = 2 [ default = "DownpourBrpcPsClient" ];
|
|
optional string service_class = 3 [ default = "DownpourPsService"];
|
|
optional uint32 start_server_port = 4 [ default = 0 ]; //will find a available port from it
|
|
optional uint32 server_thread_num = 5 [ default = 12 ];
|
|
}
|
|
|
|
enum TableType {
|
|
PS_SPARSE_TABLE = 0;
|
|
PS_DENSE_TABLE = 1;
|
|
}
|
|
|
|
message TableParameter {
|
|
optional uint64 table_id = 1;
|
|
optional string table_class = 2;
|
|
optional uint64 shard_num = 3 [ default = 1000 ];
|
|
optional TableAccessorParameter accessor = 4;
|
|
optional TableType type = 5;
|
|
optional bool compress_in_save = 6 [default = false];
|
|
//for cache model
|
|
optional bool enable_sparse_table_cache = 7 [default = true];
|
|
optional double sparse_table_cache_rate = 8 [default = 0.00055];
|
|
optional uint32 sparse_table_cache_file_num = 9 [default = 16];
|
|
optional double sparse_table_mem_cache_rate = 10 [default = 0.5];
|
|
|
|
}
|
|
|
|
message TableAccessorParameter {
|
|
optional string accessor_class = 1;
|
|
optional SparseSGDRuleParameter sparse_sgd_param = 2;
|
|
optional DenseSGDRuleParameter dense_sgd_param = 3;
|
|
optional uint32 fea_dim = 4 [default = 11];
|
|
optional uint32 embedx_dim = 5 [default = 8];
|
|
optional uint32 embedx_threshold = 6 [default = 10];
|
|
optional DownpourTableAccessorParameter downpour_accessor_param = 7;
|
|
repeated TableAccessorSaveParameter table_accessor_save_param = 8;
|
|
optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9;
|
|
optional SparseCommonSGDRuleParameter embed_sgd_param = 10;
|
|
optional SparseCommonSGDRuleParameter embedx_sgd_param = 11;
|
|
}
|
|
|
|
message DownpourTableAccessorParameter {
|
|
optional float nonclk_coeff = 1 [default = 0.1]; // to calculate show_click_score
|
|
optional float click_coeff = 2 [default = 1]; // to calculate show_click_score
|
|
optional float base_threshold = 3 [default = 1.5]; // show_click_score > base_threshold, this feature can be saved
|
|
optional float delta_threshold = 4 [default = 0.25]; // delta_score > delta_threshold, this feature can be saved
|
|
optional float delta_keep_days = 5 [default = 16]; // unseen_day < delta_keep_days, this feature can be saved
|
|
optional float show_click_decay_rate = 6 [default = 0.98]; // show/click will update to show/click * show_click_decay_rate after a day
|
|
optional float delete_threshold = 7 [default = 0.8]; // threshold to shrink a feasign
|
|
optional float delete_after_unseen_days = 8 [default = 30]; // unseen_day > delete_after_unseen_days, this feature will be delete in shrink_model
|
|
optional int32 ssd_unseenday_threshold = 9 [default = 1]; // threshold to save ssd
|
|
}
|
|
|
|
message TableAccessorSaveParameter {
|
|
optional uint32 param = 1;
|
|
optional string converter = 2;
|
|
optional string deconverter = 3;
|
|
}
|
|
|
|
enum PsCmdID {
|
|
PS_PULL_DENSE_TABLE = 0;
|
|
PS_PUSH_DENSE_TABLE = 1;
|
|
PS_PULL_SPARSE_TABLE = 2;
|
|
PS_PUSH_SPARSE_TABLE = 3;
|
|
PS_SHRINK_TABLE = 4;
|
|
PS_SAVE_ONE_TABLE = 5;
|
|
PS_SAVE_ALL_TABLE = 6;
|
|
PS_LOAD_ONE_TABLE = 7;
|
|
PS_LOAD_ALL_TABLE = 8;
|
|
PS_CLEAR_ONE_TABLE = 9;
|
|
PS_CLEAR_ALL_TABLE = 10;
|
|
PS_PUSH_DENSE_PARAM = 11;
|
|
PS_STOP_SERVER = 12;
|
|
PS_SAVE_ONE_CACHE_TABLE = 13;
|
|
PS_GET_CACHE_THRESHOLD = 14;
|
|
PS_CACHE_SHUFFLE = 15;
|
|
PS_COPY_TABLE = 16;
|
|
PS_COPY_TABLE_BY_FEASIGN = 17;
|
|
PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY = 18;
|
|
PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY = 19;
|
|
PS_PRINT_TABLE_STAT = 20;
|
|
PS_SAVE_ONE_TABLE_PREFIX = 21;
|
|
PS_SAVE_MEM_CACHE_TABLE = 22;
|
|
//pserver2pserver cmd start from 100
|
|
PS_S2S_MSG = 101;
|
|
//local_client2local_client cmd start from 200
|
|
PS_C2C_PULL_SPARSE_TABLE = 201;
|
|
}
|
|
|
|
message PsRequestMessage {
|
|
required uint32 cmd_id = 1;
|
|
optional uint32 table_id = 2;
|
|
repeated bytes params = 3;
|
|
optional int32 client_id = 4;
|
|
optional bytes data = 5;
|
|
};
|
|
|
|
message SparseSGDRuleParameter {
|
|
optional double learning_rate = 1 [default = 0.05];
|
|
optional double initial_g2sum = 2 [default = 3.0];
|
|
optional double initial_range = 3 [default = 0.0001];
|
|
repeated float weight_bounds = 4;
|
|
}
|
|
|
|
message SparseCommonSGDRuleParameter {
|
|
optional string name = 1;
|
|
optional SparseNaiveSGDRuleParameter naive = 2;
|
|
optional SparseAdagradSGDRuleParameter adagrad = 3;
|
|
optional SparseAdamSGDParameter adam = 4;
|
|
}
|
|
|
|
message SparseNaiveSGDRuleParameter {
|
|
optional double learning_rate = 1 [default = 0.05];
|
|
optional double initial_range = 2 [default = 0.0001];
|
|
repeated float weight_bounds = 3;
|
|
}
|
|
|
|
message SparseAdagradSGDRuleParameter {
|
|
optional double learning_rate = 1 [default = 0.05];
|
|
optional double initial_g2sum = 2 [default = 3.0];
|
|
optional double initial_range = 3 [default = 0.0001];
|
|
repeated float weight_bounds = 4;
|
|
}
|
|
|
|
message SparseAdamSGDParameter {
|
|
optional double learning_rate = 1 [default = 0.001];
|
|
optional double initial_range = 2 [default = 0.0001];
|
|
optional double beta1_decay_rate = 3 [default = 0.9];
|
|
optional double beta2_decay_rate = 4 [default = 0.999];
|
|
optional double ada_epsilon = 5 [default = 1e-08];
|
|
repeated float weight_bounds = 6;
|
|
}
|
|
|
|
message DenseSGDRuleParameter {
|
|
optional string name = 1;
|
|
optional AdamSGDParameter adam = 2;
|
|
optional NaiveSGDParameter naive = 3;
|
|
optional SummarySGDParameter summary = 4;
|
|
optional MovingAverageRuleParameter moving_average = 5;
|
|
}
|
|
|
|
message AdamSGDParameter {
|
|
optional double learning_rate = 1 [default = 5e-06]; // \u5B66\u4E60\u7387
|
|
optional double avg_decay_rate = 2 [default = 0.999993]; // avg_weight\u7684\u8870\u51CF\u7CFB\u6570
|
|
optional double ada_decay_rate = 3 [default = 0.9999];
|
|
optional double ada_epsilon = 4 [default = 1e-08];
|
|
optional double mom_decay_rate = 5 [default = 0.99];
|
|
}
|
|
|
|
message NaiveSGDParameter {
|
|
optional double learning_rate = 1 [default = 0.0002];
|
|
optional double avg_decay_rate = 2;
|
|
}
|
|
|
|
message SummarySGDParameter {
|
|
optional double summary_decay_rate = 1 [default = 0.999999]; // \u6743\u91CD\u7684\u8870\u51CF\u7CFB\u6570
|
|
}
|
|
|
|
message MovingAverageRuleParameter {
|
|
optional double momentum = 1;
|
|
}
|
|
|
|
message PsResponseMessage {
|
|
required int32 err_code = 1 [default = 0];
|
|
required string err_msg = 2 [default = ""];
|
|
optional bytes data = 3;
|
|
};
|
|
|
|
service PsService {
|
|
rpc service(PsRequestMessage) returns (PsResponseMessage);
|
|
};
|
|
|
|
message FsClientParameter {
|
|
enum FsApiType {
|
|
HDFS = 0;
|
|
AFS = 1;
|
|
}
|
|
optional FsApiType fs_type = 1 [default = HDFS];
|
|
optional string uri = 2; //such as afs://tianqi.afs.baidu.com:9902
|
|
optional string user = 3; //user_name to access fs
|
|
optional string passwd = 4; //password
|
|
optional int32 buffer_size = 5; //buffer for read/write
|
|
optional string hadoop_bin = 51;
|
|
optional string afs_conf = 101;
|
|
}
|