chore: import upstream snapshot with attribution

python/paddle/base/trainer_desc.py

@@ -0,0 +1,289 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Definition of trainers."""
+
+__all__ = []
+
+
+class TrainerDesc:
+    '''
+    Set proto from python to c++.
+    Can be initialized from train_desc.
+    '''
+
+    def __init__(self):
+        '''
+        self.proto_desc = data_feed_pb2.DataFeedDesc()
+        with open(proto_file, 'r') as f:
+            text_format.Parse(f.read(), self.proto_desc)
+        '''
+        from .proto import trainer_desc_pb2
+
+        self.proto_desc = trainer_desc_pb2.TrainerDesc()
+        import multiprocessing as mp
+
+        # set default thread num == cpu count
+        self.proto_desc.thread_num = mp.cpu_count()
+        self._fleet_desc = None
+        self._device_worker = None
+        self._program = None
+        self._infer = False
+
+    def _set_fetch_var_and_info(self, fetch_vars, fetch_info, print_period):
+        # convert fetch_info to list
+        fetch_info = list(fetch_info)
+        for i, v in enumerate(fetch_vars):
+            self.proto_desc.fetch_config.fetch_var_names.extend([v.name])
+            self.proto_desc.fetch_config.fetch_var_str_format.extend(
+                [fetch_info[i]]
+            )
+        self.proto_desc.fetch_config.print_period = print_period
+
+    def _set_debug(self, debug):
+        self.proto_desc.debug = debug
+
+    def _set_thread(self, thread_num):
+        self.proto_desc.thread_num = thread_num
+
+    def _set_device_worker(self, device_worker):
+        self._device_worker = device_worker
+
+    def _set_infer(self, infer):
+        self._infer = infer
+
+    def _set_fleet_desc(self, fleet_desc):
+        self._fleet_desc = fleet_desc
+        # serialize fleet_desc
+        from google.protobuf import text_format
+
+        fleet_desc_str = text_format.MessageToString(fleet_desc)
+        self.proto_desc.fleet_desc = fleet_desc_str
+
+    def _gen_trainer_desc(self):
+        pass
+
+    def _set_program(self, program):
+        self._program = program
+
+    def _set_trainer_id(self, trainer_id):
+        self.proto_desc.trainer_id = trainer_id
+
+    def _set_trainers(self, trainers):
+        for trainer_num in trainers:
+            self.proto_desc.trainers.append(trainer_num)
+
+    def _set_use_cvm(self, use_cvm=False):
+        self.proto_desc.use_cvm = use_cvm
+
+    def _set_no_cvm(self, no_cvm=False):
+        self.proto_desc.no_cvm = no_cvm
+
+    def _set_scale_sparse_grad_with_batch_size(
+        self, scale_sparse_gradient_with_batch_size=True
+    ):
+        self.proto_desc.scale_sparse_gradient_with_batch_size = (
+            scale_sparse_gradient_with_batch_size
+        )
+
+    def _set_scale_datanorm(self, scale_datanorm=-1):
+        self.proto_desc.scale_datanorm = scale_datanorm
+
+    def _set_dump_slot(self, dump_slot):
+        self.proto_desc.dump_slot = dump_slot
+
+    def _set_mpi_rank(self, mpi_rank):
+        self.proto_desc.mpi_rank = mpi_rank
+
+    def _set_mpi_size(self, mpi_size):
+        self.proto_desc.mpi_size = mpi_size
+
+    def _set_dump_fields(self, dump_fields):
+        for field in dump_fields:
+            self.proto_desc.dump_fields.append(field)
+
+    def _set_is_dump_in_simple_mode(self, is_dump_in_simple_mode):
+        self.proto_desc.is_dump_in_simple_mode = is_dump_in_simple_mode
+
+    def _set_dump_num_decimals(self, dump_num_decimals):
+        self.proto_desc.dump_num_decimals = dump_num_decimals
+
+    def _set_dump_fields_path(self, path):
+        self.proto_desc.dump_fields_path = path
+
+    def _set_dump_file_num(self, dump_file_num):
+        self.proto_desc.dump_file_num = dump_file_num
+
+    def _set_user_define_dump_filename(self, user_define_dump_filename):
+        self.proto_desc.user_define_dump_filename = user_define_dump_filename
+
+    def _set_dump_converter(self, converter):
+        self.proto_desc.dump_converter = converter
+
+    def _set_enable_random_dump(self, enable_random_dump):
+        self.proto_desc.enable_random_dump = enable_random_dump
+
+    def _set_dump_interval(self, dump_interval):
+        self.proto_desc.dump_interval = dump_interval
+
+    def _set_random_with_lineid(self, random_with_lineid):
+        self.proto_desc.random_with_lineid = random_with_lineid
+
+    def _set_dump_param(self, dump_param):
+        for param in dump_param:
+            self.proto_desc.dump_param.append(param)
+
+    def _set_dump_fields_mode(self, mode):
+        self.proto_desc.dump_fields_mode = mode
+
+    def _set_worker_places(self, worker_places):
+        for place in worker_places:
+            self.proto_desc.worker_places.append(place)
+
+    def _set_use_ps_gpu(self, use_ps_gpu=False):
+        self.proto_desc.use_ps_gpu = use_ps_gpu
+
+    def _set_use_gpu_graph(self, use_gpu_graph=False):
+        self.proto_desc.use_gpu_graph = use_gpu_graph
+
+    def _set_thread_barrier(self, thread_barrier):
+        self.proto_desc.thread_barrier = thread_barrier
+
+    def _set_check_nan_var_names(self, check_nan_var_names):
+        for var in check_nan_var_names:
+            self.proto_desc.check_nan_var_names.append(var)
+
+    def _set_loss_names(self, loss_names):
+        for loss in loss_names:
+            self.proto_desc.loss_names.append(loss)
+
+    def _set_adjust_ins_weight(self, config_dict):
+        self.proto_desc.adjust_ins_weight_config.need_adjust = config_dict.get(
+            "need_adjust", False
+        )
+        self.proto_desc.adjust_ins_weight_config.nid_slot = config_dict.get(
+            "nid_slot", ""
+        )
+        self.proto_desc.adjust_ins_weight_config.nid_adjw_threshold = (
+            config_dict.get("nid_adjw_threshold", 0.0)
+        )
+        self.proto_desc.adjust_ins_weight_config.nid_adjw_ratio = (
+            config_dict.get("nid_adjw_ratio", 0.0)
+        )
+        self.proto_desc.adjust_ins_weight_config.ins_weight_slot = (
+            config_dict.get("ins_weight_slot", "")
+        )
+
+    def _set_copy_table_config(self, config_dict):
+        config = self.proto_desc.copy_table_config
+        config.need_copy = config_dict.get("need_copy", False)
+        config.batch_num = config_dict.get("batch_num", 100)
+
+        src_sparse_tables = config_dict.get("src_sparse_tables", [])
+        if not isinstance(src_sparse_tables, list):
+            src_sparse_tables = [src_sparse_tables]
+        dest_sparse_tables = config_dict.get("dest_sparse_tables", [])
+        if not isinstance(dest_sparse_tables, list):
+            dest_sparse_tables = [dest_sparse_tables]
+        if len(src_sparse_tables) != len(dest_sparse_tables):
+            raise ValueError(
+                "len(src_sparse_tables) != len(dest_sparse_tables),"
+                f" {len(src_sparse_tables)} vs {len(dest_sparse_tables)}"
+            )
+        for i in src_sparse_tables:
+            config.src_sparse_tables.append(i)
+        for i in dest_sparse_tables:
+            config.dest_sparse_tables.append(i)
+
+        src_dense_tables = config_dict.get("src_dense_tables", [])
+        if not isinstance(src_dense_tables, list):
+            src_dense_tables = [src_dense_tables]
+        dest_dense_tables = config_dict.get("dest_dense_tables", [])
+        if not isinstance(dest_dense_tables, list):
+            dest_dense_tables = [dest_dense_tables]
+        if len(src_dense_tables) != len(dest_dense_tables):
+            raise ValueError(
+                "len(src_dense_tables) != len(dest_dense_tables),"
+                f" {len(src_dense_tables)} vs {len(dest_dense_tables)}"
+            )
+        for i in src_dense_tables:
+            config.src_dense_tables.append(i)
+        for i in dest_dense_tables:
+            config.dest_dense_tables.append(i)
+
+        # user can also specify dense variables to copy,
+        # instead of copy dense table
+        src_var_list = config_dict.get("src_var_list", [])
+        if not isinstance(src_var_list, list):
+            src_var_list = [src_var_list]
+        dest_var_list = config_dict.get("dest_var_list", [])
+        if not isinstance(dest_var_list, list):
+            dest_var_list = [dest_var_list]
+        if len(src_var_list) != len(dest_var_list):
+            raise ValueError(
+                f"len(src_var_list) != len(dest_var_list), {len(src_var_list)} vs"
+                f" {len(dest_var_list)}"
+            )
+        for i in src_var_list:
+            config.src_var_list.append(i)
+        for i in dest_var_list:
+            config.dest_var_list.append(i)
+
+        dependency_map = config_dict.get("dependency_map", {})
+        for key in dependency_map:
+            m = config.table_dependency_map.add()
+            m.key = key
+            values = dependency_map[key]
+            if not isinstance(values, list):
+                values = [values]
+            if len(values) != 1:
+                raise ValueError(f"dependency len {len(values)} != 1")
+            for value in values:
+                m.values.append(value)
+        config.dense_pull_after_copy = config_dict.get(
+            "dense_pull_after_copy", True
+        )
+        config.enable_dependency = config_dict.get("enable_dependency", False)
+        config.sparse_copy_by_feasign = config_dict.get(
+            "sparse_copy_by_feasign", True
+        )
+
+    def _desc(self):
+        return self.proto_desc.SerializeToString()
+
+    def __str__(self):
+        from google.protobuf import text_format
+
+        return text_format.MessageToString(self.proto_desc)
+
+
+class MultiTrainer(TrainerDesc):
+    '''
+    Implement of MultiTrainer.
+    Can be init from TrainerDesc.
+    '''
+
+    def __init__(self):
+        super().__init__()
+        pass
+
+    def _set_program(self, program):
+        super()._set_program(program)
+        self._program = program
+
+    def _gen_trainer_desc(self):
+        super()._gen_trainer_desc()
+        self.proto_desc.class_name = "MultiTrainer"
+        self._device_worker._set_infer(self._infer)
+        self._device_worker._set_program(self._program)
+        self._device_worker._gen_worker_desc(self.proto_desc)