chore: import upstream snapshot with attribution

python/paddle/incubate/distributed/fleet/parameter_server/__init__.py

@@ -0,0 +1,13 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py

@@ -0,0 +1,956 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Convert the static program to distributed data-parallelism programs.
+"""
+
+import os
+import sys
+
+import paddle
+from paddle.base.compiler import CompiledProgram
+from paddle.distributed.fleet.base.private_helper_function import (
+    wait_server_ready,
+)
+from paddle.distributed.transpiler.distribute_transpiler import (
+    DistributeTranspilerConfig,
+)
+from paddle.framework import core
+from paddle.incubate.distributed.fleet.base import (
+    DistributedOptimizer,
+    Fleet,
+    Mode,
+)
+from paddle.incubate.distributed.fleet.parameter_server import version
+from paddle.incubate.distributed.fleet.parameter_server.distribute_transpiler.distributed_strategy import (
+    AsyncStrategy,
+    DistributedStrategy,
+    GeoStrategy,
+    HalfAsyncStrategy,
+    StrategyFactory,
+    SyncStrategy,
+    TrainerRuntimeConfig,  # noqa: F401
+)
+from paddle.incubate.distributed.fleet.parameter_server.ir import (
+    pserver_pass as server,
+    public,
+    trainer_pass as worker,
+)
+from paddle.incubate.distributed.fleet.parameter_server.ir.public import (
+    _get_lr_ops,
+    _has_global_step,
+    get_sparse_tablenames,
+)
+from paddle.incubate.distributed.fleet.parameter_server.mode import PSMode
+from paddle.incubate.distributed.fleet.parameter_server.pslib.optimizer_factory import (
+    DistributedAdam,  # noqa: F401
+)
+from paddle.incubate.distributed.fleet.role_maker import MPISymmetricRoleMaker
+from paddle.static import (
+    Executor,
+    Program,
+    default_main_program,
+    default_startup_program,
+)
+
+
+class FleetTranspiler(Fleet):
+    """
+    A subclass for compatibility with distributed.transpiler.DistributeTranspiler.
+    """
+
+    def __init__(self):
+        super().__init__(Mode.TRANSPILER)
+
+        self._inner_mode = None
+
+        if version.is_transpiler():
+            self._inner_mode = PSMode.TRANSPILER
+        else:
+            self._inner_mode = PSMode.PSLIB
+
+        self._strategy = None
+        self._transpiler = None
+        self._origin_main_program = None
+        self._origin_startup_program = None
+        self._communicator = None
+        self.startup_program = None
+        self.main_program = None
+
+        self._opt_info = None
+        self._local_ip = 0
+        self._fleet_ptr = None
+        self._main_programs = []
+        self._scopes = []
+        self._client2client_request_timeout_ms = 500000
+        self._client2client_connect_timeout_ms = 10000
+        self._client2client_max_retry = 3
+
+    def init(self, role_maker=None):
+        if role_maker is None:
+            role_maker = MPISymmetricRoleMaker()
+        super().init(role_maker)
+        if self._fleet_ptr is None:
+            self._fleet_ptr = core.Fleet()
+
+    def _init_transpiler_worker(self):
+        """
+        `init_worker` has many many functions to do before training,
+        first, wait for all parameter servers launch completely.
+        second, run executor to initialize startup program
+        third, wait for all worker initialize completely.
+
+        Returns:
+            None
+        """
+
+        def sync_strategy_envs():
+            kwargs = {}
+            kwargs["pserver_endpoints"] = (
+                self._role_maker.get_pserver_endpoints()
+            )
+            kwargs["trainer_id"] = self._role_maker.worker_index()
+            return kwargs
+
+        def geo_strategy_envs():
+            def get_sparse_attrs():
+                opt_init_map = {}
+                opt_init_map["gaussian_random"] = ["seed", "mean", "std"]
+                opt_init_map["fill_constant"] = ["value"]
+                opt_init_map["uniform_random"] = ["seed", "min", "max"]
+                opt_init_map["truncated_gaussian_random"] = [
+                    "seed",
+                    "mean",
+                    "std",
+                ]
+
+                dist_varnames = get_sparse_tablenames(
+                    self._origin_main_program, True
+                )
+                sparse_varnames = get_sparse_tablenames(
+                    self._origin_main_program, False
+                )
+
+                if len(dist_varnames) != 0:
+                    raise ValueError(
+                        "GeoStrategy can not support large scale embedding now, please use paddle.static.nn.embedding"
+                    )
+
+                init_attrs = []
+                for value_name in sparse_varnames:
+                    value_var = self._origin_main_program.global_block().vars[
+                        value_name
+                    ]
+                    value_attr = [
+                        value_name,
+                        ",".join([str(dim) for dim in value_var.shape]),
+                    ]
+                    for op in self._origin_startup_program.global_block().ops:
+                        if (
+                            op.type in opt_init_map.keys()
+                            and value_name == op.output("Out")[0]
+                        ):
+                            init_attr = [op.type]
+                            for attr in opt_init_map[op.type]:
+                                init_attr.append(str(op.attr(attr)))
+                            value_attr.append("&".join(init_attr))
+                            init_attrs.append(":".join(value_attr))
+                            break
+                return "#".join(init_attrs)
+
+            kwargs = {}
+            kwargs["trainers"] = self.worker_num()
+            kwargs["sparse_attrs"] = get_sparse_attrs()
+            return kwargs
+
+        # if MPISymmetricRoleMaker is defined
+        # we suppose a user wants to submit job on mpi cluster
+
+        if isinstance(self._role_maker, MPISymmetricRoleMaker):
+            # check whether server has been initialized
+            wait_server_ready(self.server_endpoints(to_string=False))
+
+        trainer_config = self._strategy.get_trainer_runtime_config()
+
+        print(trainer_config)
+
+        lrs = _has_global_step(_get_lr_ops(self._origin_main_program))
+
+        if lrs > 0:
+            kwargs = {"need_global_step": "1"}
+        else:
+            kwargs = {"need_global_step": "0"}
+
+        if isinstance(self._strategy, GeoStrategy):
+            geo_kwargs = geo_strategy_envs()
+            kwargs.update(geo_kwargs)
+        if isinstance(self._strategy, SyncStrategy):
+            sync_kwargs = sync_strategy_envs()
+            kwargs.update(sync_kwargs)
+
+        kwargs = kwargs if kwargs else None
+
+        send_ctx = fleet.compiled_config.get_communicator_send_context()
+
+        if self.compiled_config.is_geo_mode():
+            recv_ctx = fleet.compiled_config.get_communicator_recv_context(
+                recv_type=4
+            )
+        else:
+            recv_ctx = fleet.compiled_config.get_communicator_recv_context(
+                recv_type=1
+            )
+
+        from paddle.distributed.communicator import Communicator
+
+        self._communicator = Communicator(
+            trainer_config.mode, kwargs, trainer_config.get_communicator_flags()
+        )
+
+        self._communicator.init_with_ctx(send_ctx, recv_ctx)
+
+        if not self._communicator.is_running():
+            self._communicator.start()
+        else:
+            raise ValueError(
+                "Communicator can only be inited once, please check"
+            )
+
+    def init_worker(self):
+        """
+        `init_worker` has many many functions to do before training,
+        first, wait for all parameter servers launch completely.
+        second, run executor to initialize startup program
+        third, wait for all worker initialize completely.
+
+        Returns:
+            None
+        """
+        if self._inner_mode == PSMode.TRANSPILER:
+            self._init_transpiler_worker()
+        else:
+            raise NotImplementedError("add implement later")
+
+    def _init_transpiler_server(self, model_dir=None):
+        if not self.startup_program:
+            raise ValueError(
+                "startup_program is None, need invoke DistributedOptimizer.minimize first"
+            )
+
+        self._executor.run(self.startup_program)
+
+        if model_dir:
+            if not os.path.isdir(model_dir):
+                raise ValueError(f"There is no directory named '{model_dir}'")
+
+            sparse_varnames = self.compiled_config.get_sparse_varname_on_ps(
+                True
+            )
+            distributed_varnames = (
+                self.compiled_config.get_sparse_varname_on_ps(False)
+            )
+
+            remaining_vars = list(
+                filter(
+                    FleetTranspiler.__exclude_vars(
+                        sparse_varnames + distributed_varnames
+                    ),
+                    self.main_program.list_vars(),
+                )
+            )
+
+            paddle.static.load_vars(
+                self._executor,
+                main_program=self.main_program,
+                dirname=model_dir,
+                vars=remaining_vars,
+            )
+
+            self._load_sparse_params(
+                dirname=model_dir, varnames=sparse_varnames
+            )
+
+            # todo(tangwei12) load distributed vars
+            # self._load_sparse_params(dirname=model_dir, varnames=distributed_varnames)
+
+    def init_server(self, model_dir=None, **kwargs):
+        """
+        `init_server` has many many functions to do before start pserver,
+        first, run executor to initialize startup program,
+        second, if the `model_dir` is not empty, it will load parameters from it for increment training.
+
+        Args:
+            model_dir(str): The directory path.
+
+        Returns:
+            None
+        """
+
+        if self._inner_mode == PSMode.TRANSPILER:
+            self._init_transpiler_server(model_dir)
+        else:
+            raise NotImplementedError("add implement later")
+
+    def run_server(self):
+        """
+        `run_server` execute executor to start pserver main program.
+
+        Returns:
+            None
+        """
+
+        if self._inner_mode == PSMode.TRANSPILER:
+            if not self.main_program:
+                raise ValueError(
+                    "main_program is None, need invoke DistributedOptimizer.minimize first"
+                )
+
+            self._executor.run(self.main_program)
+        else:
+            raise NotImplementedError("add implement later")
+
+    def stop_worker(self):
+        """
+        Close this executor.
+
+        For the distributed training, this method would free the resource on PServers related to
+        the current Trainer.
+
+        Returns:
+            None
+        """
+
+        if self._inner_mode == PSMode.TRANSPILER:
+            self._communicator.stop()
+            if isinstance(self._role_maker, MPISymmetricRoleMaker):
+                self._role_maker._finalize()
+            self._executor.close()
+        else:
+            raise NotImplementedError("add implement later")
+
+    def distributed_optimizer(self, optimizer, strategy=None):
+        """
+        Optimizer for distributed training.
+
+        For the distributed training, this method would rebuild a new instance of DistributedOptimizer.
+        Which has basic Optimizer function and special features for distributed training.
+
+        Args:
+            optimizer(Optimizer): The executor to run for init server.
+            strategy(DistributeTranspilerConfig): Extra properties for distributed optimizer.
+
+        Returns:
+            TranspilerOptimizer: subclass of DistributedOptimizer.
+        """
+
+        if not isinstance(optimizer, paddle.optimizer.Optimizer):
+            raise ValueError("optimizer must be an instance of Optimizer")
+        if not self._is_initialized:
+            raise ValueError(
+                "fleet.init(role) to initialize before optimizer.minimize(loss)"
+            )
+
+        if not strategy:
+            _strategy = StrategyFactory.create_async_strategy()
+
+        if isinstance(strategy, DistributedStrategy):
+            _strategy = strategy
+        elif isinstance(strategy, DistributeTranspilerConfig):
+            if strategy.sync_mode:
+                _strategy = SyncStrategy()
+            else:
+                if strategy.runtime_split_send_recv:
+                    if strategy.geo_sgd_mode:
+                        _strategy = GeoStrategy(strategy.geo_sgd_need_push_nums)
+                    elif strategy.half_async:
+                        _strategy = HalfAsyncStrategy()
+                    else:
+                        _strategy = AsyncStrategy()
+                else:
+                    _strategy = HalfAsyncStrategy()
+                    # for half_async compatibility
+                    strategy.half_async = True
+                    strategy.runtime_split_send_recv = True
+            _strategy.set_program_config(strategy)
+        elif isinstance(strategy, dict):
+            if self._inner_mode != PSMode.PSLIB:
+                raise TypeError("Dict strategy can only be used at PSLIB Mode")
+
+            _strategy = StrategyFactory.create_async_strategy()
+            _strategy.set_pslib_runtime_config(strategy)
+        else:
+            raise TypeError(
+                "strategy must be an instance of DistributeTranspilerConfig, DistributedStrategy"
+            )
+
+        self._strategy = _strategy
+        self._optimizer = ParameterServerOptimizer(optimizer, _strategy)
+        return self._optimizer
+
+    def save_inference_model(
+        self,
+        executor,
+        dirname,
+        feeded_var_names,
+        target_vars,
+        main_program=None,
+        export_for_deployment=True,
+        legacy_format=False,
+    ):
+        """
+        Prune the given `main_program` to build a new program especially for inference,
+        and then save it and all related parameters to given `dirname` by the `executor`.
+        """
+
+        if self._inner_mode == PSMode.PSLIB:
+            raise NotImplementedError("add implement later")
+
+        if not isinstance(executor, Executor):
+            raise TypeError(
+                "in fleet.save_inference_model() function, executor must be as Executor type"
+            )
+
+        # Todo(MrChengmo): support recv&save GPU-Kernel for ps-gpu model save
+        if not isinstance(executor.place, paddle.CPUPlace):
+            save_executor = Executor(paddle.CPUPlace())
+        else:
+            save_executor = executor
+
+        if main_program is not None:
+            if isinstance(main_program, CompiledProgram):
+                raise TypeError(
+                    "in fleet.save_inference_model() function, main_program must be as Program type, CompiledProgram is not allowed"
+                )
+            paddle.static.save_inference_model(
+                dirname,
+                feeded_var_names,
+                target_vars,
+                executor,
+                main_program,
+                None,
+                None,
+                export_for_deployment,
+                legacy_format=legacy_format,
+            )
+        else:
+            paddle.static.save_inference_model(
+                dirname,
+                feeded_var_names,
+                target_vars,
+                executor,
+                self._origin_main_program,
+                None,
+                None,
+                export_for_deployment,
+                True,
+                legacy_format=legacy_format,
+            )
+
+            model_basename = "__model__"
+            model_filename = os.path.join(dirname, model_basename)
+
+            with open(model_filename, "rb") as f:
+                program_desc_str = f.read()
+
+            program = Program.parse_from_string(program_desc_str)
+            program._copy_dist_param_info_from(self.main_program)
+            self.save_persistables(executor, dirname, program)
+
+    def _load_sparse_params(self, dirname, varnames):
+        from paddle.distributed.communicator import LargeScaleKV
+
+        scale_kv = LargeScaleKV()
+        for varname in varnames:
+            origin_varname, _, _ = public._get_varname_parts(varname)
+            sparse_dir = os.path.join(dirname, origin_varname, varname)
+            scale_kv.load(varname, sparse_dir)
+
+    def _get_optimizer_status(self, op, param_name):
+        supported_opts = [
+            "sgd",
+            "adam",
+            "adagrad",
+            "adamax",
+            "momentum",
+            "lars_momentum",
+            "rmsprop",
+            "decayed_adagrad",
+            "ftrl",
+        ]
+
+        reshaped_val_map = {}
+        reshaped_val_map["sgd"] = []
+        reshaped_val_map["adam"] = ["moment1_0", "moment2_0"]
+        reshaped_val_map["adagrad"] = ["moment_0"]
+        reshaped_val_map["adamax"] = ["moment_0", "inf_norm_0"]
+        reshaped_val_map["momentum"] = ["velocity_0"]
+        reshaped_val_map["lars_momentum"] = ["velocity_0"]
+        reshaped_val_map["rmsprop"] = [
+            "momentum_0",
+            "mean_square_0",
+            "mean_grad_0",
+        ]
+        reshaped_val_map["decayed_adagrad"] = ["moment_0"]
+        reshaped_val_map["ftrl"] = ["squared_0", "linear_0"]
+
+        orishaped_val_map = {}
+        orishaped_val_map["adam"] = ["beta1_pow_acc_0", "beta2_pow_acc_0"]
+        orishaped_val_map["adamax"] = ["beta1_pow_acc_0"]
+
+        if op not in supported_opts:
+            raise ValueError(
+                f"fleet can not support optimizer: {op}, only this can be supported: {supported_opts}"
+            )
+
+        reshaped_names = [
+            param_name + "_" + val for val in reshaped_val_map[op]
+        ]
+
+        if op not in orishaped_val_map:
+            origin_names = []
+        else:
+            origin_names = [
+                param_name + "_" + val for val in orishaped_val_map[op]
+            ]
+        return reshaped_names, origin_names
+
+    def _get_optimizer_op(self, param_name):
+        opts = public._get_optimize_ops(self._origin_main_program)
+        for op in opts:
+            if (
+                "Param" in op.input_names
+                and "LearningRate" in op.input_names
+                and op.input("Param")[0] == param_name
+            ):
+                return op
+
+    def _save_dense_params(self, executor, dirname, context, main_program):
+        self._communicator.recv()
+
+        prog = Program()
+        block = prog.global_block()
+        local_vars = []
+
+        for name, var_ctx in context.items():
+            if len(var_ctx.origin_varnames()) != 1:
+                raise ValueError("Dense can not support split now.")
+
+            varname = var_ctx.origin_varnames()[0]
+            local_vars.append(varname)
+
+            optimizer = self._get_optimizer_op(varname)
+            reshaped_varnames, origin_varnames = self._get_optimizer_status(
+                optimizer.type, varname
+            )
+
+            for var_name in [varname, *reshaped_varnames, *origin_varnames]:
+                var = self._origin_main_program.global_block().vars[var_name]
+                block.append_op(
+                    type='recv_save',
+                    attrs={
+                        "trainer_id": self._role_maker.worker_index(),
+                        "shape": var.shape,
+                        "slice_shapes": [",".join([str(i) for i in var.shape])],
+                        "slice_varnames": [var.name],
+                        "remote_varnames": [var.name],
+                        "is_sparse": False,
+                        "endpoints": var_ctx.split_endpoints(),
+                        "file_path": os.path.join(dirname, var.name),
+                    },
+                )
+
+        executor.run(prog)
+        return local_vars
+
+    def _save_sparse_params(self, executor, dirname, context, main_program):
+        prog = Program()
+        block = prog.global_block()
+        local_vars = []
+
+        for name, var_ctx in context.items():
+            if len(var_ctx.origin_varnames()) != 1:
+                raise ValueError("Dense can not support split now.")
+
+            varname = var_ctx.origin_varnames()[0]
+            local_vars.append(varname)
+
+            optimizer = self._get_optimizer_op(varname)
+            reshaped_varnames, origin_varnames = self._get_optimizer_status(
+                optimizer.type, varname
+            )
+
+            var = self._origin_main_program.global_block().vars[varname]
+            slice_shapes = []
+            dims1 = ",".join([str(i) for i in var.shape[1:]])
+
+            for section in var_ctx.sections():
+                slice_shapes.append(str(section) + dims1)
+
+            block.append_op(
+                type='recv_save',
+                attrs={
+                    "trainer_id": self._role_maker.worker_index(),
+                    "shape": var.shape,
+                    "slice_shapes": slice_shapes,
+                    "slice_varnames": var_ctx.split_varnames(),
+                    "remote_varnames": var_ctx.split_varnames(),
+                    "is_sparse": True,
+                    "endpoints": var_ctx.split_endpoints(),
+                    "pserver_num": len(
+                        self._role_maker.get_pserver_endpoints()
+                    ),
+                    "file_path": os.path.join(dirname, var.name),
+                },
+            )
+
+            for reshaped_varname in reshaped_varnames:
+                var = self._origin_main_program.global_block().vars[
+                    reshaped_varname
+                ]
+
+                slice_varnames = []
+                remote_varnames = []
+                for i in range(len(var_ctx.split_varnames())):
+                    slice_varnames.append(f"{reshaped_varname}.block{i}")
+                    remote_varnames.append(reshaped_varname)
+
+                block.append_op(
+                    type='recv_save',
+                    attrs={
+                        "trainer_id": self._role_maker.worker_index(),
+                        "shape": var.shape,
+                        "slice_shapes": slice_shapes,
+                        "slice_varnames": slice_varnames,
+                        "remote_varnames": remote_varnames,
+                        "is_sparse": True,
+                        "endpoints": var_ctx.split_endpoints(),
+                        "pserver_num": len(
+                            self._role_maker.get_pserver_endpoints()
+                        ),
+                        "file_path": os.path.join(dirname, var.name),
+                    },
+                )
+
+            for origin_varname in origin_varnames:
+                var = self._origin_main_program.global_block().vars[
+                    origin_varname
+                ]
+
+                block.append_op(
+                    type='recv_save',
+                    attrs={
+                        "trainer_id": self._role_maker.worker_index(),
+                        "shape": var.shape,
+                        "slice_shapes": [",".join([str(i) for i in var.shape])],
+                        "slice_varnames": [origin_varname],
+                        "remote_varnames": [origin_varname],
+                        "is_sparse": False,
+                        "endpoints": var_ctx.split_endpoints()[:1],
+                        "file_path": os.path.join(dirname, var.name),
+                    },
+                )
+        executor.run(prog)
+        return context.keys()
+
+    def _save_distributed_params(
+        self, executor, dirname, context, main_program
+    ):
+        prog = Program()
+        block = prog.global_block()
+
+        for name, var_ctx in context.items():
+            block.append_op(
+                type='checkpoint_notify',
+                attrs={
+                    "varname": name,
+                    "is_slice": True,
+                    "slice_varnames": var_ctx.split_varnames(),
+                    "remote_varnames": var_ctx.split_varnames(),
+                    "endpoints": var_ctx.split_endpoints(),
+                    "dirname": dirname,
+                },
+            )
+
+        executor.run(prog)
+        return context.keys()
+
+    def _save_distributed_persistables(self, executor, dirname, main_program):
+        dense_ctx = fleet.compiled_config.get_communicator_recv_context(
+            recv_type=1
+        )
+
+        sparse_ctx = fleet.compiled_config.get_communicator_recv_context(
+            recv_type=2
+        )
+
+        distributed_ctx = fleet.compiled_config.get_communicator_recv_context(
+            recv_type=3
+        )
+
+        recv_dense_varnames = self._save_dense_params(
+            executor, dirname, dense_ctx, main_program
+        )
+
+        recv_sparse_varnames = self._save_sparse_params(
+            executor, dirname, sparse_ctx, main_program
+        )
+
+        recv_distributed_varnames = self._save_distributed_params(
+            executor, dirname, distributed_ctx, main_program
+        )
+
+        saved_varnames = (
+            recv_dense_varnames
+            + list(recv_sparse_varnames)
+            + list(recv_distributed_varnames)
+        )
+
+        remaining_vars = list(
+            filter(
+                FleetTranspiler.__exclude_vars(saved_varnames),
+                main_program.list_vars(),
+            )
+        )
+
+        paddle.static.save_vars(
+            executor,
+            main_program=main_program,
+            dirname=dirname,
+            vars=remaining_vars,
+        )
+
+    def save_persistables(self, executor, dirname, main_program=None, **kwargs):
+        """
+                This function filters out all variables with `persistable==True` from the
+                give `main_program` and then saves these variables to the folder `dirname`
+                or file `filename`.
+
+                The `dirname` is used to specify the folder where persistable variables
+                are going to be saved. If you would like to save variables in separate
+                files, set `filename` None;
+        if you would like to save all variables in a
+                single file, use `filename` to specify the file name.
+        """
+
+        if self._inner_mode == PSMode.PSLIB:
+            raise NotImplementedError("add implement later")
+
+        if not isinstance(executor, Executor):
+            raise TypeError(
+                "in fleet.save_persistables() function, executor must be as Executor type"
+            )
+        # Todo(MrChengmo): support recv&save GPU-Kernel for ps-gpu model save
+        if not isinstance(executor.place, paddle.CPUPlace):
+            save_executor = Executor(paddle.CPUPlace())
+        else:
+            save_executor = executor
+
+        if main_program is None:
+            main_program = self.main_program
+
+        if isinstance(main_program, CompiledProgram):
+            raise TypeError(
+                "in fleet.save_persistables() function, main_program must be as Program type, CompiledProgram is not allowed"
+            )
+
+        self._save_distributed_persistables(
+            save_executor, dirname, main_program
+        )
+
+    @staticmethod
+    def __exclude_vars(exclude_var_names=[]):
+        def is_valid(var):
+            if var.name in exclude_var_names:
+                return False
+
+            origin_varname, _, _ = public._get_varname_parts(var.name)
+            if origin_varname.endswith("@GRAD"):
+                return False
+
+            if origin_varname == "learning_rate_0":
+                return False
+
+            if (
+                var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH
+                or var.desc.type() == core.VarDesc.VarType.FETCH_LIST
+                or var.desc.type() == core.VarDesc.VarType.READER
+            ):
+                return False
+            return var.persistable
+
+        return is_valid
+
+
+# fleet is a global instance for parameter server.
+fleet = FleetTranspiler()
+
+
+class ParameterServerOptimizer(DistributedOptimizer):
+    """
+    DistributedOptimizer is a wrapper for paddle.base.optimizer
+    A user should pass a paddle.base.optimizer to DistributedOptimizer
+    minimize() function is implemented.
+    DistributedOptimizer is the starting point for a user who wants to
+    run distributed training. The optimized information will be stored in
+    Fleet() instance who holds the global information about current distributed
+    training.
+
+    Args:
+        optimizer(Optimizer): subclass of Optimizer.
+        strategy(DistributeTranspilerConfig): instance of DistributeTranspilerConfig.
+
+    Returns:
+        None
+    """
+
+    def __init__(self, optimizer, strategy, mode=PSMode.TRANSPILER):
+        super().__init__(optimizer, strategy)
+        self._mode = mode
+        if self._mode == PSMode.PSLIB:
+            self._optimizer_name = f"Distributed{optimizer.type.capitalize()}"
+            if optimizer.type != "adam":
+                print(
+                    "Currently, distributed optimizer only support Adam"
+                    "Will config built-in adam for you."
+                    "We will support more functions in DistributedOptimizer",
+                    sys.stderr,
+                )
+                self._optimizer_name = "DistributedAdam"
+
+            self._optimizer = globals()[self._optimizer_name](optimizer)
+        else:
+            self._optimizer = optimizer
+
+        self._window = 1
+        self.type = "downpour"
+        self.data_norm_name = [
+            ".batch_size",
+            ".batch_square_sum",
+            ".batch_sum",
+            ".batch_size@GRAD",
+            ".batch_square_sum@GRAD",
+            ".batch_sum@GRAD",
+        ]
+
+    def backward(
+        self,
+        loss,
+        startup_program=None,
+        parameter_list=None,
+        no_grad_set=None,
+        callbacks=None,
+    ):
+        raise NotImplementedError
+
+    def apply_gradients(self, params_grads):
+        raise NotImplementedError
+
+    def _build_trainer_programs(self, compiled_config):
+        _main = fleet._origin_main_program.clone()
+        _startup = fleet._origin_startup_program.clone()
+
+        if not compiled_config.is_geo_mode():
+            # for main program
+            _main = worker.delete_optimizer_pass(_main, compiled_config)
+            _main = worker.distributed_ops_pass(_main, compiled_config)
+            _main = worker.append_send_ops_pass(_main, compiled_config)
+
+            # for startup program
+            _startup = worker.fake_init_ops_pass(_startup, compiled_config)
+            _startup = worker.init_from_server_pass(_startup, compiled_config)
+            _startup = worker.delete_extra_optimizes_pass(
+                _startup, compiled_config
+            )
+        else:
+            _main = worker.append_send_ops_pass(_main, compiled_config)
+            _startup = _startup
+
+        return _main, _startup
+
+    def _build_pserver_programs(self, compiled_config):
+        _main = paddle.static.Program()
+        _startup = paddle.static.Program()
+
+        if not compiled_config.is_geo_mode():
+            _main = server.add_listen_and_serv_pass(_main, compiled_config)
+            _main = server.add_rpc_global_flags_pass(_main, compiled_config)
+            _main = server.add_optimizer_pass(_main, compiled_config)
+            _main = server.large_scale_sparse_pass(
+                _main, _main, compiled_config, False
+            )
+            _startup = server.build_pserver_startup_program_pass(
+                _startup, _main, compiled_config
+            )
+            _startup = server.large_scale_sparse_pass(
+                _startup, _main, compiled_config, True
+            )
+
+            if not compiled_config.is_sync_mode():
+                _main = server.delete_unused_in_main_pass(
+                    _main, compiled_config
+                )
+
+            _startup = server.delete_unused_in_startup_pass(
+                _startup, _main, compiled_config
+            )
+        else:
+            _main = server.add_listen_and_serv_pass(_main, compiled_config)
+            _main = server.add_rpc_global_flags_pass(_main, compiled_config)
+            _main = server.add_geo_optimizer_pass(_main, compiled_config)
+            _main = server.large_scale_sparse_pass(
+                _main, _main, compiled_config, False
+            )
+            _startup = server.build_pserver_startup_program_pass(
+                _startup, _main, compiled_config
+            )
+            _startup = server.large_scale_sparse_pass(
+                _startup, _main, compiled_config, True
+            )
+            _startup = server.delete_unused_in_startup_pass(
+                _startup, _main, compiled_config
+            )
+
+        return _main, _startup
+
+    def minimize(
+        self,
+        losses,
+        scopes=None,
+        startup_programs=None,
+        parameter_list=None,
+        no_grad_set=None,
+    ):
+        if isinstance(losses, list):
+            raise ValueError("need implement later")
+
+        self._optimizer.minimize(
+            losses, startup_programs, parameter_list, no_grad_set
+        )
+
+        fleet._origin_main_program = default_main_program().clone()
+        fleet._origin_startup_program = default_startup_program().clone()
+
+        compiled_config = public.CompileTimeStrategy(
+            fleet._origin_main_program,
+            fleet._origin_startup_program,
+            self._strategy,
+            fleet._role_maker,
+        )
+
+        fleet.compiled_config = compiled_config
+        fleet.main_program, fleet.startup_program = (
+            self._build_trainer_programs(compiled_config)
+            if fleet.is_worker()
+            else self._build_pserver_programs(compiled_config)
+        )

python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/distributed_strategy.py

@@ -0,0 +1,421 @@
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = []
+
+import os
+
+from paddle import base
+from paddle.distributed.transpiler.distribute_transpiler import (
+    DistributeTranspilerConfig,
+    ServerRuntimeConfig,
+)
+from paddle.incubate.distributed.fleet.parameter_server.mode import (
+    DistributedMode,
+)
+
+
+class TrainerRuntimeConfig:
+    def __init__(self):
+        self.mode = None
+        num_threads = os.getenv("CPU_NUM", "1")
+
+        self.runtime_configs = {}
+        self.runtime_configs['communicator_max_merge_var_num'] = os.getenv(
+            "FLAGS_communicator_max_merge_var_num", num_threads
+        )
+        self.runtime_configs['communicator_send_queue_size'] = os.getenv(
+            "FLAGS_communicator_send_queue_size", num_threads
+        )
+        self.runtime_configs['communicator_independent_recv_thread'] = (
+            os.getenv("FLAGS_communicator_independent_recv_thread", "1")
+        )
+        self.runtime_configs['communicator_min_send_grad_num_before_recv'] = (
+            os.getenv(
+                "FLAGS_communicator_min_send_grad_num_before_recv", num_threads
+            )
+        )
+        self.runtime_configs['communicator_thread_pool_size'] = os.getenv(
+            "FLAGS_communicator_thread_pool_size", "5"
+        )
+        self.runtime_configs['communicator_send_wait_times'] = os.getenv(
+            "FLAGS_communicator_send_wait_times", "5"
+        )
+        self.runtime_configs['communicator_is_sgd_optimizer'] = os.getenv(
+            "FLAGS_communicator_is_sgd_optimizer", "1"
+        )
+
+        # not used
+        self.runtime_configs['rpc_deadline'] = os.getenv(
+            "FLAGS_rpc_deadline", "180000"
+        )
+        self.runtime_configs['rpc_retry_times'] = os.getenv(
+            "FLAGS_rpc_retry_times", "3"
+        )
+
+    def get_communicator_flags(self):
+        need_keys = []
+        num_threads = os.getenv("CPU_NUM", "1")
+        mode_str = ""
+        if self.mode is None or self.mode == DistributedMode.ASYNC:
+            need_keys = self.runtime_configs.keys()
+            mode_str = "async"
+        elif (
+            self.mode == DistributedMode.SYNC
+            or self.mode == DistributedMode.HALF_ASYNC
+        ):
+            mode_str = "sync or half_async"
+            need_keys = [
+                'communicator_max_merge_var_num',
+                'communicator_send_wait_times',
+                'communicator_thread_pool_size',
+                'communicator_send_queue_size',
+            ]
+        elif self.mode == DistributedMode.GEO:
+            mode_str = "GEO"
+            need_keys = [
+                'communicator_thread_pool_size',
+                'communicator_send_wait_times',
+                'communicator_max_merge_var_num',
+                'communicator_send_queue_size',
+            ]
+        else:
+            raise ValueError("Unsupported Mode")
+
+        if (
+            self.mode == DistributedMode.SYNC
+            or self.mode == DistributedMode.HALF_ASYNC
+        ):
+            max_merge_var_num = self.runtime_configs[
+                'communicator_max_merge_var_num'
+            ]
+            send_queue_size = self.runtime_configs[
+                'communicator_send_queue_size'
+            ]
+            if max_merge_var_num != num_threads:
+                print(
+                    f'WARNING: In {mode_str} mode, communicator_max_merge_var_num '
+                    'must be equal to CPU_NUM. But received, '
+                    f'communicator_max_merge_var_num = {max_merge_var_num}, CPU_NUM = '
+                    f'{num_threads}. communicator_max_merge_var_num will be forced to {num_threads}.'
+                )
+                self.runtime_configs['communicator_max_merge_var_num'] = (
+                    num_threads
+                )
+            if send_queue_size != num_threads:
+                print(
+                    f'WARNING: In {mode_str} mode, communicator_send_queue_size '
+                    'must be equal to CPU_NUM. But received, '
+                    f'communicator_send_queue_size = {send_queue_size}, CPU_NUM = '
+                    f'{num_threads}. communicator_send_queue_size will be forced to {num_threads}.'
+                )
+                self.runtime_configs['communicator_send_queue_size'] = (
+                    num_threads
+                )
+
+        return {key: str(self.runtime_configs[key]) for key in need_keys}
+
+    def display(self, configs):
+        raw0, raw1, length = 45, 5, 50
+        h_format = "{:^45s}{:<5s}\n"
+        l_format = "{:<45s}{:<5s}\n"
+
+        border = "".join(["="] * length)
+        line = "".join(["-"] * length)
+
+        draws = ""
+        draws += border + "\n"
+        draws += h_format.format("TrainerRuntimeConfig Overview", "Value")
+        draws += line + "\n"
+
+        for k, v in configs.items():
+            draws += l_format.format(k, v)
+
+        draws += border
+
+        _str = f"\n{draws}\n"
+        return _str
+
+    def __repr__(self):
+        return self.display(self.get_communicator_flags())
+
+
+class PSLibRuntimeConfig:
+    def __init__(self):
+        self.runtime_configs = {}
+
+    def get_runtime_configs(self):
+        return self.runtime_configs
+
+
+class DistributedStrategy:
+    def __init__(self):
+        self._program_config = DistributeTranspilerConfig()
+        self._trainer_runtime_config = TrainerRuntimeConfig()
+        self._pslib_runtime_config = PSLibRuntimeConfig()
+        self._server_runtime_config = ServerRuntimeConfig()
+        num_threads = int(os.getenv("CPU_NUM", "1"))
+
+        self._build_strategy = base.BuildStrategy()
+
+        if num_threads > 1:
+            self._build_strategy.reduce_strategy = (
+                base.BuildStrategy.ReduceStrategy.Reduce
+            )
+        self.debug_opt = None
+        self.use_ps_gpu = False
+
+    def set_debug_opt(self, opt_info):
+        self.debug_opt = opt_info
+
+    def get_debug_opt(self):
+        opt_info = {}
+        if self.debug_opt is not None and isinstance(self.debug_opt, dict):
+            opt_info["dump_slot"] = bool(self.debug_opt.get("dump_slot", 0))
+            opt_info["dump_converter"] = str(
+                self.debug_opt.get("dump_converter", "")
+            )
+            opt_info["dump_fields"] = self.debug_opt.get("dump_fields", [])
+            opt_info["dump_file_num"] = self.debug_opt.get("dump_file_num", 16)
+            opt_info["dump_fields_path"] = self.debug_opt.get(
+                "dump_fields_path", ""
+            )
+            opt_info["dump_param"] = self.debug_opt.get("dump_param", [])
+        return opt_info
+
+    def get_program_config(self):
+        return self._program_config
+
+    def set_program_config(self, config):
+        if isinstance(config, DistributeTranspilerConfig):
+            self._program_config = config
+        elif isinstance(config, dict):
+            for key in config:
+                if hasattr(self._program_config, key):
+                    setattr(self._program_config, key, config[key])
+                else:
+                    raise ValueError(
+                        f"DistributeTranspilerConfig doesn't have key: {key}"
+                    )
+        else:
+            raise TypeError(
+                "program_config only accept input type: dict or DistributeTranspilerConfig"
+            )
+        self.check_program_config()
+
+    def check_program_config(self):
+        raise NotImplementedError(
+            "check_program_config must be implemented by derived class. You should use StrategyFactory to create DistributedStrategy."
+        )
+
+    def get_trainer_runtime_config(self):
+        return self._trainer_runtime_config
+
+    def set_trainer_runtime_config(self, config):
+        if isinstance(config, TrainerRuntimeConfig):
+            self._trainer_runtime_config = config
+        elif isinstance(config, dict):
+            for key, Value in config.items():
+                if key in self._trainer_runtime_config.runtime_configs:
+                    self._trainer_runtime_config.runtime_configs[key] = Value
+                else:
+                    raise ValueError(
+                        f"TrainerRuntimeConfig doesn't have key: {key}"
+                    )
+        else:
+            raise TypeError(
+                "trainer_runtime_config only accept input type: dict or TrainerRuntimeConfig"
+            )
+        self.check_trainer_runtime_config()
+
+    def check_trainer_runtime_config(self):
+        raise NotImplementedError(
+            "check_trainer_runtime_config must be implemented by derived class. You should use StrategyFactory to create DistributedStrategy."
+        )
+
+    def get_pslib_runtime_config(self):
+        return self._pslib_runtime_config
+
+    def set_pslib_runtime_config(self, config):
+        self._pslib_runtime_config.runtime_configs = config
+
+    def get_server_runtime_config(self):
+        return self._server_runtime_config
+
+    def set_server_runtime_config(self, config):
+        if isinstance(config, ServerRuntimeConfig):
+            self._server_runtime_config = config
+        elif isinstance(config, dict):
+            for key in config:
+                if hasattr(self._server_runtime_config, key):
+                    setattr(self._server_runtime_config, key, config[key])
+                else:
+                    raise ValueError(
+                        f"ServerRuntimeConfig doesn't have key: {key}"
+                    )
+        else:
+            raise TypeError(
+                "server_runtime_config only accept input type: dict or ServerRuntimeConfig"
+            )
+        self.check_server_runtime_config()
+
+    def check_server_runtime_config(self):
+        raise NotImplementedError(
+            "check_server_runtime_config must be implemented by derived class. You should use StrategyFactory to create DistributedStrategy."
+        )
+
+    def get_build_strategy(self):
+        return self._build_strategy
+
+    def set_build_strategy(self, config):
+        if isinstance(config, base.BuildStrategy):
+            self._build_strategy = config
+        elif isinstance(config, dict):
+            for key in config:
+                if hasattr(self._build_strategy, key):
+                    setattr(self._build_strategy, key, config[key])
+                else:
+                    raise ValueError(f"BuildStrategy doesn't have key: {key}")
+        else:
+            raise TypeError(
+                "build_strategy only accept input type: dict or BuildStrategy"
+            )
+        self.check_build_strategy()
+
+    def check_build_strategy(self):
+        raise NotImplementedError(
+            "check_build_strategy must be implemented by derived class. You should use StrategyFactory to create DistributedStrategy."
+        )
+
+
+class SyncStrategy(DistributedStrategy):
+    def __init__(self):
+        super().__init__()
+        self.check_program_config()
+        self.check_trainer_runtime_config()
+        self.check_server_runtime_config()
+        self.check_build_strategy()
+
+    def check_trainer_runtime_config(self):
+        self._trainer_runtime_config.mode = DistributedMode.SYNC
+
+    def check_program_config(self):
+        self._program_config.sync_mode = False
+        self._program_config.runtime_split_send_recv = True
+        self._program_config.half_async = True
+        self._program_config.completely_not_async = True
+
+    def check_server_runtime_config(self):
+        pass
+
+    def check_build_strategy(self):
+        self._build_strategy.async_mode = True
+
+
+class AsyncStrategy(DistributedStrategy):
+    def __init__(self):
+        super().__init__()
+        self.check_program_config()
+        self.check_trainer_runtime_config()
+        self.check_server_runtime_config()
+        self.check_build_strategy()
+
+    def check_trainer_runtime_config(self):
+        self._trainer_runtime_config.mode = DistributedMode.ASYNC
+
+    def check_program_config(self):
+        self._program_config.sync_mode = False
+        self._program_config.runtime_split_send_recv = True
+
+    def check_server_runtime_config(self):
+        pass
+
+    def check_build_strategy(self):
+        self._build_strategy.async_mode = True
+
+
+class HalfAsyncStrategy(DistributedStrategy):
+    def __init__(self):
+        super().__init__()
+        self.check_program_config()
+        self.check_trainer_runtime_config()
+        self.check_server_runtime_config()
+        self.check_build_strategy()
+
+    def check_trainer_runtime_config(self):
+        self._trainer_runtime_config.mode = DistributedMode.HALF_ASYNC
+
+    def check_program_config(self):
+        self._program_config.sync_mode = False
+        self._program_config.runtime_split_send_recv = True
+        self._program_config.half_async = True
+
+    def check_server_runtime_config(self):
+        pass
+
+    def check_build_strategy(self):
+        self._build_strategy.async_mode = True
+
+
+class GeoStrategy(DistributedStrategy):
+    def __init__(self, update_frequency=100):
+        super().__init__()
+        self._program_config.geo_sgd_need_push_nums = update_frequency
+        self.check_program_config()
+        self.check_trainer_runtime_config()
+        self.check_server_runtime_config()
+        self.check_build_strategy()
+
+    def check_program_config(self):
+        self._program_config.sync_mode = False
+        self._program_config.runtime_split_send_recv = True
+        self._program_config.geo_sgd_mode = True
+
+    def check_trainer_runtime_config(self):
+        self._trainer_runtime_config.mode = DistributedMode.GEO
+
+        self._trainer_runtime_config.runtime_configs[
+            'communicator_send_queue_size'
+        ] = self._program_config.geo_sgd_need_push_nums
+
+        self._trainer_runtime_config.runtime_configs[
+            'communicator_max_merge_var_num'
+        ] = self._program_config.geo_sgd_need_push_nums
+
+    def check_server_runtime_config(self):
+        pass
+
+    def check_build_strategy(self):
+        self._build_strategy.async_mode = True
+
+
+class StrategyFactory:
+    def __init_(self):
+        pass
+
+    @staticmethod
+    def create_sync_strategy():
+        return SyncStrategy()
+
+    @staticmethod
+    def create_half_async_strategy():
+        return HalfAsyncStrategy()
+
+    @staticmethod
+    def create_async_strategy():
+        return AsyncStrategy()
+
+    @staticmethod
+    def create_geo_strategy(update_frequency=100):
+        return GeoStrategy(update_frequency)

python/paddle/incubate/distributed/fleet/parameter_server/ir/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

python/paddle/incubate/distributed/fleet/parameter_server/ir/heter_trainer_pass.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+
+import paddle
+from paddle.incubate.distributed.fleet.parameter_server.ir.trainer_pass import (
+    create_heter_program,
+    create_trainer_program,
+    find_block_joints,
+    find_heter_ops,
+    union_forward_gradient_op,
+)
+
+
+def split_heter_worker_ops_pass(program, config, stage_id, device):
+    """
+    split heter worker program from origin-program
+    1. find heter op (located on different device)
+    2. find input&output of every heter-block
+    3. create heter worker program, add listen&serv op
+    """
+    default_device = "cpu"
+    program, heter_ops, _, program_block_ops = find_heter_ops(
+        program, default_device
+    )
+    if len(heter_ops) == 0:
+        warnings.warn(
+            "Currently running in Heter Parameter Server mode, but no OP running on heterogeneous devices, Please check your code."
+        )
+        return program
+
+    program_block_ops = union_forward_gradient_op(program_block_ops)
+    block_vars_detail = find_block_joints(program, program_block_ops, heter_ops)
+    heter_program = paddle.static.Program()
+    create_heter_program(
+        program,
+        config,
+        heter_program,
+        program_block_ops,
+        heter_ops,
+        block_vars_detail,
+        device,
+        stage_id,
+    )
+    return heter_program
+
+
+def split_trainer_ops_pass(program, config, default_device="cpu"):
+    """
+    split cpu-trainer program from origin-program
+    1. find heter op (located on different device)
+    2. find input&output of every heter-block
+    3. create cpu-trainer program, add send&recv op
+    """
+    # Todo: support user define default_device (MrChengmo)
+    default_device_ = default_device
+    program, heter_ops, default_ops, program_block_ops = find_heter_ops(
+        program, default_device_
+    )
+    program_block_ops = union_forward_gradient_op(program_block_ops)
+
+    block_vars_detail = find_block_joints(program, program_block_ops, heter_ops)
+    trainer_program = program.clone()
+    create_trainer_program(
+        trainer_program, program, config, program_block_ops, block_vars_detail
+    )
+    return trainer_program

python/paddle/incubate/distributed/fleet/parameter_server/ir/ps_dispatcher.py

@@ -0,0 +1,127 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class PSDispatcher:
+    """
+    PSDispatcher is the base class for dispatching vars
+    into different pserver instance.
+    You need to implement the `dispatch` interface.
+    """
+
+    def __init__(self, pserver_endpoints):
+        self._eps = pserver_endpoints
+        self._step = 0
+
+    @property
+    def eps(self):
+        return self._eps
+
+    def reset(self):
+        """
+        reset the step counter, set it zero.
+        """
+        self._step = 0
+
+    def dispatch(self, varlist):
+        """
+        Args:
+            varlist(list): a list of Variables
+        Returns:
+            a map of pserver endpoint -> varname
+        """
+        raise NotImplementedError("Interface has not been implemented.")
+
+
+class HashName(PSDispatcher):
+    """
+    Hash variable names to several endpoints using python
+    "hash()" function.
+
+    Args:
+        pserver_endpoints (list): list of endpoint(ip:port).
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> from paddle.incubate.distributed.fleet.parameter_server.ir.ps_dispatcher import RoundRobin
+
+            >>> pserver_endpoints = ["127.0.0.1:6007", "127.0.0.1:6008"]
+            >>> vars = ["var1", "var2", "var3", "var4", "var5"]
+
+            >>> rr = HashName(pserver_endpoints)
+            >>> rr.dispatch(vars)
+
+    """
+
+    def __init__(self, pserver_endpoints):
+        super().__init__(pserver_endpoints)
+
+    def _hash_block(self, block_str, total):
+        return hash(block_str) % total
+
+    def dispatch(self, varlist):
+        """
+        use `HashName` method to dispatch variables with each parameter server.
+        Args:
+            varlist (list): a list of Variables
+
+        """
+        eplist = []
+        for var in varlist:
+            server_id = self._hash_block(var.name(), len(self._eps))
+            server_for_param = self._eps[server_id]
+            eplist.append(server_for_param)
+        return eplist
+
+
+class RoundRobin(PSDispatcher):
+    """
+    Distribute variables to several endpoints using
+    RondRobin<https://en.wikipedia.org/wiki/Round-robin_scheduling> method.
+
+    Args:
+        pserver_endpoints (list): list of endpoint(ip:port).
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> from paddle.incubate.distributed.fleet.parameter_server.ir.ps_dispatcher import RoundRobin
+
+            >>> pserver_endpoints = ["127.0.0.1:6007", "127.0.0.1:6008"]
+            >>> vars = ["var1", "var2", "var3", "var4", "var5"]
+
+            >>> rr = RoundRobin(pserver_endpoints)
+            >>> rr.dispatch(vars)
+
+    """
+
+    def __init__(self, pserver_endpoints):
+        super().__init__(pserver_endpoints)
+
+    def dispatch(self, varlist):
+        """
+        use `RoundRobin` method to dispatch variables with each parameter server.
+        Args:
+            varlist (list): a list of Variables
+
+        """
+        eplist = []
+        for var in varlist:
+            server_for_param = self._eps[self._step]
+            eplist.append(server_for_param)
+            self._step += 1
+            if self._step >= len(self._eps):
+                self._step = 0
+        return eplist

python/paddle/incubate/distributed/fleet/parameter_server/ir/ufind.py

@@ -0,0 +1,64 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class UnionFind:
+    """Union-find data structure.
+
+    Union-find is a data structure that keeps track of a set of elements partitioned
+    into a number of disjoint (non-overlapping) subsets.
+
+    Reference:
+    https://en.wikipedia.org/wiki/Disjoint-set_data_structure
+
+    Args:
+      elements(list): The initialize element list.
+    """
+
+    def __init__(self, elements=None):
+        self._parents = []  # index -> parent index
+        self._index = {}  # element -> index
+        self._curr_idx = 0
+        if not elements:
+            elements = []
+        for ele in elements:
+            self._parents.append(self._curr_idx)
+            self._index.update({ele: self._curr_idx})
+            self._curr_idx += 1
+
+    def find(self, x):
+        # Find the root index of given element x,
+        # execute the path compress while finding the root index
+        if x not in self._index:
+            return -1
+        idx = self._index[x]
+        while idx != self._parents[idx]:
+            t = self._parents[idx]
+            self._parents[idx] = self._parents[t]
+            idx = t
+        return idx
+
+    def union(self, x, y):
+        # Union two given element
+        x_root = self.find(x)
+        y_root = self.find(y)
+
+        if x_root == y_root:
+            return
+        self._parents[x_root] = y_root
+
+    def is_connected(self, x, y):
+        # If two given elements have the same root index,
+        # then they are connected.
+        return self.find(x) == self.find(y)

python/paddle/incubate/distributed/fleet/parameter_server/ir/vars_metatools.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from functools import reduce
+
+from paddle.framework import core
+from paddle.framework.io import Variable
+
+dtype_to_size = {
+    core.VarDesc.VarType.FP16: 2,
+    core.VarDesc.VarType.FP32: 4,
+    core.VarDesc.VarType.FP64: 8,
+    core.VarDesc.VarType.INT16: 2,
+    core.VarDesc.VarType.INT32: 4,
+    core.VarDesc.VarType.INT64: 8,
+    core.VarDesc.VarType.BOOL: 1,
+    core.VarDesc.VarType.UINT8: 1,
+}
+
+
+class VarBlock:
+    def __init__(self, varname, offset, size):
+        self.varname = varname
+        # NOTE: real offset is offset * size
+        self.offset = offset
+        self.size = size
+
+    def __str__(self):
+        return f"{self.varname}:{int(self.offset)}:{int(self.size)}"
+
+
+def create_var_struct(var):
+    if var.type == core.VarDesc.VarType.SELECTED_ROWS:
+        lod_level = None
+    elif var.type == core.VarDesc.VarType.DENSE_TENSOR:
+        lod_level = var.lod_level
+    else:
+        raise ValueError("can only support SELECTED_ROWS/DENSE_TENSOR now")
+
+    return VarStruct(
+        var.name, var.shape, var.dtype, var.type, lod_level, var.persistable
+    )
+
+
+class VarStruct:
+    """
+    record part properties of a Variable in python.
+    """
+
+    def __init__(self, name, shape, dtype, type, lod_level, persistable):
+        self.name = name
+        self.shape = shape
+        self.dtype = dtype
+        self.type = type
+        self.lod_level = lod_level
+        self.persistable = persistable
+        self.m_size = 1
+        self.m_size = reduce(lambda x, y: x * y, shape, 1)
+        self.m_size *= dtype_to_size[dtype]
+
+    def __str__(self):
+        return f"N: {self.name}, S: {self.shape}, D: {self.dtype}, T: {self.type}, LL: {self.lod_level}, P: {self.persistable}, M: {self.m_size}"
+
+
+class VarDistributed:
+    """
+    a class to record the var distributed on parameter servers.
+    the class will record the relationship between origin var and slice var.
+    the slice var's properties, such as type/shape/offset/endpoint.
+    """
+
+    def __init__(
+        self,
+        origin_var,
+        slice_var,
+        is_slice=None,
+        block_id=None,
+        offset=None,
+        vtype=None,
+        endpoint=None,
+    ):
+        """
+        Args:
+            origin_var(Variable|VarStruct): origin var properties
+            slice_var(Variable|VarStruct): slice var properties
+            is_slice(bool|None): slice or not, slice_var=True/False and its block size > 8192 are the judgement standard.
+            block_id(int|None): the number about the slice var.
+            offset(int|None): if the slice var is sliced, offset is the numel before the var.
+            vtype(str|None): a tag, such as Optimizer/Param/RemotePrefetch.
+            endpoint(str|None): which parameter the slice var on, such as "127.0.0.1:1001"
+        """
+
+        if isinstance(origin_var, Variable):
+            self.origin = create_var_struct(origin_var)
+        else:
+            self.origin = origin_var
+
+        if isinstance(slice_var, Variable):
+            self.slice = create_var_struct(slice_var)
+        else:
+            self.slice = slice_var
+
+        if self.equal(self.origin, self.slice):
+            self.is_slice = False
+            self.block_id = 0
+            self.offset = 0
+        else:
+            self.is_slice = True
+            self.block_id = 0
+            self.offset = 0
+
+        if is_slice is not None:
+            self.is_slice = is_slice
+        if block_id is not None:
+            self.block_id = block_id
+        if offset is not None:
+            self.offset = offset
+
+        self.vtype = vtype
+        self.endpoint = endpoint
+
+    @staticmethod
+    def equal(var1, var2):
+        """
+        the two var is equal or not.
+        Returns:
+            bool: equal will return True else False
+        """
+        assert isinstance(var1, VarStruct) and isinstance(var2, VarStruct)
+
+        return (
+            var1.name == var2.name
+            and var1.type == var2.type
+            and var1.shape == var2.shape
+            and var1.dtype == var2.dtype
+            and var1.lod_level == var2.lod_level
+            and var1.persistable == var2.persistable
+        )
+
+    def __str__(self):
+        origin_var_str = f"{self.origin.name} : base.{self.origin.type}.shape{self.origin.shape}.astype({self.origin.dtype})"
+
+        slice_var_str = (
+            f"{self.slice.name} : base.{self.slice.type}.shape{self.slice.shape}.astype({self.slice.dtype})"
+            f".slice({self.is_slice}).block({self.block_id}).offset({self.offset})"
+        )
+
+        return f"var owned: {self.vtype}, origin var: ( {origin_var_str} ), slice var: ( {slice_var_str} ), endpoint: {self.endpoint} "
+
+
+class VarsDistributed:
+    """
+    a gather about VarDistributed with many methods to find distributed vars.
+    through the class, we can get overview about the distributed parameters on parameter servers.
+    this class may centralized and convenient for developer to manage and get variable's distribute.
+    other module can also use this to find variables such io.py.
+    """
+
+    def __init__(self):
+        self.distributed_vars = []
+
+    def add_distributed_var(
+        self,
+        origin_var,
+        slice_var,
+        is_slice=None,
+        block_id=None,
+        offset=None,
+        vtype=None,
+        endpoint=None,
+    ):
+        """
+        add distributed var in this.
+
+        Args:
+            origin_var(Variable|VarStruct): origin var properties
+            slice_var(Variable|VarStruct): slice var properties
+            is_slice(bool|None): slice or not, slice_var=True/False and its block size > 8192 are the judgement standard.
+            block_id(int|None): the number about the slice var.
+            offset(int|None): if the slice var is sliced, offset is the numel before the var.
+            vtype(str|None): a tag, such as Optimizer/Param/RemotePrefetch.
+            endpoint(str|None): which parameter the slice var on, such as "127.0.0.1:1001"
+        Returns:
+            None
+        """
+        self.distributed_vars.append(
+            VarDistributed(
+                origin_var,
+                slice_var,
+                is_slice,
+                block_id,
+                offset,
+                vtype,
+                endpoint,
+            )
+        )

python/paddle/incubate/distributed/fleet/parameter_server/mode.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class PSMode:
+    """
+    There are various mode for fleet, each of them is designed for different model.
+    """
+
+    TRANSPILER = 1
+    PSLIB = 2
+
+
+class DistributedMode:
+    SYNC = 0
+    ASYNC = 1
+    HALF_ASYNC = 2
+    GEO = 3

python/paddle/incubate/distributed/fleet/parameter_server/pslib/.gitignore

@@ -0,0 +1 @@
+ps_pb2.py

python/paddle/incubate/distributed/fleet/parameter_server/pslib/node.py

@@ -0,0 +1,798 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+"""Definition of Server and Worker."""
+
+# NOTE: reduce removed in functools in python3
+from functools import reduce
+
+from . import ps_pb2 as pslib
+
+
+class Server:
+    """
+    A Server basic class
+    it's a base class, does not have implementation
+    """
+
+    def __init__(self):
+        pass
+
+
+class Worker:
+    """
+    A Worker basic class.
+    it's a base class, does not have implementation
+    """
+
+    def __init__(self):
+        pass
+
+
+class DownpourServer(Server):
+    """
+    DownpourServer class is used to generate server program_desc
+    Args:
+        server: it is pslib.ServerParameter()
+    Examples:
+        server = DownpourServer()
+    """
+
+    def __init__(self):
+        self._server = pslib.ServerParameter()
+        self._server.downpour_server_param.service_param.server_class = (
+            "DownpourBrpcPsServer"
+        )
+        self._server.downpour_server_param.service_param.client_class = (
+            "DownpourBrpcPsClient"
+        )
+        self._server.downpour_server_param.service_param.service_class = (
+            "DownpourPsService"
+        )
+        self._server.downpour_server_param.service_param.start_server_port = 0
+        self._server.downpour_server_param.service_param.server_thread_num = 12
+
+    def add_sparse_table(self, table_id, strategy):
+        """
+        Args:
+            table_id(int): id of sparse params table
+            strategy(dict): the config dict.
+        Returns:
+            return None
+        """
+
+        for table in self._server.downpour_server_param.downpour_table_param:
+            if table.table_id == table_id:
+                if table.type == pslib.PS_SPARSE_TABLE:
+                    return
+                else:
+                    raise ValueError(
+                        f"expect table {table_id} type={pslib.PS_SPARSE_TABLE}, but actual type={table.type}"
+                    )
+        if strategy is None:
+            strategy = {}
+        table = self._server.downpour_server_param.downpour_table_param.add()
+        table.table_id = table_id
+        table.type = pslib.PS_SPARSE_TABLE
+
+        support_sparse_key_list = [
+            'sparse_table_class',
+            'sparse_compress_in_save',
+            'sparse_shard_num',
+            'sparse_accessor_class',
+            'sparse_learning_rate',
+            'sparse_initial_g2sum',
+            'sparse_initial_range',
+            'sparse_weight_bounds',
+            'sparse_embedx_dim',
+            'sparse_embedx_threshold',
+            'sparse_nonclk_coeff',
+            'sparse_click_coeff',
+            'sparse_base_threshold',
+            'sparse_delta_threshold',
+            'sparse_delta_keep_days',
+            'sparse_delete_after_unseen_days',
+            'sparse_show_click_decay_rate',
+            'sparse_delete_threshold',
+            'sparse_converter',
+            'sparse_deconverter',
+            'sparse_enable_cache',
+            'sparse_cache_rate',
+            'sparse_cache_file_num',
+            'sparse_beta1_decay_rate',
+            'sparse_beta2_decay_rate',
+            'sparse_ada_epsilon',
+            'sparse_optimizer',
+            'sparse_ssd_unseenday_threshold',
+            'embed_sparse_optimizer',
+            'embed_sparse_learning_rate',
+            'embed_sparse_weight_bounds',
+            'embed_sparse_initial_range',
+            'embed_sparse_initial_g2sum',
+            'embed_sparse_beta1_decay_rate',
+            'embed_sparse_beta2_decay_rate',
+            'embedx_sparse_optimizer',
+            'embedx_sparse_learning_rate',
+            'embedx_sparse_weight_bounds',
+            'embedx_sparse_initial_range',
+            'embedx_sparse_initial_g2sum',
+            'embedx_sparse_beta1_decay_rate',
+            'embedx_sparse_beta2_decay_rate',
+        ]
+
+        for key in strategy:
+            if key not in support_sparse_key_list:
+                raise ValueError(f"strategy key '{key}' not support")
+
+        support_table_class = ['DownpourSparseTable', 'DownpourSparseSSDTable']
+        if strategy.get('sparse_table_class') is not None:
+            table_class = strategy.get('sparse_table_class')
+            if table_class not in support_table_class:
+                raise ValueError(
+                    f"support sparse_table_class: [ 'DownpourSparseTable', 'DownpourSparseSSDTable'], \
+                        but actual {table_class}"
+                )
+        else:
+            table_class = 'DownpourSparseTable'
+
+        table.table_class = table_class
+
+        if (
+            table_class == 'DownpourSparseTable'
+            or table_class == 'DownpourSparseSSDTable'
+        ):
+            table.enable_sparse_table_cache = strategy.get(
+                'sparse_enable_cache', True
+            )
+            table.sparse_table_cache_rate = strategy.get(
+                'sparse_cache_rate', 0.00055
+            )
+            table.sparse_table_cache_file_num = strategy.get(
+                'sparse_cache_file_num', 16
+            )
+            table.compress_in_save = strategy.get(
+                'sparse_compress_in_save', True
+            )
+            table.shard_num = strategy.get('sparse_shard_num', 1000)
+            # DownpourFeatureValueAccessor: for ctr task, has cvm, embedding and sgd info
+            # DownpourCtrAccessor         : for ctr task, has cvm, slot, embedding and sgd info
+            # DownpourSparseValueAccessor : for general task, has embedding and sgd info
+            # DownpourCtrDoubleAccessor   : for ctr task, which show clk are in double
+            # DownpourUnitAccessor        : for ctr task, has cvm, slot, embedding and sgd info
+
+            support_accessor_class = [
+                'DownpourFeatureValueAccessor',
+                'DownpourCtrAccessor',
+                'DownpourCtrDymfAccessor',
+                'DownpourSparseValueAccessor',
+                'DownpourCtrDoubleAccessor',
+                'DownpourUnitAccessor',
+                'DownpourDoubleUnitAccessor',
+            ]
+            if strategy.get('sparse_accessor_class') is not None:
+                accessor_class = strategy.get('sparse_accessor_class')
+                if accessor_class not in support_accessor_class:
+                    raise ValueError(
+                        f"support sparse_accessor_class: ['DownpourFeatureValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDymfAccessor', \
+                        'DownpourSparseValueAccessor', 'DownpourCtrDoubleAccessor'], \
+                            but actual {accessor_class}"
+                    )
+            else:
+                accessor_class = 'DownpourCtrAccessor'
+
+            table.accessor.accessor_class = accessor_class
+
+            if (
+                accessor_class == 'DownpourFeatureValueAccessor'
+                or accessor_class == 'DownpourCtrAccessor'
+                or accessor_class == 'DownpourCtrDoubleAccessor'
+            ):
+                table.accessor.sparse_sgd_param.learning_rate = strategy.get(
+                    'sparse_learning_rate', 0.05
+                )
+                table.accessor.sparse_sgd_param.initial_g2sum = strategy.get(
+                    'sparse_initial_g2sum', 3
+                )
+                table.accessor.sparse_sgd_param.initial_range = strategy.get(
+                    'sparse_initial_range', 1e-4
+                )
+                if strategy.get('sparse_weight_bounds') is None:
+                    table.accessor.sparse_sgd_param.weight_bounds.extend(
+                        [-10, 10]
+                    )
+                else:
+                    table.accessor.sparse_sgd_param.weight_bounds.extend(
+                        strategy.get('sparse_weight_bounds')
+                    )
+                table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8)
+                table.accessor.embedx_threshold = strategy.get(
+                    'sparse_embedx_threshold', 10
+                )
+                table.accessor.fea_dim = int(table.accessor.embedx_dim) + 3
+                table.accessor.downpour_accessor_param.nonclk_coeff = (
+                    strategy.get('sparse_nonclk_coeff', 0.1)
+                )
+                table.accessor.downpour_accessor_param.click_coeff = (
+                    strategy.get('sparse_click_coeff', 1)
+                )
+                table.accessor.downpour_accessor_param.base_threshold = (
+                    strategy.get('sparse_base_threshold', 1.5)
+                )
+                table.accessor.downpour_accessor_param.delta_threshold = (
+                    strategy.get('sparse_delta_threshold', 0.25)
+                )
+                table.accessor.downpour_accessor_param.delta_keep_days = (
+                    strategy.get('sparse_delta_keep_days', 16)
+                )
+                table.accessor.downpour_accessor_param.delete_after_unseen_days = strategy.get(
+                    'sparse_delete_after_unseen_days', 30
+                )
+                table.accessor.downpour_accessor_param.ssd_unseenday_threshold = strategy.get(
+                    'sparse_ssd_unseenday_threshold', 1
+                )
+                table.accessor.downpour_accessor_param.show_click_decay_rate = (
+                    strategy.get('sparse_show_click_decay_rate', 0.98)
+                )
+                table.accessor.downpour_accessor_param.delete_threshold = (
+                    strategy.get('sparse_delete_threshold', 0.8)
+                )
+                converter = strategy.get(
+                    'sparse_converter',
+                    "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)",
+                )
+                deconverter = strategy.get(
+                    'sparse_deconverter',
+                    "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)",
+                )
+
+                table1 = table.accessor.table_accessor_save_param.add()
+                table1.param = 1
+                table1.converter = converter
+                table1.deconverter = deconverter
+
+                table2 = table.accessor.table_accessor_save_param.add()
+                table2.param = 2
+                table2.converter = converter
+                table2.deconverter = deconverter
+            elif accessor_class == 'DownpourSparseValueAccessor':
+                optimizer_name = strategy.get("sparse_optimizer", "adam")
+                table.accessor.sparse_commonsgd_param.name = optimizer_name
+                table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8)
+                table.accessor.fea_dim = int(table.accessor.embedx_dim)
+                if optimizer_name == "naive":
+                    table.accessor.sparse_commonsgd_param.naive.learning_rate = strategy.get(
+                        'sparse_learning_rate', 0.05
+                    )
+                    table.accessor.sparse_commonsgd_param.naive.initial_range = strategy.get(
+                        'sparse_initial_range', 1e-4
+                    )
+                    if strategy.get('sparse_weight_bounds') is None:
+                        table.accessor.sparse_commonsgd_param.naive.weight_bounds.extend(
+                            [-10, 10]
+                        )
+                    else:
+                        table.accessor.sparse_commonsgd_param.naive.weight_bounds.extend(
+                            strategy.get('sparse_weight_bounds')
+                        )
+                elif optimizer_name == "adagrad":
+                    table.accessor.sparse_commonsgd_param.adagrad.learning_rate = strategy.get(
+                        'sparse_learning_rate', 0.05
+                    )
+                    table.accessor.sparse_commonsgd_param.adagrad.initial_range = strategy.get(
+                        'sparse_initial_range', 1e-4
+                    )
+                    table.accessor.sparse_commonsgd_param.adagrad.initial_g2sum = strategy.get(
+                        'sparse_initial_g2sum', 3
+                    )
+                    if strategy.get('sparse_weight_bounds') is None:
+                        table.accessor.sparse_commonsgd_param.adagrad.weight_bounds.extend(
+                            [-10, 10]
+                        )
+                    else:
+                        table.accessor.sparse_commonsgd_param.adagrad.weight_bounds.extend(
+                            strategy.get('sparse_weight_bounds')
+                        )
+                elif optimizer_name == "adam":
+                    table.accessor.sparse_commonsgd_param.adam.learning_rate = (
+                        strategy.get('sparse_learning_rate', 0.001)
+                    )
+                    table.accessor.sparse_commonsgd_param.adam.initial_range = (
+                        strategy.get('sparse_initial_range', 1e-4)
+                    )
+                    table.accessor.sparse_commonsgd_param.adam.beta1_decay_rate = strategy.get(
+                        'sparse_beta1_decay_rate', 0.9
+                    )
+                    table.accessor.sparse_commonsgd_param.adam.beta2_decay_rate = strategy.get(
+                        'sparse_beta2_decay_rate', 0.999
+                    )
+                    table.accessor.sparse_commonsgd_param.adam.ada_epsilon = (
+                        strategy.get('sparse_ada_epsilon', 1e-8)
+                    )
+                    if strategy.get('sparse_weight_bounds') is None:
+                        table.accessor.sparse_commonsgd_param.adam.weight_bounds.extend(
+                            [-10, 10]
+                        )
+                    else:
+                        table.accessor.sparse_commonsgd_param.adam.weight_bounds.extend(
+                            strategy.get('sparse_weight_bounds')
+                        )
+                converter = strategy.get(
+                    'sparse_converter',
+                    "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)",
+                )
+                deconverter = strategy.get(
+                    'sparse_deconverter',
+                    "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)",
+                )
+
+                table1 = table.accessor.table_accessor_save_param.add()
+                table1.param = 1
+                table1.converter = converter
+                table1.deconverter = deconverter
+
+                table2 = table.accessor.table_accessor_save_param.add()
+                table2.param = 2
+                table2.converter = converter
+                table2.deconverter = deconverter
+            elif (
+                accessor_class == 'DownpourUnitAccessor'
+                or accessor_class == 'DownpourDoubleUnitAccessor'
+                or accessor_class == 'DownpourCtrDymfAccessor'
+            ):
+                self.add_sparse_table_common_config(table, strategy)
+                self.add_sparse_optimizer(
+                    table.accessor.embed_sgd_param, strategy, "embed_"
+                )
+                self.add_sparse_optimizer(
+                    table.accessor.embedx_sgd_param, strategy, "embedx_"
+                )
+
+    def add_dense_table(
+        self, table_id, param_var, grad_var, strategy, sparse_table_names
+    ):
+        """
+        Args:
+            table_id(int): id of sparse params table
+            param_var(list): param vars
+            grad_var(list): param grad vars
+            strategy(dict): the dense config dict
+            sparse_table_names(list): sparse table names
+        Returns:
+            return None
+        """
+        fea_dim = 0
+        dense_param_vars = []
+        for p in param_var:
+            if p.name not in sparse_table_names:
+                dense_param_vars.append(p)
+
+        for param in dense_param_vars:
+            fea_dim += reduce(lambda x, y: x * y, param.shape, 1)
+
+        for table in self._server.downpour_server_param.downpour_table_param:
+            if table.table_id == table_id:
+                if table.type == pslib.PS_DENSE_TABLE:
+                    table.accessor.fea_dim = fea_dim
+                    return
+                else:
+                    raise ValueError(
+                        f"expect table {table_id} type={pslib.PS_DENSE_TABLE}, but actual type={table.type}"
+                    )
+
+        if strategy is None:
+            strategy = {}
+        table = self._server.downpour_server_param.downpour_table_param.add()
+        table.table_id = table_id
+        support_dense_key_list = [
+            'dense_table_class',
+            'dense_compress_in_save',
+            'dense_accessor_class',
+            'dense_optimizer',
+            'dense_learning_rate',
+            'dense_avg_decay',
+            'dense_ada_decay',
+            'dense_ada_epsilon',
+            'dense_mom_decay',
+            'dense_naive_lr',
+        ]
+
+        for key in strategy:
+            if key not in support_dense_key_list:
+                raise ValueError(f"strategy key '{key}' not support")
+
+        table.table_class = strategy.get(
+            'dense_table_class', "DownpourDenseTable"
+        )
+        table.type = pslib.PS_DENSE_TABLE
+        table.compress_in_save = strategy.get('dense_compress_in_save', True)
+        table.accessor.accessor_class = strategy.get(
+            'dense_accessor_class', "DownpourDenseValueAccessor"
+        )
+        table.accessor.dense_sgd_param.name = strategy.get(
+            'dense_optimizer', "adam"
+        )
+        table.accessor.dense_sgd_param.adam.learning_rate = strategy.get(
+            'dense_learning_rate', 5e-06
+        )
+        table.accessor.dense_sgd_param.adam.avg_decay_rate = strategy.get(
+            'dense_avg_decay', 0.999993
+        )
+        table.accessor.dense_sgd_param.adam.ada_decay_rate = strategy.get(
+            'dense_ada_decay', 0.9999
+        )
+        table.accessor.dense_sgd_param.adam.ada_epsilon = strategy.get(
+            'dense_ada_epsilon', 1e-8
+        )
+        table.accessor.dense_sgd_param.adam.mom_decay_rate = strategy.get(
+            'dense_mom_decay', 0.99
+        )
+        table.accessor.dense_sgd_param.naive.learning_rate = strategy.get(
+            'dense_naive_lr', 0.0002
+        )
+        table.accessor.fea_dim = fea_dim
+
+    def add_data_norm_table(
+        self,
+        table_id,
+        learning_rate,
+        param_var,
+        grad_var,
+        strategy,
+        sparse_table_names,
+    ):
+        """
+        Args:
+            table_id(int): id of datanorm table
+            learning_rate(float): the learning rate used to update parameters
+            param_var(list): param vars
+            grad_var(list): param grad vars
+            strategy(dict): the datanorm config dict
+            sparse_table_names(list): sparse table names
+        Returns:
+            return None
+        """
+        fea_dim = 0
+        dense_param_vars = []
+        for p in param_var:
+            if p.name not in sparse_table_names:
+                dense_param_vars.append(p)
+
+        for param in dense_param_vars:
+            fea_dim += reduce(lambda x, y: x * y, param.shape, 1)
+
+        for table in self._server.downpour_server_param.downpour_table_param:
+            if table.table_id == table_id:
+                if table.type == pslib.PS_DENSE_TABLE:
+                    table.accessor.fea_dim = fea_dim
+                    return
+                else:
+                    raise ValueError(
+                        f"expect table {table_id} type={pslib.PS_DENSE_TABLE}, but actual type={table.type}"
+                    )
+        if strategy is None:
+            strategy = {}
+
+        support_datanorm_key_list = [
+            'datanorm_table_class',
+            'datanorm_compress_in_save',
+            'datanorm_accessor_class',
+            'datanorm_operation',
+            'datanorm_decay_rate',
+        ]
+
+        for key in strategy:
+            if key not in support_datanorm_key_list:
+                raise ValueError(f"strategy key '{key}' not support")
+
+        table = self._server.downpour_server_param.downpour_table_param.add()
+        table.table_id = table_id
+        table.table_class = strategy.get(
+            'datanorm_table_class', 'DownpourDenseTable'
+        )
+        table.type = pslib.PS_DENSE_TABLE
+        table.compress_in_save = strategy.get('datanorm_compress_in_save', True)
+        table.accessor.accessor_class = strategy.get(
+            'datanorm_accessor_class', 'DownpourDenseValueAccessor'
+        )
+        table.accessor.dense_sgd_param.name = strategy.get(
+            'datanorm_operation', 'summary'
+        )
+        table.accessor.dense_sgd_param.summary.summary_decay_rate = (
+            strategy.get('datanorm_decay_rate', 0.999999)
+        )
+        table.accessor.fea_dim = fea_dim
+
+    def add_sparse_optimizer(self, sgd, strategy, prefix):
+        optimizer_name = strategy.get(prefix + "sparse_optimizer", "adagrad")
+        sgd.name = optimizer_name
+        if optimizer_name == "naive":
+            sgd.naive.learning_rate = strategy.get(
+                prefix + 'sparse_learning_rate', 0.05
+            )
+            sgd.naive.initial_range = strategy.get(
+                prefix + 'sparse_initial_range', 1e-4
+            )
+            bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10])
+            sgd.naive.weight_bounds.extend(bounds)
+        elif optimizer_name == "adagrad":
+            sgd.adagrad.learning_rate = strategy.get(
+                prefix + 'sparse_learning_rate', 0.05
+            )
+            sgd.adagrad.initial_range = strategy.get(
+                prefix + 'sparse_initial_range', 1e-4
+            )
+            if prefix == "embed_":
+                sgd.adagrad.initial_range = 0
+            sgd.adagrad.initial_g2sum = strategy.get(
+                prefix + 'sparse_initial_g2sum', 3
+            )
+            bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10])
+            sgd.adagrad.weight_bounds.extend(bounds)
+        elif optimizer_name == "std_adagrad":
+            sgd.adagrad.learning_rate = strategy.get(
+                prefix + 'sparse_learning_rate', 0.05
+            )
+            sgd.adagrad.initial_range = strategy.get(
+                prefix + 'sparse_initial_range', 1e-4
+            )
+            if prefix == "embed_":
+                sgd.adagrad.initial_range = 0
+            sgd.adagrad.initial_g2sum = strategy.get(
+                prefix + 'sparse_initial_g2sum', 3
+            )
+            bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10])
+            sgd.adagrad.weight_bounds.extend(bounds)
+        elif optimizer_name == "adam":
+            sgd.adam.learning_rate = strategy.get(
+                prefix + 'sparse_learning_rate', 0.001
+            )
+            sgd.adam.initial_range = strategy.get(
+                prefix + 'sparse_initial_range', 1e-4
+            )
+            sgd.adam.beta1_decay_rate = strategy.get(
+                prefix + 'sparse_beta1_decay_rate', 0.9
+            )
+            sgd.adam.beta2_decay_rate = strategy.get(
+                prefix + 'sparse_beta2_decay_rate', 0.999
+            )
+            sgd.adam.ada_epsilon = strategy.get(
+                prefix + 'sparse_ada_epsilon', 1e-8
+            )
+            bounds = strategy.get(prefix + 'sparse_weight_bounds', [-10, 10])
+            sgd.adam.weight_bounds.extend(bounds)
+
+    def add_sparse_table_common_config(self, table, strategy):
+        table.accessor.embedx_dim = strategy.get('sparse_embedx_dim', 8)
+        table.accessor.embedx_threshold = strategy.get(
+            'sparse_embedx_threshold', 10
+        )
+        table.accessor.fea_dim = int(table.accessor.embedx_dim) + 3
+        table.accessor.downpour_accessor_param.nonclk_coeff = strategy.get(
+            'sparse_nonclk_coeff', 0.1
+        )
+        table.accessor.downpour_accessor_param.click_coeff = strategy.get(
+            'sparse_click_coeff', 1
+        )
+        table.accessor.downpour_accessor_param.base_threshold = strategy.get(
+            'sparse_base_threshold', 1.5
+        )
+        table.accessor.downpour_accessor_param.delta_threshold = strategy.get(
+            'sparse_delta_threshold', 0.25
+        )
+        table.accessor.downpour_accessor_param.delta_keep_days = strategy.get(
+            'sparse_delta_keep_days', 16
+        )
+        table.accessor.downpour_accessor_param.delete_after_unseen_days = (
+            strategy.get('sparse_delete_after_unseen_days', 30)
+        )
+        table.accessor.downpour_accessor_param.show_click_decay_rate = (
+            strategy.get('sparse_show_click_decay_rate', 0.98)
+        )
+        table.accessor.downpour_accessor_param.delete_threshold = strategy.get(
+            'sparse_delete_threshold', 0.8
+        )
+        converter = strategy.get(
+            'sparse_converter',
+            "(scripts/xbox_compressor_mf.py | bin/xbox_pb_converter)",
+        )
+        deconverter = strategy.get(
+            'sparse_deconverter',
+            "(bin/xbox_pb_deconverter | scripts/xbox_decompressor_mf.awk)",
+        )
+
+        table1 = table.accessor.table_accessor_save_param.add()
+        table1.param = 1
+        table1.converter = converter
+        table1.deconverter = deconverter
+
+        table2 = table.accessor.table_accessor_save_param.add()
+        table2.param = 2
+        table2.converter = converter
+        table2.deconverter = deconverter
+
+    def get_desc(self):
+        """
+        Return downpour server program_desc
+        """
+        return self._server
+
+
+class DownpourWorker(Worker):
+    """
+    DownpourWorker class is used to generate worker program_desc
+    Args:
+        window (int): push params frequency
+        worker: it is pslib.DownpourTrainerParameter
+    Examples:
+        worker = DownpourWorker(1)
+    """
+
+    def __init__(self, window):
+        self.window = window
+        self._worker = pslib.DownpourTrainerParameter()
+
+    def add_sparse_table(
+        self, table_id, slot_key_vars, slot_value_vars, slot_value_grads=None
+    ):
+        """
+        Args:
+            table_id(int): id of sparse params table
+            slot_key_vars(list): slot key id
+            slot_value_vars(list): slot key value after embedding
+            slot_value_grads(list): grad of all params, default is None
+        Returns:
+            return None
+        """
+        if slot_value_grads is None:
+            slot_value_grad_names = [
+                var.name + "@GRAD" for var in slot_value_vars
+            ]
+        else:
+            value_to_key = {}
+            for i in range(len(slot_key_vars)):
+                value_to_key[slot_value_vars[i].name] = slot_key_vars[i]
+            slot_value_grad_names = []
+            all_grad_names = [var.name for var in slot_value_grads]
+            for var in slot_value_vars:
+                if var.name + "@GRAD" in all_grad_names:
+                    slot_value_grad_names.append(var.name + "@GRAD")
+            sorted_slot_value_vars = [
+                i
+                for i in slot_value_vars
+                if i.name + "@GRAD" in slot_value_grad_names
+            ]
+            sorted_slot_value_vars += [
+                i
+                for i in slot_value_vars
+                if i.name + "@GRAD" not in slot_value_grad_names
+            ]
+            sorted_slot_key_vars = [
+                value_to_key[v.name] for v in sorted_slot_value_vars
+            ]
+
+        target_table = None
+        for table in self._worker.sparse_table:
+            if table.table_id == table_id:
+                keys = table.slot_key
+                key_names = [var.name for var in sorted_slot_key_vars]
+                for key_name in key_names:
+                    if key_name not in keys:
+                        raise ValueError(
+                            f"sparse table {table_id} slot_key error"
+                        )
+                target_table = table
+                break
+
+        table = target_table
+        if table is not None:
+            self._worker.sparse_table.remove(table)
+        table = self._worker.sparse_table.add()
+        table.table_id = table_id
+        table.slot_key.extend([var.name for var in sorted_slot_key_vars])
+        table.slot_value.extend([var.name for var in sorted_slot_value_vars])
+        table.slot_gradient.extend(slot_value_grad_names)
+
+    def add_dense_table(
+        self,
+        table_id,
+        learning_rate,
+        param_vars,
+        grad_vars,
+        dense_start_table_id,
+        sparse_table_names,
+    ):
+        r"""
+        Args:
+            table_id(int): id of sparse params table
+            learning_rate(float): the learning rate used to update parameters. \
+                Can be a float value
+            param_vars(list): all dense param. it is a list.
+            grad_vars(list): all dense grad param it is a list.
+            dense_start_table_id(int): dense table start index
+            sparse_table_names(list): sparse table names
+        Returns:
+            return None
+        """
+        sparse_table_name_grad = []
+        for name in sparse_table_names:
+            sparse_table_name_grad.append(name + "@GRAD")
+
+        dense_param_name = []
+        for p in param_vars:
+            if p.name not in sparse_table_names:
+                dense_param_name.append(p.name)
+
+        dense_grad_name = []
+        for g in grad_vars:
+            if g.name not in sparse_table_name_grad:
+                dense_grad_name.append(g.name)
+
+        dense_param_name.sort()
+        dense_grad_name.sort()
+
+        for table in self._worker.dense_table:
+            if table.table_id == table_id:
+                desc_dense_param_name = list(table.dense_variable_name)
+                desc_dense_param_name.sort()
+
+                if dense_param_name == desc_dense_param_name:
+                    desc_dense_grad_name = list(
+                        table.dense_gradient_variable_name
+                    )
+                    desc_dense_grad_name.sort()
+                    if dense_grad_name == desc_dense_grad_name:
+                        return
+                    else:
+                        raise ValueError(
+                            f"dense table {table_id} dense_gradient_variable_name "
+                            "error"
+                        )
+                else:
+                    raise ValueError(
+                        f"dense table {table_id} dense_variable_name error"
+                    )
+
+        table = self._worker.dense_table.add()
+        table.table_id = table_id
+
+        # def cmp_fc(x, y):
+        #    if x.startswith("fc_") and y.startswith("fc_"):
+        #        index_x = x.find('.')
+        #        index_y = y.find('.')
+        #        if index_x > 0 and index_y > 0:
+        #            num_x = x[3:index_x]
+        #            num_y = y[3:index_y]
+        #            if num_x.isdigit() and num_y.isdigit():
+        #                if int(num_x) < int(num_y):
+        #                    return -1
+        #                if int(num_x) > int(num_y):
+        #                    return 1
+        #                if x[index_x + 1] == 'w' and y[index_y + 1] == 'b':
+        #                    return -1
+        #                if x[index_x + 1] == 'b' and y[index_y + 1] == 'w':
+        #                    return 1
+        #    if x < y:
+        #        return -1
+        #    else:
+        #        return 1
+
+        # table.dense_variable_name.extend(sorted(dense_param_name, cmp_fc))
+        # table.dense_gradient_variable_name.extend(
+        #    sorted(dense_grad_name, cmp_fc))
+        table.dense_variable_name.extend(dense_param_name)
+        table.dense_gradient_variable_name.extend(dense_grad_name)
+
+    def get_desc(self):
+        """
+        Return downpour worker program_desc
+        """
+        return self._worker