// Copyright (c) 2023 CINN Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "paddle/cinn/optim/resize_buffer.h" #include #include "paddle/cinn/common/integer_set.h" #include "paddle/cinn/ir/ir.h" #include "paddle/cinn/ir/ir_mutator.h" #include "paddle/cinn/ir/ir_printer.h" #include "paddle/cinn/ir/op/ir_operators.h" #include "paddle/cinn/ir/utils/ir_copy.h" #include "paddle/cinn/optim/ir_simplify.h" #include "paddle/cinn/optim/replace_mod_to_max.h" #include "paddle/cinn/optim/replace_var_with_expr.h" #include "paddle/cinn/utils/string.h" namespace cinn { namespace optim { class AnalyzeLoopVarRange : public ir::IRMutator<> { public: void operator()(ir::Expr* expr) { ir::IRMutator<>::Visit(expr, expr); } void Visit(const ir::IfThenElse* op, Expr* expr) override { PADDLE_ENFORCE_NOT_NULL( expr->As(), ::common::errors::InvalidArgument( "The expression could not be cast to ir::IfThenElse. Please check " "the expression type.")); const ir::IfThenElse* if_ir = expr->As(); const ir::LT* less_than_ir = if_ir->condition.As(); if (less_than_ir != nullptr) { std::stringstream oss; oss << less_than_ir->a(); std::string var_name = oss.str(); if (utils::StartsWith(var_name, "blockIdx") || utils::StartsWith(var_name, "threadIdx")) { var_name_to_extent_[var_name] = less_than_ir->b(); } } ir::IRMutator<>::Visit(op, expr); } // Visit for and collect extent void Visit(const ir::For* op, Expr* expr) override { PADDLE_ENFORCE_NOT_NULL(expr->As(), ::common::errors::InvalidArgument( "The expression could not be cast to ir::For. " "Please check the expression type.")); ir::For* for_ir = expr->As(); std::string var_name = for_ir->loop_var->name; Expr extent = for_ir->extent; var_name_to_extent_[var_name] = extent; if (for_ir->is_binded()) { const ir::BindInfo& bind_info = for_ir->bind_info(); if (bind_info.valid()) { std::string bind_var_str = static_cast(bind_info); var_name_to_extent_[bind_var_str] = extent; } } ir::IRMutator<>::Visit(op, expr); } // Analyze the buffer access inside store void Visit(const ir::Store* op, Expr* expr) override { ir::Store* store = expr->As(); ir::Tensor tensor = store->tensor.as_tensor_ref(); AnalyzeTensorRange(store->indices, tensor); AnalyzeBufferSize(store->indices, tensor); ir::IRMutator<>::Visit(op, expr); } // Analyze the buffer access inside load void Visit(const ir::Load* op, Expr* expr) override { ir::Load* load = expr->As(); ir::Tensor tensor = load->tensor.as_tensor_ref(); AnalyzeTensorRange(load->indices, tensor); ir::IRMutator<>::Visit(op, expr); } void Visit(const ir::ScheduleBlockRealize* x, Expr* expr) override { const ir::ScheduleBlock* schedule_block = x->schedule_block.As(); const std::vector& iter_vars = schedule_block->iter_vars; const std::vector& iter_values = x->iter_values; for (int i = 0; i < iter_vars.size(); ++i) { const std::string& var_name = iter_vars[i]->name; VLOG(6) << "Analyzing var_name = " << var_name << ", expression = " << iter_values[i]; Expr bind_value = MaxIndexRange(iter_values[i]); VLOG(6) << "Get extent of " << var_name << ", bind_value = " << bind_value; var_name_to_extent_[var_name] = bind_value; } ir::IRMutator<>::Visit(x, expr); } private: void AnalyzeTensorRange(const std::vector& indices, const ir::Tensor& tensor) { if (!tensor->buffer.defined()) return; if (tensor->buffer->memory_type == ir::MemoryType::Heap) return; std::vector indice_extent; for (int i = 0; i < indices.size(); ++i) { Expr simplified_idx_extent = MaxIndexRange(indices[i]); indice_extent.push_back(simplified_idx_extent); } std::string buffer_name = tensor->buffer->name; if (buffer_name_to_indice_extent.count(buffer_name)) { std::vector& stored_indice_extent = buffer_name_to_indice_extent[buffer_name]; if (indice_extent.size() > stored_indice_extent.size()) { // multi-dimension access vs single index access, we treat // multi-dimension access as better buffer size computation. buffer_name_to_indice_extent[buffer_name] = indice_extent; } else if (indice_extent.size() == stored_indice_extent.size()) { for (int i = 0; i < indice_extent.size(); ++i) { if (stored_indice_extent[i].is_constant() && indice_extent[i].is_constant()) { int64_t stored_extent = stored_indice_extent[i].as_int64(); int64_t cur_extent = indice_extent[i].as_int64(); if (cur_extent > stored_extent) { stored_indice_extent[i] = ir::Expr(cur_extent); stored_indice_extent[i]->set_type(indice_extent[i].type()); } } // if there indice extent is not constant, which means dynamic shape // we don't change the value now. } } } else { buffer_name_to_indice_extent[buffer_name] = indice_extent; } VLOG(6) << "buffer_name = " << buffer_name << ", indice_extent = " << buffer_name_to_indice_extent[buffer_name]; } void AnalyzeBufferSize(const std::vector& indices, const ir::Tensor& tensor) { if (!tensor->buffer.defined()) return; if (tensor->buffer->memory_type == ir::MemoryType::Heap) return; const std::string& buffer_name = tensor->buffer->name; buffer_name_to_size[buffer_name] = AnalyzeBufferSize(indices); VLOG(6) << "buffer_name = " << buffer_name << ", size = " << buffer_name_to_size[buffer_name]; } ir::Expr AnalyzeBufferSize(const std::vector& indices) { const auto GetIterVarNames = [](const std::vector& indices) -> std::set { std::set iter_var_names; for (const ir::Expr& e : indices) { ir::ir_utils::CollectIRNodes(e, [&](const ir::Expr* x) { if (x->as_var() && !x->as_var()->is_symbolic_constant) { iter_var_names.insert(x->as_var()->name); } return false; }); } return iter_var_names; }; std::set iter_var_names = GetIterVarNames(indices); ir::Expr size(1); for (const std::string& var_name : iter_var_names) { PADDLE_ENFORCE_GT(var_name_to_extent_.count(var_name), 0, ::common::errors::PreconditionNotMet( "Cannot find the extent of var %s", var_name)); size = optim::ArithSimplify(size * var_name_to_extent_.at(var_name)); } return size; } // A recursion function to calculate the max index range // The index may contain some vars like index = 8 * i / j, where we know the // range of i, j, we search all values to get the max index range Expr MaxIndexRange(const ir::Expr& index) { ir::Expr copy = ir::ir_utils::IRCopy(index); std::vector vars = ir::ir_utils::CollectIRNodesInOrder( copy, [](const ir::Expr* expr) { return expr->As(); }); // We only use the maximal of var, maximal of Mod operation, // which may not be the maximal of index // mathematically, but it works for current CINN. // // We may add better computation of MaxIndexRange if we need for (int i = 0; i < vars.size(); ++i) { for (auto kv : var_name_to_extent_) { auto var_name = vars[i].as_var_ref()->name; if (var_name_to_extent_.count(var_name) != 0) { Expr max_var_value = ir::Sub::Make( var_name_to_extent_.at(vars[i].as_var_ref()->name), ir::Expr(1)); ReplaceModToMax(©); ReplaceVarWithExpr(©, vars[i], max_var_value); } } } ir::Expr tmp = ir::Add::Make(copy, ir::Expr(1)); ir::Expr simplified = optim::ArithSimplify(tmp); if (simplified.As()) { ir::Expr lhs = simplified.As()->a(); ir::Expr rhs = simplified.As()->b(); common::cas_intervals_t var_intervals = common::CollectVarIntervalsOfExprs({lhs, rhs}); common::SymbolicExprAnalyzer analyzer(var_intervals); if (analyzer.ProveLE(lhs, rhs)) { return lhs; } else if (analyzer.ProveGE(lhs, rhs)) { return rhs; } } return simplified; } public: std::unordered_map> buffer_name_to_indice_extent; std::unordered_map buffer_name_to_size; private: std::unordered_map var_name_to_extent_; }; class ResizeBufferFromAnalyzedRange : public ir::IRMutator<> { public: ResizeBufferFromAnalyzedRange( const std::unordered_map>& buffer_name_to_shape, const std::unordered_map& buffer_name_to_size) : buffer_name_to_shape_(buffer_name_to_shape), buffer_name_to_size_(buffer_name_to_size) {} void operator()(ir::Expr* expr) { ir::IRMutator<>::Visit(expr, expr); } void Visit(const ir::Store* op, Expr* expr) override { ir::Store* store = expr->As(); ir::Tensor tensor = store->tensor.as_tensor_ref(); ResizeTensor(&tensor); ReplaceTensorIndices(store); ir::IRMutator<>::Visit(op, expr); } void Visit(const ir::Load* op, Expr* expr) override { auto load = expr->As(); if (!load->tensor.as_tensor_ref()->buffer.defined()) { return; } if (load->tensor.as_tensor_ref()->buffer->memory_type == ir::MemoryType::Heap) { ir::IRMutator<>::Visit(op, expr); return; } ir::Tensor tensor = load->tensor.as_tensor_ref(); ResizeTensor(&tensor); // For the moment, align the load tensor indices with the tensor shape using // the trick method. A better way would be to modify the FlattenLoop // Schedule. int cnt = load->indices.size() - load->tensor.as_tensor_ref()->shape.size(); for (int i = 0; i < cnt; i++) { load->indices.erase(load->indices.begin()); } ReplaceTensorIndices(load); ir::IRMutator<>::Visit(op, expr); } private: void ResizeTensor(ir::Tensor* tensor_ptr) { ir::Buffer buffer = (*tensor_ptr)->buffer; if (!buffer.defined()) return; if (buffer->memory_type == ir::MemoryType::Heap) return; const std::string& buffer_name = buffer->name; if (buffer_name_to_shape_.count(buffer_name)) { const std::vector& analyzed_shape = buffer_name_to_shape_.at(buffer_name); VLOG(6) << "Replacing shape of tensor " << (*tensor_ptr)->name << " with shape " << analyzed_shape; (*tensor_ptr)->shape = analyzed_shape; buffer->shape = analyzed_shape; } if (buffer_name_to_size_.count(buffer_name) > 0) { const ir::Expr& analyzed_size = buffer_name_to_size_.at(buffer_name); VLOG(6) << "Replacing shape of buffer " << buffer->name << " with shape " << analyzed_size; buffer->shape = {analyzed_size}; } } template void ReplaceTensorIndices(T* op) { ir::Tensor tensor = op->tensor.as_tensor_ref(); ir::Buffer buffer = tensor->buffer; if (!buffer.defined()) return; if (buffer->memory_type != ir::MemoryType::GPULocal) return; VLOG(4) << "replacing index of tensor: " << tensor->name; ir::Expr index_expr = op->index(); std::unordered_map var_name_to_expr; ir::ir_utils::CollectIRNodes(index_expr, [&](const ir::Expr* x) { const ir::_Var_* var = x->as_var(); if (var) { var_name_to_expr[var->name] = var->Copy(); } return false; }); if (var_name_to_expr.size() != 1) { return; } ir::Expr single_var = var_name_to_expr.begin()->second; VLOG(4) << "found single var: " << single_var; for (size_t i = 0; i + 1 < op->indices.size(); i++) { op->indices[i] = ir::Expr(0); } op->indices.back() = single_var; } private: const std::unordered_map>& buffer_name_to_shape_; const std::unordered_map& buffer_name_to_size_; }; void ResizeBufferToMaxVarRange(ir::Expr* expr) { VLOG(6) << "Before ResizeBufferToMaxVarRange, Expr = \n" << *expr; AnalyzeLoopVarRange analyze_functor; analyze_functor(expr); ResizeBufferFromAnalyzedRange resize_functor( analyze_functor.buffer_name_to_indice_extent, analyze_functor.buffer_name_to_size); resize_functor(expr); VLOG(6) << "After ResizeBufferToMaxVarRange, Expr = \n" << *expr; } } // namespace optim } // namespace cinn